From 04d8bc2c9b043efa1529d06e2f4c3052b8953357 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Mon, 6 Nov 2017 19:49:46 -0800
Subject: [PATCH 001/161] First pass at abstract base flow control class

---
 .../chttp2/transport/chttp2_transport.cc      |  8 +-
 .../transport/chttp2/transport/flow_control.h | 95 +++++++++++--------
 .../ext/transport/chttp2/transport/internal.h |  8 +-
 3 files changed, 70 insertions(+), 41 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 63ac65ac788..a239c6ee4be 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -544,7 +544,8 @@ static void init_transport(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
     }
   }
 
-  t->flow_control.Init(exec_ctx, t, enable_bdp);
+  t->flow_control.Init<grpc_core::chttp2::TransportFlowControl>(exec_ctx, t,
+                                                                enable_bdp);
 
   /* No pings allowed before receiving a header or data frame. */
   t->ping_state.pings_before_data_required = 0;
@@ -716,7 +717,10 @@ static int init_stream(grpc_exec_ctx* exec_ctx, grpc_transport* gt,
     post_destructive_reclaimer(exec_ctx, t);
   }
 
-  s->flow_control.Init(t->flow_control.get(), s);
+  s->flow_control.Init<grpc_core::chttp2::StreamFlowControl>(
+      static_cast<grpc_core::chttp2::TransportFlowControl *>(
+          t->flow_control.get()),
+      s);
   GPR_TIMER_END("init_stream", 0);
 
   return 0;
diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h
index 2515c94309a..3d9f3a181f8 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.h
+++ b/src/core/ext/transport/chttp2/transport/flow_control.h
@@ -24,6 +24,7 @@
 
 #include <grpc/support/useful.h>
 #include "src/core/ext/transport/chttp2/transport/http2_settings.h"
+#include "src/core/lib/support/abstract.h"
 #include "src/core/lib/support/manual_constructor.h"
 #include "src/core/lib/transport/bdp_estimator.h"
 #include "src/core/lib/transport/pid_controller.h"
@@ -132,7 +133,30 @@ class FlowControlTrace {
   int64_t announced_window_delta_;
 };
 
-class TransportFlowControl {
+class TransportFlowControlBase {
+ public:
+  TransportFlowControlBase() {}
+  virtual ~TransportFlowControlBase() {}
+  virtual uint32_t MaybeSendUpdate(bool writing_anyway) {abort();}
+  virtual FlowControlAction MakeAction() {abort();}
+  virtual FlowControlAction PeriodicUpdate(grpc_exec_ctx* exec_ctx) {abort();}
+  virtual void StreamSentData(int64_t size) {abort();}
+  virtual grpc_error* RecvData(int64_t incoming_frame_size) {abort();}
+  virtual void RecvUpdate(uint32_t size) {abort();}
+  virtual BdpEstimator* bdp_estimator() { return nullptr; }
+  int64_t remote_window() const { return remote_window_; }
+  virtual int64_t target_window() const { return target_initial_window_size_; }
+  int64_t announced_window()  const{ return announced_window_; }
+
+  GRPC_ABSTRACT_BASE_CLASS
+
+ protected:
+  int64_t remote_window_ = kDefaultWindow;
+  int64_t target_initial_window_size_ = kDefaultWindow;
+  int64_t announced_window_ = kDefaultWindow;
+};
+
+class TransportFlowControl final : public TransportFlowControlBase {
  public:
   TransportFlowControl(grpc_exec_ctx* exec_ctx, const grpc_chttp2_transport* t,
                        bool enable_bdp_probe);
@@ -144,25 +168,25 @@ class TransportFlowControl {
   // else returns zero; writing_anyway indicates if a write would happen
   // regardless of the send - if it is false and this function returns non-zero,
   // this announce will cause a write to occur
-  uint32_t MaybeSendUpdate(bool writing_anyway);
+  uint32_t MaybeSendUpdate(bool writing_anyway) override;
 
   // Reads the flow control data and returns and actionable struct that will
   // tell chttp2 exactly what it needs to do
-  FlowControlAction MakeAction() { return UpdateAction(FlowControlAction()); }
+  FlowControlAction MakeAction() override { return UpdateAction(FlowControlAction()); }
 
   // Call periodically (at a low-ish rate, 100ms - 10s makes sense)
   // to perform more complex flow control calculations and return an action
   // to let chttp2 change its parameters
-  FlowControlAction PeriodicUpdate(grpc_exec_ctx* exec_ctx);
+  FlowControlAction PeriodicUpdate(grpc_exec_ctx* exec_ctx) override;
 
-  void StreamSentData(int64_t size) { remote_window_ -= size; }
+  void StreamSentData(int64_t size) override { remote_window_ -= size; }
 
   grpc_error* ValidateRecvData(int64_t incoming_frame_size);
   void CommitRecvData(int64_t incoming_frame_size) {
     announced_window_ -= incoming_frame_size;
   }
 
-  grpc_error* RecvData(int64_t incoming_frame_size) {
+  grpc_error* RecvData(int64_t incoming_frame_size) override {
     FlowControlTrace trace("  data recv", this, nullptr);
     grpc_error* error = ValidateRecvData(incoming_frame_size);
     if (error != GRPC_ERROR_NONE) return error;
@@ -171,18 +195,16 @@ class TransportFlowControl {
   }
 
   // we have received a WINDOW_UPDATE frame for a transport
-  void RecvUpdate(uint32_t size) {
+  void RecvUpdate(uint32_t size) override {
     FlowControlTrace trace("t updt recv", this, nullptr);
     remote_window_ += size;
   }
 
-  int64_t remote_window() const { return remote_window_; }
-  int64_t target_window() const {
+  int64_t target_window() const override {
     return (uint32_t)GPR_MIN((int64_t)((1u << 31) - 1),
                              announced_stream_total_over_incoming_window_ +
                                  target_initial_window_size_);
   }
-  int64_t announced_window() const { return announced_window_; }
 
   const grpc_chttp2_transport* transport() const { return t_; }
 
@@ -202,7 +224,7 @@ class TransportFlowControl {
     }
   }
 
-  BdpEstimator* bdp_estimator() { return &bdp_estimator_; }
+  BdpEstimator* bdp_estimator() override { return &bdp_estimator_; }
 
   void TestOnlyForceHugeWindow() {
     announced_window_ = 1024 * 1024 * 1024;
@@ -226,9 +248,6 @@ class TransportFlowControl {
 
   const grpc_chttp2_transport* const t_;
 
-  /** Our bookkeeping for the remote peer's available window */
-  int64_t remote_window_ = kDefaultWindow;
-
   /** calculating what we should give for local window:
       we track the total amount of flow control over initial window size
       across all streams: this is data that we want to receive right now (it
@@ -240,13 +259,6 @@ class TransportFlowControl {
   int64_t announced_stream_total_over_incoming_window_ = 0;
   int64_t announced_stream_total_under_incoming_window_ = 0;
 
-  /** This is out window according to what we have sent to our remote peer. The
-   * difference between this and target window is what we use to decide when
-   * to send WINDOW_UPDATE frames. */
-  int64_t announced_window_ = kDefaultWindow;
-
-  int32_t target_initial_window_size_ = kDefaultWindow;
-
   /** should we probe bdp? */
   const bool enable_bdp_probe_;
 
@@ -258,7 +270,31 @@ class TransportFlowControl {
   grpc_millis last_pid_update_ = 0;
 };
 
-class StreamFlowControl {
+class StreamFlowControlBase {
+ public:
+  StreamFlowControlBase() {}
+  virtual ~StreamFlowControlBase() {}
+  virtual FlowControlAction UpdateAction(FlowControlAction action) {abort();}
+  virtual FlowControlAction MakeAction() {abort();}
+  virtual void SentData(int64_t outgoing_frame_size) {abort();}
+  virtual grpc_error* RecvData(int64_t incoming_frame_size) {abort();}
+  virtual uint32_t MaybeSendUpdate() {abort();}
+  virtual void RecvUpdate(uint32_t size) {abort();}
+  virtual void IncomingByteStreamUpdate(size_t max_size_hint,
+                                        size_t have_already) {abort();}
+  int64_t remote_window_delta() { return remote_window_delta_; }
+  int64_t local_window_delta() { return local_window_delta_; }
+  int64_t announced_window_delta() { return announced_window_delta_; }
+
+  GRPC_ABSTRACT_BASE_CLASS
+
+ protected:
+  int64_t remote_window_delta_ = 0;
+  int64_t local_window_delta_ = 0;
+  int64_t announced_window_delta_ = 0;
+};
+
+class StreamFlowControl final : public StreamFlowControlBase {
  public:
   StreamFlowControl(TransportFlowControl* tfc, const grpc_chttp2_stream* s);
   ~StreamFlowControl() {
@@ -314,21 +350,6 @@ class StreamFlowControl {
     announced_window_delta_ += change;
     tfc->PostUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_);
   }
-
-  /** window available for us to send to peer, over or under the initial
-   * window
-   * size of the transport... ie:
-   * remote_window = remote_window_delta + transport.initial_window_size */
-  int64_t remote_window_delta_ = 0;
-
-  /** window available for peer to send to us (as a delta on
-   * transport.initial_window_size)
-   * local_window = local_window_delta + transport.initial_window_size */
-  int64_t local_window_delta_ = 0;
-
-  /** window available for peer to send to us over this stream that we have
-   * announced to the peer */
-  int64_t announced_window_delta_ = 0;
 };
 
 }  // namespace chttp2
diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h
index f6fd6795d0a..0441f1c5413 100644
--- a/src/core/ext/transport/chttp2/transport/internal.h
+++ b/src/core/ext/transport/chttp2/transport/internal.h
@@ -351,7 +351,9 @@ struct grpc_chttp2_transport {
   /** parser for goaway frames */
   grpc_chttp2_goaway_parser goaway_parser;
 
-  grpc_core::ManualConstructor<grpc_core::chttp2::TransportFlowControl>
+  grpc_core::PolymorphicManualConstructor<
+      grpc_core::chttp2::TransportFlowControlBase,
+      grpc_core::chttp2::TransportFlowControl>
       flow_control;
   /** initial window change. This is tracked as we parse settings frames from
    * the remote peer. If there is a positive delta, then we will make all
@@ -526,7 +528,9 @@ struct grpc_chttp2_stream {
   bool sent_initial_metadata;
   bool sent_trailing_metadata;
 
-  grpc_core::ManualConstructor<grpc_core::chttp2::StreamFlowControl>
+  grpc_core::PolymorphicManualConstructor<
+      grpc_core::chttp2::StreamFlowControlBase,
+      grpc_core::chttp2::StreamFlowControl>
       flow_control;
 
   grpc_slice_buffer flow_controlled_buffer;

From 7ccb79bfbd5afb9ba100643581157486b8510483 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Mon, 27 Nov 2017 10:22:44 -0500
Subject: [PATCH 002/161] First pass at mocked out Flow Control classes

---
 .../chttp2/transport/chttp2_transport.cc      | 37 ++++++++--
 .../transport/chttp2/transport/flow_control.h | 73 +++++++++++++++----
 .../ext/transport/chttp2/transport/internal.h |  6 +-
 3 files changed, 91 insertions(+), 25 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index a239c6ee4be..70b30b7db20 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -544,8 +544,22 @@ static void init_transport(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
     }
   }
 
-  t->flow_control.Init<grpc_core::chttp2::TransportFlowControl>(exec_ctx, t,
-                                                                enable_bdp);
+  // Tune the heck out of this
+  const uint32_t kFrameSize = 1024 * 1024;
+
+  if (true /* disable flow control*/) {
+    t->flow_control.Init<grpc_core::chttp2::TransportFlowControlDisabled>();
+    t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
+        kFrameSize;
+    t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
+        kFrameSize;
+    t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
+        kFrameSize;
+    enable_bdp = false;
+  } else {
+    t->flow_control.Init<grpc_core::chttp2::TransportFlowControl>(exec_ctx, t,
+                                                                  enable_bdp);
+  }
 
   /* No pings allowed before receiving a header or data frame. */
   t->ping_state.pings_before_data_required = 0;
@@ -717,10 +731,14 @@ static int init_stream(grpc_exec_ctx* exec_ctx, grpc_transport* gt,
     post_destructive_reclaimer(exec_ctx, t);
   }
 
-  s->flow_control.Init<grpc_core::chttp2::StreamFlowControl>(
-      static_cast<grpc_core::chttp2::TransportFlowControl *>(
-          t->flow_control.get()),
-      s);
+  if (true /* disable flow control */) {
+    s->flow_control.Init<grpc_core::chttp2::StreamFlowControlDisabled>();
+  } else {
+    s->flow_control.Init<grpc_core::chttp2::StreamFlowControl>(
+        static_cast<grpc_core::chttp2::TransportFlowControl*>(
+            t->flow_control.get()),
+        s);
+  }
   GPR_TIMER_END("init_stream", 0);
 
   return 0;
@@ -2518,8 +2536,11 @@ static void read_action_locked(grpc_exec_ctx* exec_ctx, void* tp,
     grpc_error* errors[3] = {GRPC_ERROR_REF(error), GRPC_ERROR_NONE,
                              GRPC_ERROR_NONE};
     for (; i < t->read_buffer.count && errors[1] == GRPC_ERROR_NONE; i++) {
-      t->flow_control->bdp_estimator()->AddIncomingBytes(
-          (int64_t)GRPC_SLICE_LENGTH(t->read_buffer.slices[i]));
+      grpc_core::BdpEstimator* bdp_est = t->flow_control->bdp_estimator();
+      if (bdp_est) {
+        bdp_est->AddIncomingBytes(
+            (int64_t)GRPC_SLICE_LENGTH(t->read_buffer.slices[i]));
+      }
       errors[1] =
           grpc_chttp2_perform_read(exec_ctx, t, t->read_buffer.slices[i]);
     }
diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h
index 3d9f3a181f8..d28e31d4030 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.h
+++ b/src/core/ext/transport/chttp2/transport/flow_control.h
@@ -137,16 +137,17 @@ class TransportFlowControlBase {
  public:
   TransportFlowControlBase() {}
   virtual ~TransportFlowControlBase() {}
-  virtual uint32_t MaybeSendUpdate(bool writing_anyway) {abort();}
-  virtual FlowControlAction MakeAction() {abort();}
-  virtual FlowControlAction PeriodicUpdate(grpc_exec_ctx* exec_ctx) {abort();}
-  virtual void StreamSentData(int64_t size) {abort();}
-  virtual grpc_error* RecvData(int64_t incoming_frame_size) {abort();}
-  virtual void RecvUpdate(uint32_t size) {abort();}
+  virtual uint32_t MaybeSendUpdate(bool writing_anyway) { abort(); }
+  virtual FlowControlAction MakeAction() { abort(); }
+  virtual FlowControlAction PeriodicUpdate(grpc_exec_ctx* exec_ctx) { abort(); }
+  virtual void StreamSentData(int64_t size) { abort(); }
+  virtual grpc_error* RecvData(int64_t incoming_frame_size) { abort(); }
+  virtual void RecvUpdate(uint32_t size) { abort(); }
+  // TODO(ncteisen): maybe completely encapsulate this inside FlowControl
   virtual BdpEstimator* bdp_estimator() { return nullptr; }
   int64_t remote_window() const { return remote_window_; }
   virtual int64_t target_window() const { return target_initial_window_size_; }
-  int64_t announced_window()  const{ return announced_window_; }
+  int64_t announced_window() const { return announced_window_; }
 
   GRPC_ABSTRACT_BASE_CLASS
 
@@ -156,6 +157,28 @@ class TransportFlowControlBase {
   int64_t announced_window_ = kDefaultWindow;
 };
 
+const int64_t kMaxWindow = (int64_t)((1u << 31) - 1);
+
+class TransportFlowControlDisabled final : public TransportFlowControlBase {
+ public:
+  TransportFlowControlDisabled() {
+    remote_window_ = kMaxWindow;
+    target_initial_window_size_ = kMaxWindow;
+    announced_window_ = kMaxWindow;
+  }
+  virtual uint32_t MaybeSendUpdate(bool writing_anyway) { return 0; }
+  virtual FlowControlAction MakeAction() { return FlowControlAction(); }
+  virtual FlowControlAction PeriodicUpdate(grpc_exec_ctx* exec_ctx) {
+    return FlowControlAction();
+  }
+  virtual void StreamSentData(int64_t size) {}
+  virtual grpc_error* RecvData(int64_t incoming_frame_size) {
+    return GRPC_ERROR_NONE;
+  }
+  virtual void RecvUpdate(uint32_t size) {}
+  virtual int64_t target_window() const { return kMaxWindow; }
+};
+
 class TransportFlowControl final : public TransportFlowControlBase {
  public:
   TransportFlowControl(grpc_exec_ctx* exec_ctx, const grpc_chttp2_transport* t,
@@ -172,7 +195,9 @@ class TransportFlowControl final : public TransportFlowControlBase {
 
   // Reads the flow control data and returns and actionable struct that will
   // tell chttp2 exactly what it needs to do
-  FlowControlAction MakeAction() override { return UpdateAction(FlowControlAction()); }
+  FlowControlAction MakeAction() override {
+    return UpdateAction(FlowControlAction());
+  }
 
   // Call periodically (at a low-ish rate, 100ms - 10s makes sense)
   // to perform more complex flow control calculations and return an action
@@ -274,14 +299,16 @@ class StreamFlowControlBase {
  public:
   StreamFlowControlBase() {}
   virtual ~StreamFlowControlBase() {}
-  virtual FlowControlAction UpdateAction(FlowControlAction action) {abort();}
-  virtual FlowControlAction MakeAction() {abort();}
-  virtual void SentData(int64_t outgoing_frame_size) {abort();}
-  virtual grpc_error* RecvData(int64_t incoming_frame_size) {abort();}
-  virtual uint32_t MaybeSendUpdate() {abort();}
-  virtual void RecvUpdate(uint32_t size) {abort();}
+  virtual FlowControlAction UpdateAction(FlowControlAction action) { abort(); }
+  virtual FlowControlAction MakeAction() { abort(); }
+  virtual void SentData(int64_t outgoing_frame_size) { abort(); }
+  virtual grpc_error* RecvData(int64_t incoming_frame_size) { abort(); }
+  virtual uint32_t MaybeSendUpdate() { abort(); }
+  virtual void RecvUpdate(uint32_t size) { abort(); }
   virtual void IncomingByteStreamUpdate(size_t max_size_hint,
-                                        size_t have_already) {abort();}
+                                        size_t have_already) {
+    abort();
+  }
   int64_t remote_window_delta() { return remote_window_delta_; }
   int64_t local_window_delta() { return local_window_delta_; }
   int64_t announced_window_delta() { return announced_window_delta_; }
@@ -294,6 +321,22 @@ class StreamFlowControlBase {
   int64_t announced_window_delta_ = 0;
 };
 
+class StreamFlowControlDisabled : public StreamFlowControlBase {
+ public:
+  virtual FlowControlAction UpdateAction(FlowControlAction action) {
+    return action;
+  }
+  virtual FlowControlAction MakeAction() { return FlowControlAction(); }
+  virtual void SentData(int64_t outgoing_frame_size) {}
+  virtual grpc_error* RecvData(int64_t incoming_frame_size) {
+    return GRPC_ERROR_NONE;
+  }
+  virtual uint32_t MaybeSendUpdate() { return 0; }
+  virtual void RecvUpdate(uint32_t size) {}
+  virtual void IncomingByteStreamUpdate(size_t max_size_hint,
+                                        size_t have_already) {}
+};
+
 class StreamFlowControl final : public StreamFlowControlBase {
  public:
   StreamFlowControl(TransportFlowControl* tfc, const grpc_chttp2_stream* s);
diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h
index 0441f1c5413..89214a601c8 100644
--- a/src/core/ext/transport/chttp2/transport/internal.h
+++ b/src/core/ext/transport/chttp2/transport/internal.h
@@ -353,7 +353,8 @@ struct grpc_chttp2_transport {
 
   grpc_core::PolymorphicManualConstructor<
       grpc_core::chttp2::TransportFlowControlBase,
-      grpc_core::chttp2::TransportFlowControl>
+      grpc_core::chttp2::TransportFlowControl,
+      grpc_core::chttp2::TransportFlowControlDisabled>
       flow_control;
   /** initial window change. This is tracked as we parse settings frames from
    * the remote peer. If there is a positive delta, then we will make all
@@ -530,7 +531,8 @@ struct grpc_chttp2_stream {
 
   grpc_core::PolymorphicManualConstructor<
       grpc_core::chttp2::StreamFlowControlBase,
-      grpc_core::chttp2::StreamFlowControl>
+      grpc_core::chttp2::StreamFlowControl,
+      grpc_core::chttp2::StreamFlowControlDisabled>
       flow_control;
 
   grpc_slice_buffer flow_controlled_buffer;

From 99849886eb260af69eb948014c3a49bb2351112b Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Mon, 27 Nov 2017 15:37:13 -0800
Subject: [PATCH 003/161] Disable flow control settings frames

---
 src/core/ext/transport/chttp2/transport/frame_settings.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.cc b/src/core/ext/transport/chttp2/transport/frame_settings.cc
index de4340fea50..1fd3a66463a 100644
--- a/src/core/ext/transport/chttp2/transport/frame_settings.cc
+++ b/src/core/ext/transport/chttp2/transport/frame_settings.cc
@@ -187,6 +187,11 @@ grpc_error* grpc_chttp2_settings_parser_parse(grpc_exec_ctx* exec_ctx, void* p,
         if (grpc_wire_id_to_setting_id(parser->id, &id)) {
           const grpc_chttp2_setting_parameters* sp =
               &grpc_chttp2_settings_parameters[id];
+          if ((id == GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE ||
+               id == GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE) &&
+              true /* disable flow conrtol */) {
+            continue;
+          }
           if (parser->value < sp->min_value || parser->value > sp->max_value) {
             switch (sp->invalid_value_behavior) {
               case GRPC_CHTTP2_CLAMP_INVALID_VALUE:

From 15c32cd4813e20cac65e9c2a8f9cea4e1d483a05 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Tue, 5 Dec 2017 12:57:49 -0800
Subject: [PATCH 004/161] Fix parsing check

---
 .../chttp2/transport/chttp2_transport.cc       | 18 +++++++++++++-----
 .../ext/transport/chttp2/transport/parsing.cc  |  7 ++++---
 .../transport/chttp2/transport/stream_lists.cc |  2 ++
 test/core/end2end/tests/payload.cc             |  3 +++
 4 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 70b30b7db20..233ea7f6a6f 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -338,11 +338,13 @@ static void init_transport(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
     }
   }
   t->dirtied_local_settings = 1;
-  /* Hack: it's common for implementations to assume 65536 bytes initial send
-     window -- this should by rights be 0 */
-  t->force_send_settings = 1 << GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
-  t->sent_local_settings = 0;
-  t->write_buffer_size = grpc_core::chttp2::kDefaultWindow;
+  if (!true /*diable flow control */) {
+    /* Hack: it's common for implementations to assume 65536 bytes initial send
+       window -- this should by rights be 0 */
+    t->force_send_settings = 1 << GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
+    t->sent_local_settings = 0;
+    t->write_buffer_size = grpc_core::chttp2::kDefaultWindow;
+  }
 
   if (is_client) {
     grpc_slice_buffer_add(&t->outbuf, grpc_slice_from_copied_string(
@@ -555,6 +557,12 @@ static void init_transport(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
         kFrameSize;
     t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
         kFrameSize;
+    t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
+        grpc_core::chttp2::kMaxWindow;
+    t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
+        grpc_core::chttp2::kMaxWindow;
+    t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
+        grpc_core::chttp2::kMaxWindow;
     enable_bdp = false;
   } else {
     t->flow_control.Init<grpc_core::chttp2::TransportFlowControl>(exec_ctx, t,
diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc
index 46ec3fbaa61..dce6aa42276 100644
--- a/src/core/ext/transport/chttp2/transport/parsing.cc
+++ b/src/core/ext/transport/chttp2/transport/parsing.cc
@@ -197,9 +197,10 @@ grpc_error* grpc_chttp2_perform_read(grpc_exec_ctx* exec_ctx,
           return GRPC_ERROR_NONE;
         }
         goto dts_fh_0; /* loop */
-      } else if (t->incoming_frame_size >
-                 t->settings[GRPC_ACKED_SETTINGS]
-                            [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) {
+      } else if (!true /*disable flow control*/ &&
+                 t->incoming_frame_size >
+                     t->settings[GRPC_ACKED_SETTINGS]
+                                [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) {
         char* msg;
         gpr_asprintf(&msg, "Frame size %d is larger than max frame size %d",
                      t->incoming_frame_size,
diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.cc b/src/core/ext/transport/chttp2/transport/stream_lists.cc
index c95d02541a7..c6e2bada337 100644
--- a/src/core/ext/transport/chttp2/transport/stream_lists.cc
+++ b/src/core/ext/transport/chttp2/transport/stream_lists.cc
@@ -183,6 +183,7 @@ void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport* t,
 
 void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport* t,
                                                grpc_chttp2_stream* s) {
+  GPR_ASSERT(!true /*flow control disabled */);
   stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);
 }
 
@@ -198,6 +199,7 @@ void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport* t,
 
 void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport* t,
                                             grpc_chttp2_stream* s) {
+  GPR_ASSERT(!true /*flow control disabled */);
   stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM);
 }
 
diff --git a/test/core/end2end/tests/payload.cc b/test/core/end2end/tests/payload.cc
index 2e9513b9cbe..8c3fd1b668a 100644
--- a/test/core/end2end/tests/payload.cc
+++ b/test/core/end2end/tests/payload.cc
@@ -125,6 +125,7 @@ static void request_response_with_payload(grpc_end2end_test_config config,
   grpc_byte_buffer* response_payload_recv = nullptr;
   grpc_call_details call_details;
   grpc_status_code status;
+  const char* error_string = nullptr;
   grpc_call_error error;
   grpc_slice details;
   int was_cancelled = 2;
@@ -172,6 +173,7 @@ static void request_response_with_payload(grpc_end2end_test_config config,
   op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
   op->data.recv_status_on_client.status = &status;
   op->data.recv_status_on_client.status_details = &details;
+  op->data.recv_status_on_client.error_string = &error_string;
   op->flags = 0;
   op->reserved = nullptr;
   op++;
@@ -230,6 +232,7 @@ static void request_response_with_payload(grpc_end2end_test_config config,
   CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
   cq_verify(cqv);
 
+  gpr_log(GPR_ERROR, "%s", error_string);
   GPR_ASSERT(status == GRPC_STATUS_OK);
   GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz"));
   GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo"));

From 3ca7a087a9a9c77f9e95e132835bc422a01da5ef Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Tue, 5 Dec 2017 13:08:34 -0800
Subject: [PATCH 005/161] Fix buildtests compile

---
 .../transport/chttp2/transport/flow_control.h | 53 ++++++++++---------
 1 file changed, 29 insertions(+), 24 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h
index d28e31d4030..79d2dec9400 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.h
+++ b/src/core/ext/transport/chttp2/transport/flow_control.h
@@ -148,6 +148,7 @@ class TransportFlowControlBase {
   int64_t remote_window() const { return remote_window_; }
   virtual int64_t target_window() const { return target_initial_window_size_; }
   int64_t announced_window() const { return announced_window_; }
+  virtual void TestOnlyForceHugeWindow() {}
 
   GRPC_ABSTRACT_BASE_CLASS
 
@@ -166,17 +167,17 @@ class TransportFlowControlDisabled final : public TransportFlowControlBase {
     target_initial_window_size_ = kMaxWindow;
     announced_window_ = kMaxWindow;
   }
-  virtual uint32_t MaybeSendUpdate(bool writing_anyway) { return 0; }
-  virtual FlowControlAction MakeAction() { return FlowControlAction(); }
-  virtual FlowControlAction PeriodicUpdate(grpc_exec_ctx* exec_ctx) {
+  uint32_t MaybeSendUpdate(bool writing_anyway) override { return 0; }
+  FlowControlAction MakeAction() override { return FlowControlAction(); }
+  FlowControlAction PeriodicUpdate(grpc_exec_ctx* exec_ctx) override {
     return FlowControlAction();
   }
-  virtual void StreamSentData(int64_t size) {}
-  virtual grpc_error* RecvData(int64_t incoming_frame_size) {
+  void StreamSentData(int64_t size) override {}
+  grpc_error* RecvData(int64_t incoming_frame_size) override {
     return GRPC_ERROR_NONE;
   }
-  virtual void RecvUpdate(uint32_t size) {}
-  virtual int64_t target_window() const { return kMaxWindow; }
+  void RecvUpdate(uint32_t size) override {}
+  int64_t target_window() const override { return kMaxWindow; }
 };
 
 class TransportFlowControl final : public TransportFlowControlBase {
@@ -251,7 +252,7 @@ class TransportFlowControl final : public TransportFlowControlBase {
 
   BdpEstimator* bdp_estimator() override { return &bdp_estimator_; }
 
-  void TestOnlyForceHugeWindow() {
+  void TestOnlyForceHugeWindow() override {
     announced_window_ = 1024 * 1024 * 1024;
     remote_window_ = 1024 * 1024 * 1024;
   }
@@ -309,6 +310,7 @@ class StreamFlowControlBase {
                                         size_t have_already) {
     abort();
   }
+  virtual void TestOnlyForceHugeWindow() {}
   int64_t remote_window_delta() { return remote_window_delta_; }
   int64_t local_window_delta() { return local_window_delta_; }
   int64_t announced_window_delta() { return announced_window_delta_; }
@@ -323,18 +325,18 @@ class StreamFlowControlBase {
 
 class StreamFlowControlDisabled : public StreamFlowControlBase {
  public:
-  virtual FlowControlAction UpdateAction(FlowControlAction action) {
+  FlowControlAction UpdateAction(FlowControlAction action) override {
     return action;
   }
-  virtual FlowControlAction MakeAction() { return FlowControlAction(); }
-  virtual void SentData(int64_t outgoing_frame_size) {}
-  virtual grpc_error* RecvData(int64_t incoming_frame_size) {
+  FlowControlAction MakeAction() override { return FlowControlAction(); }
+  void SentData(int64_t outgoing_frame_size) override {}
+  grpc_error* RecvData(int64_t incoming_frame_size) override {
     return GRPC_ERROR_NONE;
   }
-  virtual uint32_t MaybeSendUpdate() { return 0; }
-  virtual void RecvUpdate(uint32_t size) {}
-  virtual void IncomingByteStreamUpdate(size_t max_size_hint,
-                                        size_t have_already) {}
+  uint32_t MaybeSendUpdate() override { return 0; }
+  void RecvUpdate(uint32_t size) override {}
+  void IncomingByteStreamUpdate(size_t max_size_hint,
+                                size_t have_already) override {}
 };
 
 class StreamFlowControl final : public StreamFlowControlBase {
@@ -344,32 +346,35 @@ class StreamFlowControl final : public StreamFlowControlBase {
     tfc_->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_);
   }
 
-  FlowControlAction UpdateAction(FlowControlAction action);
-  FlowControlAction MakeAction() { return UpdateAction(tfc_->MakeAction()); }
+  FlowControlAction UpdateAction(FlowControlAction action) override;
+  FlowControlAction MakeAction() override {
+    return UpdateAction(tfc_->MakeAction());
+  }
 
   // we have sent data on the wire, we must track this in our bookkeeping for
   // the remote peer's flow control.
-  void SentData(int64_t outgoing_frame_size) {
+  void SentData(int64_t outgoing_frame_size) override {
     FlowControlTrace tracer("  data sent", tfc_, this);
     tfc_->StreamSentData(outgoing_frame_size);
     remote_window_delta_ -= outgoing_frame_size;
   }
 
   // we have received data from the wire
-  grpc_error* RecvData(int64_t incoming_frame_size);
+  grpc_error* RecvData(int64_t incoming_frame_size) override;
 
   // returns an announce if we should send a stream update to our peer, else
   // returns zero
-  uint32_t MaybeSendUpdate();
+  uint32_t MaybeSendUpdate() override;
 
   // we have received a WINDOW_UPDATE frame for a stream
-  void RecvUpdate(uint32_t size) {
+  void RecvUpdate(uint32_t size) override {
     FlowControlTrace trace("s updt recv", tfc_, this);
     remote_window_delta_ += size;
   }
 
   // the application is asking for a certain amount of bytes
-  void IncomingByteStreamUpdate(size_t max_size_hint, size_t have_already);
+  void IncomingByteStreamUpdate(size_t max_size_hint,
+                                size_t have_already) override;
 
   int64_t remote_window_delta() const { return remote_window_delta_; }
   int64_t local_window_delta() const { return local_window_delta_; }
@@ -377,7 +382,7 @@ class StreamFlowControl final : public StreamFlowControlBase {
 
   const grpc_chttp2_stream* stream() const { return s_; }
 
-  void TestOnlyForceHugeWindow() {
+  void TestOnlyForceHugeWindow() override {
     announced_window_delta_ = 1024 * 1024 * 1024;
     local_window_delta_ = 1024 * 1024 * 1024;
     remote_window_delta_ = 1024 * 1024 * 1024;

From 2fe86668b43a5a1f74b4ab31a03e47f8fb4ca5c5 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Tue, 5 Dec 2017 13:27:17 -0800
Subject: [PATCH 006/161] Fix bm_trickle_compile

---
 src/core/ext/transport/chttp2/transport/flow_control.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h
index 79d2dec9400..fa7f716fbe0 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.h
+++ b/src/core/ext/transport/chttp2/transport/flow_control.h
@@ -153,6 +153,7 @@ class TransportFlowControlBase {
   GRPC_ABSTRACT_BASE_CLASS
 
  protected:
+  friend class ::grpc::testing::TrickledCHTTP2;
   int64_t remote_window_ = kDefaultWindow;
   int64_t target_initial_window_size_ = kDefaultWindow;
   int64_t announced_window_ = kDefaultWindow;
@@ -258,7 +259,6 @@ class TransportFlowControl final : public TransportFlowControlBase {
   }
 
  private:
-  friend class ::grpc::testing::TrickledCHTTP2;
   double TargetLogBdp();
   double SmoothLogBdp(grpc_exec_ctx* exec_ctx, double value);
   FlowControlAction::Urgency DeltaUrgency(int32_t value,
@@ -318,6 +318,7 @@ class StreamFlowControlBase {
   GRPC_ABSTRACT_BASE_CLASS
 
  protected:
+  friend class ::grpc::testing::TrickledCHTTP2;
   int64_t remote_window_delta_ = 0;
   int64_t local_window_delta_ = 0;
   int64_t announced_window_delta_ = 0;
@@ -389,7 +390,6 @@ class StreamFlowControl final : public StreamFlowControlBase {
   }
 
  private:
-  friend class ::grpc::testing::TrickledCHTTP2;
   TransportFlowControl* const tfc_;
   const grpc_chttp2_stream* const s_;
 

From 78478d0e9e84e0b95c9f0e914d3f80a76bec6cd2 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Tue, 5 Dec 2017 18:05:56 -0800
Subject: [PATCH 007/161] Fix write_buffering_test

---
 src/core/ext/transport/chttp2/transport/chttp2_transport.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 233ea7f6a6f..1375a0ca131 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -343,8 +343,8 @@ static void init_transport(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
        window -- this should by rights be 0 */
     t->force_send_settings = 1 << GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
     t->sent_local_settings = 0;
-    t->write_buffer_size = grpc_core::chttp2::kDefaultWindow;
   }
+  t->write_buffer_size = grpc_core::chttp2::kDefaultWindow;
 
   if (is_client) {
     grpc_slice_buffer_add(&t->outbuf, grpc_slice_from_copied_string(

From 66a72bba692f5b72af65a4b2a80bf45d93ccdfda Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Tue, 5 Dec 2017 18:09:07 -0800
Subject: [PATCH 008/161] Undo debugging output

---
 test/core/end2end/tests/payload.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/test/core/end2end/tests/payload.cc b/test/core/end2end/tests/payload.cc
index 8c3fd1b668a..2e9513b9cbe 100644
--- a/test/core/end2end/tests/payload.cc
+++ b/test/core/end2end/tests/payload.cc
@@ -125,7 +125,6 @@ static void request_response_with_payload(grpc_end2end_test_config config,
   grpc_byte_buffer* response_payload_recv = nullptr;
   grpc_call_details call_details;
   grpc_status_code status;
-  const char* error_string = nullptr;
   grpc_call_error error;
   grpc_slice details;
   int was_cancelled = 2;
@@ -173,7 +172,6 @@ static void request_response_with_payload(grpc_end2end_test_config config,
   op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
   op->data.recv_status_on_client.status = &status;
   op->data.recv_status_on_client.status_details = &details;
-  op->data.recv_status_on_client.error_string = &error_string;
   op->flags = 0;
   op->reserved = nullptr;
   op++;
@@ -232,7 +230,6 @@ static void request_response_with_payload(grpc_end2end_test_config config,
   CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
   cq_verify(cqv);
 
-  gpr_log(GPR_ERROR, "%s", error_string);
   GPR_ASSERT(status == GRPC_STATUS_OK);
   GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz"));
   GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo"));

From 734256a5d09f491459b611361afcb88be0d209a8 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Fri, 8 Dec 2017 12:37:16 -0800
Subject: [PATCH 009/161] Add comments

---
 .../transport/chttp2/transport/flow_control.h | 89 ++++++++++++++++++-
 1 file changed, 85 insertions(+), 4 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h
index fa7f716fbe0..8e05b0c08c2 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.h
+++ b/src/core/ext/transport/chttp2/transport/flow_control.h
@@ -44,10 +44,15 @@ namespace grpc_core {
 namespace chttp2 {
 
 static constexpr uint32_t kDefaultWindow = 65535;
+static constexpr int64_t kMaxWindow = (int64_t)((1u << 31) - 1);
 
 class TransportFlowControl;
 class StreamFlowControl;
 
+
+// Encapsulates a collections of actions the transport needs to take with
+// regard to flow control. Each action comes with urgencies that tell the
+// transport how quickly the action must take place.
 class FlowControlAction {
  public:
   enum class Urgency : uint8_t {
@@ -133,21 +138,53 @@ class FlowControlTrace {
   int64_t announced_window_delta_;
 };
 
+// Fat interface with all methods a flow control implementation needs to
+// support. gRPC C Core does not support pure virtual functions, so instead
+// we abort in any methods which require implementation in the base class.
 class TransportFlowControlBase {
  public:
   TransportFlowControlBase() {}
   virtual ~TransportFlowControlBase() {}
+
+  // Is flow control enabled? This is needed in other codepaths like the checks
+  // in parsing and in writing.
+  virtual bool flow_control_enabled() const { abort(); }
+
+  // Called to check if the transport needs to send a WINDOW_UPDATE frame
   virtual uint32_t MaybeSendUpdate(bool writing_anyway) { abort(); }
+
+  // Using the protected members, returns and Action to be taken by the 
+  // tranport.
   virtual FlowControlAction MakeAction() { abort(); }
+
+  // Using the protected members, returns and Action to be taken by the 
+  // tranport. Also checks for updates to our BDP estimate and acts 
+  // accordingly.
   virtual FlowControlAction PeriodicUpdate(grpc_exec_ctx* exec_ctx) { abort(); }
+
+  // Called to do bookkeeping when a stream owned by this transport sends
+  // data on the wire
   virtual void StreamSentData(int64_t size) { abort(); }
+
+  // Called to do bookkeeping when a stream owned by this transport receives 
+  // data from the wire. Also does error checking for frame size.
   virtual grpc_error* RecvData(int64_t incoming_frame_size) { abort(); }
+
+  // Called to do bookkeeping when we receive a WINDOW_UPDATE frame.
   virtual void RecvUpdate(uint32_t size) { abort(); }
-  // TODO(ncteisen): maybe completely encapsulate this inside FlowControl
+
+  // Returns the BdpEstimator held by this object. Caller is responsible for
+  // checking for nullptr. TODO(ncteisen): consider fully encapsulating all
+  // bdp estimator actions inside TransportFlowControl
   virtual BdpEstimator* bdp_estimator() { return nullptr; }
+
+  // Getters
   int64_t remote_window() const { return remote_window_; }
   virtual int64_t target_window() const { return target_initial_window_size_; }
   int64_t announced_window() const { return announced_window_; }
+
+  // Used in certain benchmarks in which we don't want FlowControl to be a
+  // factor
   virtual void TestOnlyForceHugeWindow() {}
 
   GRPC_ABSTRACT_BASE_CLASS
@@ -159,15 +196,23 @@ class TransportFlowControlBase {
   int64_t announced_window_ = kDefaultWindow;
 };
 
-const int64_t kMaxWindow = (int64_t)((1u << 31) - 1);
-
+// Implementation of flow control that does NOTHING. Always returns maximum
+// values, never initiates writes, and assumes that the remote peer is doing
+// the same. To be used to narrow down on flow control as the cause of negative
+// performance.
 class TransportFlowControlDisabled final : public TransportFlowControlBase {
  public:
+
+  // Maxes out all values
   TransportFlowControlDisabled() {
     remote_window_ = kMaxWindow;
     target_initial_window_size_ = kMaxWindow;
     announced_window_ = kMaxWindow;
   }
+
+  bool flow_control_enabled() const override { return false; }
+
+  // Never do anything.
   uint32_t MaybeSendUpdate(bool writing_anyway) override { return 0; }
   FlowControlAction MakeAction() override { return FlowControlAction(); }
   FlowControlAction PeriodicUpdate(grpc_exec_ctx* exec_ctx) override {
@@ -178,15 +223,18 @@ class TransportFlowControlDisabled final : public TransportFlowControlBase {
     return GRPC_ERROR_NONE;
   }
   void RecvUpdate(uint32_t size) override {}
-  int64_t target_window() const override { return kMaxWindow; }
 };
 
+// Implementation of flow control that abides to HTTP/2 spec and attempts
+// to be as performant as possible.
 class TransportFlowControl final : public TransportFlowControlBase {
  public:
   TransportFlowControl(grpc_exec_ctx* exec_ctx, const grpc_chttp2_transport* t,
                        bool enable_bdp_probe);
   ~TransportFlowControl() {}
 
+  bool flow_control_enabled() const override { return true; }
+
   bool bdp_probe() const { return enable_bdp_probe_; }
 
   // returns an announce if we should send a transport update to our peer,
@@ -227,6 +275,8 @@ class TransportFlowControl final : public TransportFlowControlBase {
     remote_window_ += size;
   }
 
+  // See comment above announced_stream_total_over_incoming_window_ for the 
+  // logic behind this decision.
   int64_t target_window() const override {
     return (uint32_t)GPR_MIN((int64_t)((1u << 31) - 1),
                              announced_stream_total_over_incoming_window_ +
@@ -296,21 +346,46 @@ class TransportFlowControl final : public TransportFlowControlBase {
   grpc_millis last_pid_update_ = 0;
 };
 
+// Fat interface with all methods a stream flow control implementation needs
+// to support. gRPC C Core does not support pure virtual functions, so instead
+// we abort in any methods which require implementation in the base class.
 class StreamFlowControlBase {
  public:
   StreamFlowControlBase() {}
   virtual ~StreamFlowControlBase() {}
+
+  // Updates an action using the protected members.
   virtual FlowControlAction UpdateAction(FlowControlAction action) { abort(); }
+
+  // Using the protected members, returns an Action for this stream to be 
+  // taken by the tranport.
   virtual FlowControlAction MakeAction() { abort(); }
+
+  // Bookkeeping for when data is sent on this stream.
   virtual void SentData(int64_t outgoing_frame_size) { abort(); }
+
+  // Bookkeeping and error checking for when data is received by this stream.
   virtual grpc_error* RecvData(int64_t incoming_frame_size) { abort(); }
+
+  // Called to check if this stream needs to send a WINDOW_UPDATE frame.
   virtual uint32_t MaybeSendUpdate() { abort(); }
+
+  // Bookkeeping for receiving a WINDOW_UPDATE from for this stream.
   virtual void RecvUpdate(uint32_t size) { abort(); }
+
+  // Bookkeeping for when a call pulls bytes out of the transport. At this
+  // point we consider the data 'used' and can thus let out peer know we are
+  // ready for more data.
   virtual void IncomingByteStreamUpdate(size_t max_size_hint,
                                         size_t have_already) {
     abort();
   }
+
+  // Used in certain benchmarks in which we don't want FlowControl to be a
+  // factor
   virtual void TestOnlyForceHugeWindow() {}
+
+  // Getters
   int64_t remote_window_delta() { return remote_window_delta_; }
   int64_t local_window_delta() { return local_window_delta_; }
   int64_t announced_window_delta() { return announced_window_delta_; }
@@ -324,6 +399,10 @@ class StreamFlowControlBase {
   int64_t announced_window_delta_ = 0;
 };
 
+// Implementation of flow control that does NOTHING. Always returns maximum
+// values, never initiates writes, and assumes that the remote peer is doing
+// the same. To be used to narrow down on flow control as the cause of negative
+// performance.
 class StreamFlowControlDisabled : public StreamFlowControlBase {
  public:
   FlowControlAction UpdateAction(FlowControlAction action) override {
@@ -340,6 +419,8 @@ class StreamFlowControlDisabled : public StreamFlowControlBase {
                                 size_t have_already) override {}
 };
 
+// Implementation of flow control that abides to HTTP/2 spec and attempts
+// to be as performant as possible.
 class StreamFlowControl final : public StreamFlowControlBase {
  public:
   StreamFlowControl(TransportFlowControl* tfc, const grpc_chttp2_stream* s);

From 0aaee995a3202db07ac0c07b1fbfc863824ca13c Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Fri, 8 Dec 2017 12:45:55 -0800
Subject: [PATCH 010/161] Plumb through flow_control->flow_conrol_enabled()

---
 .../chttp2/transport/chttp2_transport.cc      | 33 +++++--------------
 .../chttp2/transport/flow_control.cc          | 19 +++++++++++
 .../transport/chttp2/transport/flow_control.h | 22 +++++--------
 .../chttp2/transport/frame_settings.cc        |  7 ++--
 .../ext/transport/chttp2/transport/parsing.cc |  2 +-
 .../chttp2/transport/stream_lists.cc          |  4 +--
 6 files changed, 43 insertions(+), 44 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 1375a0ca131..768cee11906 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -338,12 +338,10 @@ static void init_transport(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
     }
   }
   t->dirtied_local_settings = 1;
-  if (!true /*diable flow control */) {
-    /* Hack: it's common for implementations to assume 65536 bytes initial send
-       window -- this should by rights be 0 */
-    t->force_send_settings = 1 << GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
-    t->sent_local_settings = 0;
-  }
+  /* Hack: it's common for implementations to assume 65536 bytes initial send
+     window -- this should by rights be 0 */
+  t->force_send_settings = 1 << GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
+  t->sent_local_settings = 0;
   t->write_buffer_size = grpc_core::chttp2::kDefaultWindow;
 
   if (is_client) {
@@ -546,23 +544,8 @@ static void init_transport(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
     }
   }
 
-  // Tune the heck out of this
-  const uint32_t kFrameSize = 1024 * 1024;
-
   if (true /* disable flow control*/) {
-    t->flow_control.Init<grpc_core::chttp2::TransportFlowControlDisabled>();
-    t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
-        kFrameSize;
-    t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
-        kFrameSize;
-    t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
-        kFrameSize;
-    t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
-        grpc_core::chttp2::kMaxWindow;
-    t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
-        grpc_core::chttp2::kMaxWindow;
-    t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
-        grpc_core::chttp2::kMaxWindow;
+    t->flow_control.Init<grpc_core::chttp2::TransportFlowControlDisabled>(t);
     enable_bdp = false;
   } else {
     t->flow_control.Init<grpc_core::chttp2::TransportFlowControl>(exec_ctx, t,
@@ -739,13 +722,13 @@ static int init_stream(grpc_exec_ctx* exec_ctx, grpc_transport* gt,
     post_destructive_reclaimer(exec_ctx, t);
   }
 
-  if (true /* disable flow control */) {
-    s->flow_control.Init<grpc_core::chttp2::StreamFlowControlDisabled>();
-  } else {
+  if (t->flow_control->flow_control_enabled()) {
     s->flow_control.Init<grpc_core::chttp2::StreamFlowControl>(
         static_cast<grpc_core::chttp2::TransportFlowControl*>(
             t->flow_control.get()),
         s);
+  } else {
+    s->flow_control.Init<grpc_core::chttp2::StreamFlowControlDisabled>();
   }
   GPR_TIMER_END("init_stream", 0);
 
diff --git a/src/core/ext/transport/chttp2/transport/flow_control.cc b/src/core/ext/transport/chttp2/transport/flow_control.cc
index 8a057bd9ff4..67601d1530a 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.cc
+++ b/src/core/ext/transport/chttp2/transport/flow_control.cc
@@ -149,6 +149,25 @@ void FlowControlAction::Trace(grpc_chttp2_transport* t) const {
   gpr_free(mf_str);
 }
 
+TransportFlowControlDisabled::TransportFlowControlDisabled(
+    grpc_chttp2_transport* t) {
+  remote_window_ = kMaxWindow;
+  target_initial_window_size_ = kMaxWindow;
+  announced_window_ = kMaxWindow;
+  t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
+      kFrameSize;
+  t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
+      kFrameSize;
+  t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
+      kFrameSize;
+  t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
+      kMaxWindow;
+  t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
+      kMaxWindow;
+  t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
+      kMaxWindow;
+}
+
 TransportFlowControl::TransportFlowControl(grpc_exec_ctx* exec_ctx,
                                            const grpc_chttp2_transport* t,
                                            bool enable_bdp_probe)
diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h
index 8e05b0c08c2..750b7556230 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.h
+++ b/src/core/ext/transport/chttp2/transport/flow_control.h
@@ -45,11 +45,12 @@ namespace chttp2 {
 
 static constexpr uint32_t kDefaultWindow = 65535;
 static constexpr int64_t kMaxWindow = (int64_t)((1u << 31) - 1);
+// TODO(ncteisen): Tune this
+static constexpr uint32_t kFrameSize = 1024 * 1024;
 
 class TransportFlowControl;
 class StreamFlowControl;
 
-
 // Encapsulates a collections of actions the transport needs to take with
 // regard to flow control. Each action comes with urgencies that tell the
 // transport how quickly the action must take place.
@@ -153,12 +154,12 @@ class TransportFlowControlBase {
   // Called to check if the transport needs to send a WINDOW_UPDATE frame
   virtual uint32_t MaybeSendUpdate(bool writing_anyway) { abort(); }
 
-  // Using the protected members, returns and Action to be taken by the 
+  // Using the protected members, returns and Action to be taken by the
   // tranport.
   virtual FlowControlAction MakeAction() { abort(); }
 
-  // Using the protected members, returns and Action to be taken by the 
-  // tranport. Also checks for updates to our BDP estimate and acts 
+  // Using the protected members, returns and Action to be taken by the
+  // tranport. Also checks for updates to our BDP estimate and acts
   // accordingly.
   virtual FlowControlAction PeriodicUpdate(grpc_exec_ctx* exec_ctx) { abort(); }
 
@@ -166,7 +167,7 @@ class TransportFlowControlBase {
   // data on the wire
   virtual void StreamSentData(int64_t size) { abort(); }
 
-  // Called to do bookkeeping when a stream owned by this transport receives 
+  // Called to do bookkeeping when a stream owned by this transport receives
   // data from the wire. Also does error checking for frame size.
   virtual grpc_error* RecvData(int64_t incoming_frame_size) { abort(); }
 
@@ -202,13 +203,8 @@ class TransportFlowControlBase {
 // performance.
 class TransportFlowControlDisabled final : public TransportFlowControlBase {
  public:
-
   // Maxes out all values
-  TransportFlowControlDisabled() {
-    remote_window_ = kMaxWindow;
-    target_initial_window_size_ = kMaxWindow;
-    announced_window_ = kMaxWindow;
-  }
+  TransportFlowControlDisabled(grpc_chttp2_transport* t);
 
   bool flow_control_enabled() const override { return false; }
 
@@ -275,7 +271,7 @@ class TransportFlowControl final : public TransportFlowControlBase {
     remote_window_ += size;
   }
 
-  // See comment above announced_stream_total_over_incoming_window_ for the 
+  // See comment above announced_stream_total_over_incoming_window_ for the
   // logic behind this decision.
   int64_t target_window() const override {
     return (uint32_t)GPR_MIN((int64_t)((1u << 31) - 1),
@@ -357,7 +353,7 @@ class StreamFlowControlBase {
   // Updates an action using the protected members.
   virtual FlowControlAction UpdateAction(FlowControlAction action) { abort(); }
 
-  // Using the protected members, returns an Action for this stream to be 
+  // Using the protected members, returns an Action for this stream to be
   // taken by the tranport.
   virtual FlowControlAction MakeAction() { abort(); }
 
diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.cc b/src/core/ext/transport/chttp2/transport/frame_settings.cc
index 1fd3a66463a..0e40213a1c2 100644
--- a/src/core/ext/transport/chttp2/transport/frame_settings.cc
+++ b/src/core/ext/transport/chttp2/transport/frame_settings.cc
@@ -187,9 +187,10 @@ grpc_error* grpc_chttp2_settings_parser_parse(grpc_exec_ctx* exec_ctx, void* p,
         if (grpc_wire_id_to_setting_id(parser->id, &id)) {
           const grpc_chttp2_setting_parameters* sp =
               &grpc_chttp2_settings_parameters[id];
-          if ((id == GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE ||
-               id == GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE) &&
-              true /* disable flow conrtol */) {
+          // If flow control is disabled we skip these.
+          if (!t->flow_control->flow_control_enabled() &&
+              (id == GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE ||
+               id == GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE)) {
             continue;
           }
           if (parser->value < sp->min_value || parser->value > sp->max_value) {
diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc
index dce6aa42276..f2a7e95fc60 100644
--- a/src/core/ext/transport/chttp2/transport/parsing.cc
+++ b/src/core/ext/transport/chttp2/transport/parsing.cc
@@ -197,7 +197,7 @@ grpc_error* grpc_chttp2_perform_read(grpc_exec_ctx* exec_ctx,
           return GRPC_ERROR_NONE;
         }
         goto dts_fh_0; /* loop */
-      } else if (!true /*disable flow control*/ &&
+      } else if (t->flow_control->flow_control_enabled() &&
                  t->incoming_frame_size >
                      t->settings[GRPC_ACKED_SETTINGS]
                                 [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) {
diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.cc b/src/core/ext/transport/chttp2/transport/stream_lists.cc
index c6e2bada337..3aad8c5823c 100644
--- a/src/core/ext/transport/chttp2/transport/stream_lists.cc
+++ b/src/core/ext/transport/chttp2/transport/stream_lists.cc
@@ -183,7 +183,7 @@ void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport* t,
 
 void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport* t,
                                                grpc_chttp2_stream* s) {
-  GPR_ASSERT(!true /*flow control disabled */);
+  GPR_ASSERT(t->flow_control->flow_control_enabled());
   stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);
 }
 
@@ -199,7 +199,7 @@ void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport* t,
 
 void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport* t,
                                             grpc_chttp2_stream* s) {
-  GPR_ASSERT(!true /*flow control disabled */);
+  GPR_ASSERT(t->flow_control->flow_control_enabled());
   stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM);
 }
 

From 228089f7b87ac74e17c6d4470cb2ebf0371a0844 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Mon, 23 Oct 2017 16:08:40 -0700
Subject: [PATCH 011/161] Add new env var

---
 doc/environment_variables.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/environment_variables.md b/doc/environment_variables.md
index 40af758f693..4796ad067d8 100644
--- a/doc/environment_variables.md
+++ b/doc/environment_variables.md
@@ -127,3 +127,8 @@ some configuration as environment variables that can be set.
   there is no active polling thread. They help reconnect disconnected client
   channels (mostly due to idleness), so that the next RPC on this channel won't
   fail. Set to 0 to turn off the backup polls.
+
+* GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL
+  if set, flow control will be effectively disabled. Max out all values and
+  assume the remote peer does the same. Thus we can ignore any flow control
+  bookkeeping, error checking, and decision making

From a77fb7dc6792a48edc30100482032844c56feedc Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Fri, 8 Dec 2017 12:57:26 -0800
Subject: [PATCH 012/161] Read new env var to toggle

---
 .../chttp2/transport/chttp2_transport.cc       | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 768cee11906..f300ae91885 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -261,6 +261,9 @@ void grpc_chttp2_ref_transport(grpc_chttp2_transport* t) { gpr_ref(&t->refs); }
 
 static const grpc_transport_vtable* get_vtable(void);
 
+// -1 == unset, 0 == disabled, 1 == enabled
+static int flow_control_enabled = -1;
+
 static void init_transport(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
                            const grpc_channel_args* channel_args,
                            grpc_endpoint* ep, bool is_client) {
@@ -544,12 +547,19 @@ static void init_transport(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
     }
   }
 
-  if (true /* disable flow control*/) {
-    t->flow_control.Init<grpc_core::chttp2::TransportFlowControlDisabled>(t);
-    enable_bdp = false;
-  } else {
+  if (flow_control_enabled == -1) {
+    char* env_variable = gpr_getenv("GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL");
+    if (env_variable != nullptr) flow_control_enabled = 0;
+    else flow_control_enabled = 1;
+    gpr_free(env_variable);
+  }
+
+  if (flow_control_enabled) {
     t->flow_control.Init<grpc_core::chttp2::TransportFlowControl>(exec_ctx, t,
                                                                   enable_bdp);
+  } else { 
+    t->flow_control.Init<grpc_core::chttp2::TransportFlowControlDisabled>(t);
+    enable_bdp = false;
   }
 
   /* No pings allowed before receiving a header or data frame. */

From ef0e64cdf5350995e16bf36b50dd7efb094a4a25 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Mon, 13 Nov 2017 07:50:37 +0100
Subject: [PATCH 013/161] Revert "Revert "Switching from UNAUTHENTICATED to
 UNAVAILABLE for auth metadata failure""

---
 src/core/lib/security/transport/client_auth_filter.cc | 2 +-
 test/cpp/end2end/end2end_test.cc                      | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/core/lib/security/transport/client_auth_filter.cc b/src/core/lib/security/transport/client_auth_filter.cc
index cd3c2e3f19d..12244748aa6 100644
--- a/src/core/lib/security/transport/client_auth_filter.cc
+++ b/src/core/lib/security/transport/client_auth_filter.cc
@@ -112,7 +112,7 @@ static void on_credentials_metadata(void* arg, grpc_error* input_error) {
     grpc_call_next_op(elem, batch);
   } else {
     error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS,
-                               GRPC_STATUS_UNAUTHENTICATED);
+                               GRPC_STATUS_UNAVAILABLE);
     grpc_transport_stream_op_batch_finish_with_failure(batch, error,
                                                        calld->call_combiner);
   }
diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc
index 4c8dfe0f401..a6ea5aada9f 100644
--- a/test/cpp/end2end/end2end_test.cc
+++ b/test/cpp/end2end/end2end_test.cc
@@ -1609,7 +1609,7 @@ TEST_P(SecureEnd2endTest, AuthMetadataPluginKeyFailure) {
 
   Status s = stub_->Echo(&context, request, &response);
   EXPECT_FALSE(s.ok());
-  EXPECT_EQ(s.error_code(), StatusCode::UNAUTHENTICATED);
+  EXPECT_EQ(s.error_code(), StatusCode::UNAVAILABLE);
 }
 
 TEST_P(SecureEnd2endTest, AuthMetadataPluginValueFailure) {
@@ -1626,7 +1626,7 @@ TEST_P(SecureEnd2endTest, AuthMetadataPluginValueFailure) {
 
   Status s = stub_->Echo(&context, request, &response);
   EXPECT_FALSE(s.ok());
-  EXPECT_EQ(s.error_code(), StatusCode::UNAUTHENTICATED);
+  EXPECT_EQ(s.error_code(), StatusCode::UNAVAILABLE);
 }
 
 TEST_P(SecureEnd2endTest, NonBlockingAuthMetadataPluginFailure) {
@@ -1644,7 +1644,7 @@ TEST_P(SecureEnd2endTest, NonBlockingAuthMetadataPluginFailure) {
 
   Status s = stub_->Echo(&context, request, &response);
   EXPECT_FALSE(s.ok());
-  EXPECT_EQ(s.error_code(), StatusCode::UNAUTHENTICATED);
+  EXPECT_EQ(s.error_code(), StatusCode::UNAVAILABLE);
   EXPECT_EQ(s.error_message(),
             grpc::string("Getting metadata from plugin failed with error: ") +
                 kTestCredsPluginErrorMsg);
@@ -1705,7 +1705,7 @@ TEST_P(SecureEnd2endTest, BlockingAuthMetadataPluginFailure) {
 
   Status s = stub_->Echo(&context, request, &response);
   EXPECT_FALSE(s.ok());
-  EXPECT_EQ(s.error_code(), StatusCode::UNAUTHENTICATED);
+  EXPECT_EQ(s.error_code(), StatusCode::UNAVAILABLE);
   EXPECT_EQ(s.error_message(),
             grpc::string("Getting metadata from plugin failed with error: ") +
                 kTestCredsPluginErrorMsg);

From a2044f83c6956d8519c12ad698891f75c9391e50 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Mon, 13 Nov 2017 08:05:34 +0100
Subject: [PATCH 014/161] metadata plugin update statuses in wrapped langs

---
 src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs | 2 +-
 src/php/tests/unit_tests/CallCredentials2Test.php             | 4 ++--
 src/ruby/spec/generic/client_stub_spec.rb                     | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs b/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
index eba6276a1f5..c83ccd26128 100644
--- a/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
+++ b/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs
@@ -162,7 +162,7 @@ namespace Grpc.IntegrationTesting
             client = new TestService.TestServiceClient(channel);
 
             var ex = Assert.Throws<RpcException>(() => client.UnaryCall(new SimpleRequest { }));
-            Assert.AreEqual(StatusCode.Unauthenticated, ex.Status.StatusCode);
+            Assert.AreEqual(StatusCode.Unavailable, ex.Status.StatusCode);
         }
 
         private class FakeTestService : TestService.TestServiceBase
diff --git a/src/php/tests/unit_tests/CallCredentials2Test.php b/src/php/tests/unit_tests/CallCredentials2Test.php
index 0a587906faa..1c7e0c0ff6f 100644
--- a/src/php/tests/unit_tests/CallCredentials2Test.php
+++ b/src/php/tests/unit_tests/CallCredentials2Test.php
@@ -147,7 +147,7 @@ class CallCredentials2Test extends PHPUnit_Framework_TestCase
 
         $this->assertTrue($event->send_metadata);
         $this->assertTrue($event->send_close);
-        $this->assertTrue($event->status->code == Grpc\STATUS_UNAUTHENTICATED);
+        $this->assertTrue($event->status->code == Grpc\STATUS_UNAVAILABLE);
     }
 
     public function invalidReturnCallbackFunc($context)
@@ -179,6 +179,6 @@ class CallCredentials2Test extends PHPUnit_Framework_TestCase
 
         $this->assertTrue($event->send_metadata);
         $this->assertTrue($event->send_close);
-        $this->assertTrue($event->status->code == Grpc\STATUS_UNAUTHENTICATED);
+        $this->assertTrue($event->status->code == Grpc\STATUS_UNAVAILABLE);
     }
 }
diff --git a/src/ruby/spec/generic/client_stub_spec.rb b/src/ruby/spec/generic/client_stub_spec.rb
index 9539e56c0ff..0ced39f116e 100644
--- a/src/ruby/spec/generic/client_stub_spec.rb
+++ b/src/ruby/spec/generic/client_stub_spec.rb
@@ -228,7 +228,7 @@ describe 'ClientStub' do
         th.join
       end
 
-      it 'should receive UNAUTHENTICATED if call credentials plugin fails' do
+      it 'should receive UNAVAILABLE if call credentials plugin fails' do
         server_port = create_secure_test_server
         th = run_request_response(@sent_msg, @resp, @pass)
 
@@ -252,7 +252,7 @@ describe 'ClientStub' do
         unauth_error_occured = false
         begin
           get_response(stub, credentials: creds)
-        rescue GRPC::Unauthenticated => e
+        rescue GRPC::Unavailable => e
           unauth_error_occured = true
           expect(e.details.include?(error_message)).to be true
         end

From a3cd72bde78b46cd09161ce40af6cf3c5382936a Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Tue, 12 Dec 2017 19:06:53 -0800
Subject: [PATCH 015/161] Add 1.8.0 for C-based languages to interop matrix

Add 1.8.0 for C#, PHP, C++, Ruby and 1.8.1 for Python

(Node specifically left out because of failures on 1.7.2
 leaving it for another PR)
---
 tools/interop_matrix/client_matrix.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py
index 71d3a79023a..e0ae854ab77 100644
--- a/tools/interop_matrix/client_matrix.py
+++ b/tools/interop_matrix/client_matrix.py
@@ -77,6 +77,9 @@ LANG_RELEASE_MATRIX = {
         {
             'v1.7.2': None
         },
+        {
+            'v1.8.0': None
+        },
     ],
     'go': [
         {
@@ -155,6 +158,9 @@ LANG_RELEASE_MATRIX = {
         {
             'v1.7.2': None
         },
+        {
+            'v1.8.1': None  # first python 1.8 release is 1.8.1
+        },
     ],
     'node': [
         {
@@ -204,6 +210,9 @@ LANG_RELEASE_MATRIX = {
         {
             'v1.7.2': None
         },
+        {
+            'v1.8.0': None
+        },
     ],
     'php': [
         {
@@ -227,6 +236,9 @@ LANG_RELEASE_MATRIX = {
         {
             'v1.7.2': None
         },
+        {
+            'v1.8.0': None
+        },
     ],
     'csharp': [
         #{'v1.0.1': None},
@@ -248,5 +260,8 @@ LANG_RELEASE_MATRIX = {
         {
             'v1.7.2': None
         },
+        {
+            'v1.8.0': None
+        },
     ],
 }

From d230ad086a894cc963235b27814e19bf686eb7aa Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Tue, 2 Jan 2018 11:43:53 -0800
Subject: [PATCH 016/161] clang fmt

---
 src/core/ext/transport/chttp2/transport/chttp2_transport.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index caf01c7e4a6..c13c4056c11 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -517,10 +517,9 @@ static void init_transport(grpc_chttp2_transport* t,
 
   if (flow_control_enabled == -1) {
     char* env_variable = gpr_getenv("GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL");
-    if (env_variable != nullptr){
+    if (env_variable != nullptr) {
       flow_control_enabled = 0;
-    }
-    else {
+    } else {
       flow_control_enabled = 1;
     }
     gpr_free(env_variable);

From 849bd73c4fe28ec880d111522fce05cda7126c93 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Tue, 2 Jan 2018 23:30:47 +0000
Subject: [PATCH 017/161] Wrap duplicated-name definitions in anonymous
 namespace

---
 .../filters/client_channel/backup_poller.cc   |  6 +++--
 .../client_channel/channel_connectivity.cc    |  6 +++--
 .../grpclb/client_load_reporting_filter.cc    |  6 +++--
 .../client_channel/lb_policy/grpclb/grpclb.cc |  6 +++--
 .../lb_policy/pick_first/pick_first.cc        |  6 +++--
 .../lb_policy/round_robin/round_robin.cc      |  8 ++++---
 .../ext/filters/client_channel/subchannel.cc  |  6 +++--
 .../filters/http/client/http_client_filter.cc | 10 ++++----
 .../message_compress_filter.cc                | 14 ++++++-----
 .../filters/http/server/http_server_filter.cc | 10 ++++----
 .../server_load_reporting_filter.cc           | 10 ++++----
 .../ext/filters/max_age/max_age_filter.cc     |  6 +++--
 .../message_size/message_size_filter.cc       | 10 ++++----
 .../workaround_cronet_compression_filter.cc   |  6 +++--
 src/core/lib/iomgr/tcp_posix.cc               | 10 ++++----
 .../security/transport/client_auth_filter.cc  | 10 ++++----
 .../security/transport/server_auth_filter.cc  | 14 ++++++-----
 src/core/lib/surface/completion_queue.cc      |  8 +++----
 src/core/lib/surface/server.cc                | 24 +++++++++----------
 19 files changed, 105 insertions(+), 71 deletions(-)

diff --git a/src/core/ext/filters/client_channel/backup_poller.cc b/src/core/ext/filters/client_channel/backup_poller.cc
index bfc549e709e..4ee5e9c1093 100644
--- a/src/core/ext/filters/client_channel/backup_poller.cc
+++ b/src/core/ext/filters/client_channel/backup_poller.cc
@@ -33,7 +33,8 @@
 
 #define DEFAULT_POLL_INTERVAL_MS 5000
 
-typedef struct backup_poller {
+namespace {
+struct backup_poller {
   grpc_timer polling_timer;
   grpc_closure run_poller_closure;
   grpc_closure shutdown_closure;
@@ -42,7 +43,8 @@ typedef struct backup_poller {
   bool shutting_down;     // guarded by pollset_mu
   gpr_refcount refs;
   gpr_refcount shutdown_refs;
-} backup_poller;
+};
+}  // namespace
 
 static gpr_once g_once = GPR_ONCE_INIT;
 static gpr_mu g_poller_mu;
diff --git a/src/core/ext/filters/client_channel/channel_connectivity.cc b/src/core/ext/filters/client_channel/channel_connectivity.cc
index 20693ba4190..a827aa30ec6 100644
--- a/src/core/ext/filters/client_channel/channel_connectivity.cc
+++ b/src/core/ext/filters/client_channel/channel_connectivity.cc
@@ -58,7 +58,8 @@ typedef enum {
   CALLING_BACK_AND_FINISHED,
 } callback_phase;
 
-typedef struct {
+namespace {
+struct state_watcher {
   gpr_mu mu;
   callback_phase phase;
   grpc_closure on_complete;
@@ -71,7 +72,8 @@ typedef struct {
   grpc_channel* channel;
   grpc_error* error;
   void* tag;
-} state_watcher;
+};
+}  // namespace
 
 static void delete_state_watcher(state_watcher* w) {
   grpc_channel_element* client_channel_elem = grpc_channel_stack_last_element(
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc
index 3eedb08ecc0..1708d81e61a 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc
@@ -32,7 +32,8 @@ static grpc_error* init_channel_elem(grpc_channel_element* elem,
 
 static void destroy_channel_elem(grpc_channel_element* elem) {}
 
-typedef struct {
+namespace {
+struct call_data {
   // Stats object to update.
   grpc_grpclb_client_stats* client_stats;
   // State for intercepting send_initial_metadata.
@@ -43,7 +44,8 @@ typedef struct {
   grpc_closure recv_initial_metadata_ready;
   grpc_closure* original_recv_initial_metadata_ready;
   bool recv_initial_metadata_succeeded;
-} call_data;
+};
+}  // namespace
 
 static void on_complete_for_send(void* arg, grpc_error* error) {
   call_data* calld = (call_data*)arg;
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index dd6fc602ab8..5f373cdb254 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -226,6 +226,7 @@ static void wrapped_rr_closure(void* arg, grpc_error* error) {
   gpr_free(wc_arg->free_when_done);
 }
 
+namespace {
 /* Linked list of pending pick requests. It stores all information needed to
  * eventually call (Round Robin's) pick() on them. They mainly stay pending
  * waiting for the RR policy to be created/updated.
@@ -234,7 +235,7 @@ static void wrapped_rr_closure(void* arg, grpc_error* error) {
  * (in \a wrapped_on_complete and \a wrapped_on_complete_arg). This is needed in
  * order to correctly unref the RR policy instance upon completion of the pick.
  * See \a wrapped_rr_closure for details. */
-typedef struct pending_pick {
+struct pending_pick {
   struct pending_pick* next;
 
   /* original pick()'s arguments */
@@ -246,7 +247,8 @@ typedef struct pending_pick {
 
   /* args for wrapped_on_complete */
   wrapped_rr_closure_arg wrapped_on_complete_arg;
-} pending_pick;
+};
+}  // namespace
 
 static void add_pending_pick(pending_pick** root,
                              const grpc_lb_policy_pick_args* pick_args,
diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
index 0861261359f..9ff40aa53c0 100644
--- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
@@ -31,12 +31,14 @@
 
 grpc_core::TraceFlag grpc_lb_pick_first_trace(false, "pick_first");
 
-typedef struct pending_pick {
+namespace {
+struct pending_pick {
   struct pending_pick* next;
   uint32_t initial_metadata_flags;
   grpc_connected_subchannel** target;
   grpc_closure* on_complete;
-} pending_pick;
+};
+}  // namespace
 
 typedef struct {
   /** base policy: must be first */
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
index b0c84017dfb..a964af06270 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
@@ -41,11 +41,12 @@
 
 grpc_core::TraceFlag grpc_lb_round_robin_trace(false, "round_robin");
 
+namespace {
 /** List of entities waiting for a pick.
  *
  * Once a pick is available, \a target is updated and \a on_complete called. */
-typedef struct pending_pick {
-  struct pending_pick* next;
+struct pending_pick {
+  pending_pick* next;
 
   /* output argument where to store the pick()ed user_data. It'll be NULL if no
    * such data is present or there's an error (the definite test for errors is
@@ -62,7 +63,8 @@ typedef struct pending_pick {
 
   /* to be invoked once the pick() has completed (regardless of success) */
   grpc_closure* on_complete;
-} pending_pick;
+};
+}  // namespace
 
 typedef struct round_robin_lb_policy {
   /** base policy: must be first */
diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc
index f07394d29b2..a604c55c58c 100644
--- a/src/core/ext/filters/client_channel/subchannel.cc
+++ b/src/core/ext/filters/client_channel/subchannel.cc
@@ -59,11 +59,13 @@
   ((grpc_connected_subchannel*)(gpr_atm_##barrier##_load( \
       &(subchannel)->connected_subchannel)))
 
-typedef struct {
+namespace {
+struct state_watcher {
   grpc_closure closure;
   grpc_subchannel* subchannel;
   grpc_connectivity_state connectivity_state;
-} state_watcher;
+};
+}  // namespace
 
 typedef struct external_state_watcher {
   grpc_subchannel* subchannel;
diff --git a/src/core/ext/filters/http/client/http_client_filter.cc b/src/core/ext/filters/http/client/http_client_filter.cc
index a1fb10f5b87..6dbd8c2a6d5 100644
--- a/src/core/ext/filters/http/client/http_client_filter.cc
+++ b/src/core/ext/filters/http/client/http_client_filter.cc
@@ -35,7 +35,8 @@
 /* default maximum size of payload eligable for GET request */
 static const size_t kMaxPayloadSizeForGet = 2048;
 
-typedef struct call_data {
+namespace {
+struct call_data {
   grpc_call_combiner* call_combiner;
   // State for handling send_initial_metadata ops.
   grpc_linked_mdelem method;
@@ -60,13 +61,14 @@ typedef struct call_data {
   grpc_closure on_send_message_next_done;
   grpc_closure* original_send_message_on_complete;
   grpc_closure send_message_on_complete;
-} call_data;
+};
 
-typedef struct channel_data {
+struct channel_data {
   grpc_mdelem static_scheme;
   grpc_mdelem user_agent;
   size_t max_payload_size_for_get;
-} channel_data;
+};
+}  // namespace
 
 static grpc_error* client_filter_incoming_metadata(grpc_call_element* elem,
                                                    grpc_metadata_batch* b) {
diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.cc b/src/core/ext/filters/http/message_compress/message_compress_filter.cc
index 9ae13d2ed27..92d17162001 100644
--- a/src/core/ext/filters/http/message_compress/message_compress_filter.cc
+++ b/src/core/ext/filters/http/message_compress/message_compress_filter.cc
@@ -35,16 +35,17 @@
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/transport/static_metadata.h"
 
-typedef enum {
+namespace {
+enum initial_metadata_state {
   // Initial metadata not yet seen.
   INITIAL_METADATA_UNSEEN = 0,
   // Initial metadata seen; compression algorithm set.
   HAS_COMPRESSION_ALGORITHM,
   // Initial metadata seen; no compression algorithm set.
   NO_COMPRESSION_ALGORITHM,
-} initial_metadata_state;
+};
 
-typedef struct call_data {
+struct call_data {
   grpc_call_combiner* call_combiner;
   grpc_linked_mdelem compression_algorithm_storage;
   grpc_linked_mdelem stream_compression_algorithm_storage;
@@ -62,9 +63,9 @@ typedef struct call_data {
   grpc_closure* original_send_message_on_complete;
   grpc_closure send_message_on_complete;
   grpc_closure on_send_message_next_done;
-} call_data;
+};
 
-typedef struct channel_data {
+struct channel_data {
   /** The default, channel-level, compression algorithm */
   grpc_compression_algorithm default_compression_algorithm;
   /** Bitset of enabled algorithms */
@@ -78,7 +79,8 @@ typedef struct channel_data {
   uint32_t enabled_stream_compression_algorithms_bitset;
   /** Supported stream compression algorithms */
   uint32_t supported_stream_compression_algorithms;
-} channel_data;
+};
+}  // namespace
 
 static bool skip_compression(grpc_call_element* elem, uint32_t flags,
                              bool has_compression_algorithm) {
diff --git a/src/core/ext/filters/http/server/http_server_filter.cc b/src/core/ext/filters/http/server/http_server_filter.cc
index b872dc98f59..508a3bf9fc4 100644
--- a/src/core/ext/filters/http/server/http_server_filter.cc
+++ b/src/core/ext/filters/http/server/http_server_filter.cc
@@ -31,7 +31,8 @@
 #define EXPECTED_CONTENT_TYPE "application/grpc"
 #define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1
 
-typedef struct call_data {
+namespace {
+struct call_data {
   grpc_call_combiner* call_combiner;
 
   grpc_linked_mdelem status;
@@ -60,11 +61,12 @@ typedef struct call_data {
   grpc_closure hs_on_recv;
   grpc_closure hs_on_complete;
   grpc_closure hs_recv_message_ready;
-} call_data;
+};
 
-typedef struct channel_data {
+struct channel_data {
   uint8_t unused;
-} channel_data;
+};
+}  // namespace
 
 static grpc_error* server_filter_outgoing_metadata(grpc_call_element* elem,
                                                    grpc_metadata_batch* b) {
diff --git a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc
index f50a928fcd9..a4142297689 100644
--- a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc
+++ b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc
@@ -31,7 +31,8 @@
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/transport/static_metadata.h"
 
-typedef struct call_data {
+namespace {
+struct call_data {
   intptr_t id; /**< an id unique to the call */
   bool have_trailing_md_string;
   grpc_slice trailing_md_string;
@@ -48,11 +49,12 @@ typedef struct call_data {
   /* to get notified of the availability of the incoming initial metadata. */
   grpc_closure on_initial_md_ready;
   grpc_metadata_batch* recv_initial_metadata;
-} call_data;
+};
 
-typedef struct channel_data {
+struct channel_data {
   intptr_t id; /**< an id unique to the channel */
-} channel_data;
+};
+}  // namespace
 
 static void on_initial_md_ready(void* user_data, grpc_error* err) {
   grpc_call_element* elem = (grpc_call_element*)user_data;
diff --git a/src/core/ext/filters/max_age/max_age_filter.cc b/src/core/ext/filters/max_age/max_age_filter.cc
index 0499c6ecfc7..7b86e4cd6c1 100644
--- a/src/core/ext/filters/max_age/max_age_filter.cc
+++ b/src/core/ext/filters/max_age/max_age_filter.cc
@@ -37,7 +37,8 @@
 #define MAX_CONNECTION_IDLE_INTEGER_OPTIONS \
   { DEFAULT_MAX_CONNECTION_IDLE_MS, 1, INT_MAX }
 
-typedef struct channel_data {
+namespace {
+struct channel_data {
   /* We take a reference to the channel stack for the timer callback */
   grpc_channel_stack* channel_stack;
   /* Guards access to max_age_timer, max_age_timer_pending, max_age_grace_timer
@@ -84,7 +85,8 @@ typedef struct channel_data {
   grpc_connectivity_state connectivity_state;
   /* Number of active calls */
   gpr_atm call_count;
-} channel_data;
+};
+}  // namespace
 
 /* Increase the nubmer of active calls. Before the increasement, if there are no
    calls, the max_idle_timer should be cancelled. */
diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc
index f8487f9a9e9..3cb7b136c00 100644
--- a/src/core/ext/filters/message_size/message_size_filter.cc
+++ b/src/core/ext/filters/message_size/message_size_filter.cc
@@ -86,7 +86,8 @@ static void* refcounted_message_size_limits_create_from_json(
   return value;
 }
 
-typedef struct call_data {
+namespace {
+struct call_data {
   grpc_call_combiner* call_combiner;
   message_size_limits limits;
   // Receive closures are chained: we inject this closure as the
@@ -97,13 +98,14 @@ typedef struct call_data {
   grpc_byte_stream** recv_message;
   // Original recv_message_ready callback, invoked after our own.
   grpc_closure* next_recv_message_ready;
-} call_data;
+};
 
-typedef struct channel_data {
+struct channel_data {
   message_size_limits limits;
   // Maps path names to refcounted_message_size_limits structs.
   grpc_slice_hash_table* method_limit_table;
-} channel_data;
+};
+}  // namespace
 
 // Callback invoked when we receive a message.  Here we check the max
 // receive message size.
diff --git a/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.cc b/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.cc
index 555a9134a21..88bb8c71cc6 100644
--- a/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.cc
+++ b/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.cc
@@ -25,7 +25,8 @@
 #include "src/core/lib/surface/channel_init.h"
 #include "src/core/lib/transport/metadata.h"
 
-typedef struct call_data {
+namespace {
+struct call_data {
   // Receive closures are chained: we inject this closure as the
   // recv_initial_metadata_ready up-call on transport_stream_op, and remember to
   // call our next_recv_initial_metadata_ready member after handling it.
@@ -37,7 +38,8 @@ typedef struct call_data {
 
   // Marks whether the workaround is active
   bool workaround_active;
-} call_data;
+};
+}  // namespace
 
 // Find the user agent metadata element in the batch
 static bool get_user_agent_mdelem(const grpc_metadata_batch* batch,
diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc
index 816acf2a233..d47a077251a 100644
--- a/src/core/lib/iomgr/tcp_posix.cc
+++ b/src/core/lib/iomgr/tcp_posix.cc
@@ -63,7 +63,8 @@ typedef size_t msg_iovlen_type;
 
 grpc_core::TraceFlag grpc_tcp_trace(false, "tcp");
 
-typedef struct {
+namespace {
+struct grpc_tcp {
   grpc_endpoint base;
   grpc_fd* em_fd;
   int fd;
@@ -96,12 +97,13 @@ typedef struct {
 
   grpc_resource_user* resource_user;
   grpc_resource_user_slice_allocator slice_allocator;
-} grpc_tcp;
+};
 
-typedef struct backup_poller {
+struct backup_poller {
   gpr_mu* pollset_mu;
   grpc_closure run_poller;
-} backup_poller;
+};
+}  // namespace
 
 #define BACKUP_POLLER_POLLSET(b) ((grpc_pollset*)((b) + 1))
 
diff --git a/src/core/lib/security/transport/client_auth_filter.cc b/src/core/lib/security/transport/client_auth_filter.cc
index cd3c2e3f19d..6a3641f112b 100644
--- a/src/core/lib/security/transport/client_auth_filter.cc
+++ b/src/core/lib/security/transport/client_auth_filter.cc
@@ -37,8 +37,9 @@
 
 #define MAX_CREDENTIALS_METADATA_COUNT 4
 
+namespace {
 /* We can have a per-call credentials. */
-typedef struct {
+struct call_data {
   grpc_call_stack* owning_call;
   grpc_call_combiner* call_combiner;
   grpc_call_credentials* creds;
@@ -57,13 +58,14 @@ typedef struct {
   grpc_closure async_result_closure;
   grpc_closure check_call_host_cancel_closure;
   grpc_closure get_request_metadata_cancel_closure;
-} call_data;
+};
 
 /* We can have a per-channel credentials. */
-typedef struct {
+struct channel_data {
   grpc_channel_security_connector* security_connector;
   grpc_auth_context* auth_context;
-} channel_data;
+};
+}  // namespace
 
 void grpc_auth_metadata_context_reset(
     grpc_auth_metadata_context* auth_md_context) {
diff --git a/src/core/lib/security/transport/server_auth_filter.cc b/src/core/lib/security/transport/server_auth_filter.cc
index 73653f2a668..f82971dc567 100644
--- a/src/core/lib/security/transport/server_auth_filter.cc
+++ b/src/core/lib/security/transport/server_auth_filter.cc
@@ -26,13 +26,14 @@
 #include "src/core/lib/security/transport/auth_filters.h"
 #include "src/core/lib/slice/slice_internal.h"
 
-typedef enum {
+namespace {
+enum async_state {
   STATE_INIT = 0,
   STATE_DONE,
   STATE_CANCELLED,
-} async_state;
+};
 
-typedef struct call_data {
+struct call_data {
   grpc_call_combiner* call_combiner;
   grpc_call_stack* owning_call;
   grpc_transport_stream_op_batch* recv_initial_metadata_batch;
@@ -44,12 +45,13 @@ typedef struct call_data {
   grpc_auth_context* auth_context;
   grpc_closure cancel_closure;
   gpr_atm state;  // async_state
-} call_data;
+};
 
-typedef struct channel_data {
+struct channel_data {
   grpc_auth_context* auth_context;
   grpc_server_credentials* creds;
-} channel_data;
+};
+}  // namespace
 
 static grpc_metadata_array metadata_batch_to_md_array(
     const grpc_metadata_batch* batch) {
diff --git a/src/core/lib/surface/completion_queue.cc b/src/core/lib/surface/completion_queue.cc
index aa5808da4c2..e3eef6ca505 100644
--- a/src/core/lib/surface/completion_queue.cc
+++ b/src/core/lib/surface/completion_queue.cc
@@ -780,10 +780,11 @@ typedef struct {
   bool first_loop;
 } cq_is_finished_arg;
 
-class ExecCtxNext : public grpc_core::ExecCtx {
+class ExecCtxNext final : public grpc_core::ExecCtx {
  public:
   ExecCtxNext(void* arg) : ExecCtx(0), check_ready_to_finish_arg_(arg) {}
 
+ private:
   bool CheckReadyToFinish() override {
     cq_is_finished_arg* a = (cq_is_finished_arg*)check_ready_to_finish_arg_;
     grpc_completion_queue* cq = a->cq;
@@ -811,7 +812,6 @@ class ExecCtxNext : public grpc_core::ExecCtx {
     return !a->first_loop && a->deadline < grpc_core::ExecCtx::Get()->Now();
   }
 
- private:
   void* check_ready_to_finish_arg_;
 };
 
@@ -1035,10 +1035,11 @@ static void del_plucker(grpc_completion_queue* cq, void* tag,
   GPR_UNREACHABLE_CODE(return );
 }
 
-class ExecCtxPluck : public grpc_core::ExecCtx {
+class ExecCtxPluck final : public grpc_core::ExecCtx {
  public:
   ExecCtxPluck(void* arg) : ExecCtx(0), check_ready_to_finish_arg_(arg) {}
 
+ private:
   bool CheckReadyToFinish() override {
     cq_is_finished_arg* a = (cq_is_finished_arg*)check_ready_to_finish_arg_;
     grpc_completion_queue* cq = a->cq;
@@ -1072,7 +1073,6 @@ class ExecCtxPluck : public grpc_core::ExecCtx {
     return !a->first_loop && a->deadline < grpc_core::ExecCtx::Get()->Now();
   }
 
- private:
   void* check_ready_to_finish_arg_;
 };
 
diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc
index f1d428f0a1b..ee98cf2693a 100644
--- a/src/core/lib/surface/server.cc
+++ b/src/core/lib/surface/server.cc
@@ -44,24 +44,23 @@
 #include "src/core/lib/transport/metadata.h"
 #include "src/core/lib/transport/static_metadata.h"
 
-typedef struct listener {
+grpc_core::TraceFlag grpc_server_channel_trace(false, "server_channel");
+
+namespace {
+struct listener {
   void* arg;
   void (*start)(grpc_server* server, void* arg, grpc_pollset** pollsets,
                 size_t pollset_count);
   void (*destroy)(grpc_server* server, void* arg, grpc_closure* closure);
   struct listener* next;
   grpc_closure destroy_done;
-} listener;
+};
 
-typedef struct call_data call_data;
-typedef struct channel_data channel_data;
-typedef struct registered_method registered_method;
+enum requested_call_type { BATCH_CALL, REGISTERED_CALL };
 
-typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type;
+struct registered_method;
 
-grpc_core::TraceFlag grpc_server_channel_trace(false, "server_channel");
-
-typedef struct requested_call {
+struct requested_call {
   gpr_mpscq_node request_link; /* must be first */
   requested_call_type type;
   size_t cq_idx;
@@ -81,15 +80,15 @@ typedef struct requested_call {
       grpc_byte_buffer** optional_payload;
     } registered;
   } data;
-} requested_call;
+};
 
-typedef struct channel_registered_method {
+struct channel_registered_method {
   registered_method* server_registered_method;
   uint32_t flags;
   bool has_host;
   grpc_slice method;
   grpc_slice host;
-} channel_registered_method;
+};
 
 struct channel_data {
   grpc_server* server;
@@ -176,6 +175,7 @@ typedef struct {
   grpc_channel** channels;
   size_t num_channels;
 } channel_broadcaster;
+}  // namespace
 
 struct grpc_server {
   grpc_channel_args* channel_args;

From ae3e857eb60e9f6d195a73b7c576f065a2152780 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Wed, 3 Jan 2018 10:45:56 -0800
Subject: [PATCH 018/161] Revert unneeded changes

---
 src/core/lib/surface/completion_queue.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/core/lib/surface/completion_queue.cc b/src/core/lib/surface/completion_queue.cc
index e3eef6ca505..aa5808da4c2 100644
--- a/src/core/lib/surface/completion_queue.cc
+++ b/src/core/lib/surface/completion_queue.cc
@@ -780,11 +780,10 @@ typedef struct {
   bool first_loop;
 } cq_is_finished_arg;
 
-class ExecCtxNext final : public grpc_core::ExecCtx {
+class ExecCtxNext : public grpc_core::ExecCtx {
  public:
   ExecCtxNext(void* arg) : ExecCtx(0), check_ready_to_finish_arg_(arg) {}
 
- private:
   bool CheckReadyToFinish() override {
     cq_is_finished_arg* a = (cq_is_finished_arg*)check_ready_to_finish_arg_;
     grpc_completion_queue* cq = a->cq;
@@ -812,6 +811,7 @@ class ExecCtxNext final : public grpc_core::ExecCtx {
     return !a->first_loop && a->deadline < grpc_core::ExecCtx::Get()->Now();
   }
 
+ private:
   void* check_ready_to_finish_arg_;
 };
 
@@ -1035,11 +1035,10 @@ static void del_plucker(grpc_completion_queue* cq, void* tag,
   GPR_UNREACHABLE_CODE(return );
 }
 
-class ExecCtxPluck final : public grpc_core::ExecCtx {
+class ExecCtxPluck : public grpc_core::ExecCtx {
  public:
   ExecCtxPluck(void* arg) : ExecCtx(0), check_ready_to_finish_arg_(arg) {}
 
- private:
   bool CheckReadyToFinish() override {
     cq_is_finished_arg* a = (cq_is_finished_arg*)check_ready_to_finish_arg_;
     grpc_completion_queue* cq = a->cq;
@@ -1073,6 +1072,7 @@ class ExecCtxPluck final : public grpc_core::ExecCtx {
     return !a->first_loop && a->deadline < grpc_core::ExecCtx::Get()->Now();
   }
 
+ private:
   void* check_ready_to_finish_arg_;
 };
 

From 3c747f1bd8294aa11c2d9d5aa17099441c193107 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Wed, 3 Jan 2018 14:24:49 -0800
Subject: [PATCH 019/161] Fix tsan

---
 .../ext/transport/chttp2/transport/chttp2_transport.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 5ee1f08e614..97f79d9bdf3 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -233,7 +233,7 @@ void grpc_chttp2_ref_transport(grpc_chttp2_transport* t) { gpr_ref(&t->refs); }
 static const grpc_transport_vtable* get_vtable(void);
 
 // -1 == unset, 0 == disabled, 1 == enabled
-static int flow_control_enabled = -1;
+static gpr_atm flow_control_enabled = -1;
 
 static void init_transport(grpc_chttp2_transport* t,
                            const grpc_channel_args* channel_args,
@@ -520,17 +520,17 @@ static void init_transport(grpc_chttp2_transport* t,
     }
   }
 
-  if (flow_control_enabled == -1) {
+  if (gpr_atm_no_barrier_load(&flow_control_enabled) == -1) {
     char* env_variable = gpr_getenv("GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL");
     if (env_variable != nullptr) {
-      flow_control_enabled = 0;
+      gpr_atm_no_barrier_store(&flow_control_enabled, 0);
     } else {
-      flow_control_enabled = 1;
+      gpr_atm_no_barrier_store(&flow_control_enabled, 1);
     }
     gpr_free(env_variable);
   }
 
-  if (flow_control_enabled) {
+  if (gpr_atm_no_barrier_load(&flow_control_enabled)) {
     t->flow_control.Init<grpc_core::chttp2::TransportFlowControl>(t,
                                                                   enable_bdp);
   } else {

From 5219f3f4c13ce412312975ec5ce7d80b164a72eb Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Wed, 3 Jan 2018 21:26:23 -0800
Subject: [PATCH 020/161] Missing s/NULL/nullptr

---
 src/core/lib/iomgr/exec_ctx.h                 |  2 +-
 src/core/lib/iomgr/gethostname_sysconf.cc     |  2 +-
 src/core/lib/iomgr/wakeup_fd_nospecial.cc     |  2 +-
 src/core/lib/support/env_posix.cc             |  4 ++--
 src/core/lib/support/fork.cc                  |  2 +-
 src/core/lib/support/log_posix.cc             |  8 ++++----
 src/core/lib/support/time_posix.cc            |  2 +-
 src/core/lib/transport/error_utils.cc         |  2 +-
 test/core/iomgr/fd_conservation_posix_test.cc |  2 +-
 test/core/iomgr/resource_quota_test.cc        | 14 +++++++-------
 10 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/src/core/lib/iomgr/exec_ctx.h b/src/core/lib/iomgr/exec_ctx.h
index 8f8d518def7..5c6007c3c70 100644
--- a/src/core/lib/iomgr/exec_ctx.h
+++ b/src/core/lib/iomgr/exec_ctx.h
@@ -111,7 +111,7 @@ class ExecCtx {
 
   /** Checks if there is work to be done */
   bool HasWork() {
-    return combiner_data_.active_combiner != NULL ||
+    return combiner_data_.active_combiner != nullptr ||
            !grpc_closure_list_empty(closure_list_);
   }
 
diff --git a/src/core/lib/iomgr/gethostname_sysconf.cc b/src/core/lib/iomgr/gethostname_sysconf.cc
index e099fbd3888..3d74e033388 100644
--- a/src/core/lib/iomgr/gethostname_sysconf.cc
+++ b/src/core/lib/iomgr/gethostname_sysconf.cc
@@ -30,7 +30,7 @@ char* grpc_gethostname() {
   char* hostname = (char*)gpr_malloc(host_name_max);
   if (gethostname(hostname, host_name_max) != 0) {
     gpr_free(hostname);
-    return NULL;
+    return nullptr;
   }
   return hostname;
 }
diff --git a/src/core/lib/iomgr/wakeup_fd_nospecial.cc b/src/core/lib/iomgr/wakeup_fd_nospecial.cc
index 4c20b8c1b71..c2b525a2544 100644
--- a/src/core/lib/iomgr/wakeup_fd_nospecial.cc
+++ b/src/core/lib/iomgr/wakeup_fd_nospecial.cc
@@ -31,6 +31,6 @@
 static int check_availability_invalid(void) { return 0; }
 
 const grpc_wakeup_fd_vtable grpc_specialized_wakeup_fd_vtable = {
-    NULL, NULL, NULL, NULL, check_availability_invalid};
+    nullptr, nullptr, nullptr, nullptr, check_availability_invalid};
 
 #endif /* GRPC_POSIX_NO_SPECIAL_WAKEUP_FD */
diff --git a/src/core/lib/support/env_posix.cc b/src/core/lib/support/env_posix.cc
index 7bea31ca550..8146330555c 100644
--- a/src/core/lib/support/env_posix.cc
+++ b/src/core/lib/support/env_posix.cc
@@ -31,12 +31,12 @@
 
 const char* gpr_getenv_silent(const char* name, char** dst) {
   *dst = gpr_getenv(name);
-  return NULL;
+  return nullptr;
 }
 
 char* gpr_getenv(const char* name) {
   char* result = getenv(name);
-  return result == NULL ? result : gpr_strdup(result);
+  return result == nullptr ? result : gpr_strdup(result);
 }
 
 void gpr_setenv(const char* name, const char* value) {
diff --git a/src/core/lib/support/fork.cc b/src/core/lib/support/fork.cc
index d59ca5584cb..dc291c40804 100644
--- a/src/core/lib/support/fork.cc
+++ b/src/core/lib/support/fork.cc
@@ -39,7 +39,7 @@ void grpc_fork_support_init() {
 #else
   fork_support_enabled = 0;
   char* env = gpr_getenv("GRPC_ENABLE_FORK_SUPPORT");
-  if (env != NULL) {
+  if (env != nullptr) {
     static const char* truthy[] = {"yes",  "Yes",  "YES", "true",
                                    "True", "TRUE", "1"};
     for (size_t i = 0; i < GPR_ARRAY_SIZE(truthy); i++) {
diff --git a/src/core/lib/support/log_posix.cc b/src/core/lib/support/log_posix.cc
index 9fab480a8d4..6f93cdefcdf 100644
--- a/src/core/lib/support/log_posix.cc
+++ b/src/core/lib/support/log_posix.cc
@@ -35,15 +35,15 @@ static intptr_t gettid(void) { return (intptr_t)pthread_self(); }
 void gpr_log(const char* file, int line, gpr_log_severity severity,
              const char* format, ...) {
   char buf[64];
-  char* allocated = NULL;
-  char* message = NULL;
+  char* allocated = nullptr;
+  char* message = nullptr;
   int ret;
   va_list args;
   va_start(args, format);
   ret = vsnprintf(buf, sizeof(buf), format, args);
   va_end(args);
   if (ret < 0) {
-    message = NULL;
+    message = nullptr;
   } else if ((size_t)ret <= sizeof(buf) - 1) {
     message = buf;
   } else {
@@ -66,7 +66,7 @@ void gpr_default_log(gpr_log_func_args* args) {
 
   timer = (time_t)now.tv_sec;
   final_slash = strrchr(args->file, '/');
-  if (final_slash == NULL)
+  if (final_slash == nullptr)
     display_file = args->file;
   else
     display_file = final_slash + 1;
diff --git a/src/core/lib/support/time_posix.cc b/src/core/lib/support/time_posix.cc
index 47a849480f5..b2087c93cfe 100644
--- a/src/core/lib/support/time_posix.cc
+++ b/src/core/lib/support/time_posix.cc
@@ -107,7 +107,7 @@ static gpr_timespec now_impl(gpr_clock_type clock) {
   now.clock_type = clock;
   switch (clock) {
     case GPR_CLOCK_REALTIME:
-      gettimeofday(&now_tv, NULL);
+      gettimeofday(&now_tv, nullptr);
       now.tv_sec = now_tv.tv_sec;
       now.tv_nsec = now_tv.tv_usec * 1000;
       break;
diff --git a/src/core/lib/transport/error_utils.cc b/src/core/lib/transport/error_utils.cc
index ffaf327081f..891576f4baf 100644
--- a/src/core/lib/transport/error_utils.cc
+++ b/src/core/lib/transport/error_utils.cc
@@ -70,7 +70,7 @@ void grpc_error_get_status(grpc_error* error, grpc_millis deadline,
   }
   if (code != nullptr) *code = status;
 
-  if (error_string != NULL && status != GRPC_STATUS_OK) {
+  if (error_string != nullptr && status != GRPC_STATUS_OK) {
     *error_string = gpr_strdup(grpc_error_string(error));
   }
 
diff --git a/test/core/iomgr/fd_conservation_posix_test.cc b/test/core/iomgr/fd_conservation_posix_test.cc
index aaa14010f8d..4866e350d53 100644
--- a/test/core/iomgr/fd_conservation_posix_test.cc
+++ b/test/core/iomgr/fd_conservation_posix_test.cc
@@ -43,7 +43,7 @@ int main(int argc, char** argv) {
         grpc_resource_quota_create("fd_conservation_posix_test");
 
     for (i = 0; i < 100; i++) {
-      p = grpc_iomgr_create_endpoint_pair("test", NULL);
+      p = grpc_iomgr_create_endpoint_pair("test", nullptr);
       grpc_endpoint_destroy(p.client);
       grpc_endpoint_destroy(p.server);
       grpc_core::ExecCtx::Get()->Flush();
diff --git a/test/core/iomgr/resource_quota_test.cc b/test/core/iomgr/resource_quota_test.cc
index ae26f72701b..07682d26308 100644
--- a/test/core/iomgr/resource_quota_test.cc
+++ b/test/core/iomgr/resource_quota_test.cc
@@ -118,7 +118,7 @@ static void test_instant_alloc_then_free(void) {
   grpc_resource_user* usr = grpc_resource_user_create(q, "usr");
   {
     grpc_core::ExecCtx exec_ctx;
-    grpc_resource_user_alloc(usr, 1024, NULL);
+    grpc_resource_user_alloc(usr, 1024, nullptr);
   }
   {
     grpc_core::ExecCtx exec_ctx;
@@ -136,7 +136,7 @@ static void test_instant_alloc_free_pair(void) {
   grpc_resource_user* usr = grpc_resource_user_create(q, "usr");
   {
     grpc_core::ExecCtx exec_ctx;
-    grpc_resource_user_alloc(usr, 1024, NULL);
+    grpc_resource_user_alloc(usr, 1024, nullptr);
     grpc_resource_user_free(usr, 1024);
   }
   grpc_resource_quota_unref(q);
@@ -565,7 +565,7 @@ static void test_resource_user_stays_allocated_until_memory_released(void) {
   grpc_resource_user* usr = grpc_resource_user_create(q, "usr");
   {
     grpc_core::ExecCtx exec_ctx;
-    grpc_resource_user_alloc(usr, 1024, NULL);
+    grpc_resource_user_alloc(usr, 1024, nullptr);
   }
   {
     grpc_core::ExecCtx exec_ctx;
@@ -608,8 +608,8 @@ test_resource_user_stays_allocated_and_reclaimers_unrun_until_memory_released(
       grpc_core::ExecCtx exec_ctx;
       grpc_resource_user_alloc(usr, 1024, set_event(&allocated));
       grpc_core::ExecCtx::Get()->Flush();
-      GPR_ASSERT(gpr_event_wait(&allocated,
-                                grpc_timeout_seconds_to_deadline(5)) != NULL);
+      GPR_ASSERT(gpr_event_wait(&allocated, grpc_timeout_seconds_to_deadline(
+                                                5)) != nullptr);
       GPR_ASSERT(gpr_event_wait(&reclaimer_cancelled,
                                 grpc_timeout_milliseconds_to_deadline(100)) ==
                  nullptr);
@@ -667,8 +667,8 @@ static void test_reclaimers_can_be_posted_repeatedly(void) {
       grpc_core::ExecCtx exec_ctx;
       grpc_resource_user_alloc(usr, 1024, set_event(&allocated));
       grpc_core::ExecCtx::Get()->Flush();
-      GPR_ASSERT(gpr_event_wait(&allocated,
-                                grpc_timeout_seconds_to_deadline(5)) != NULL);
+      GPR_ASSERT(gpr_event_wait(&allocated, grpc_timeout_seconds_to_deadline(
+                                                5)) != nullptr);
       GPR_ASSERT(gpr_event_wait(&reclaimer_done,
                                 grpc_timeout_seconds_to_deadline(5)) !=
                  nullptr);

From bddffb24b4158aeeab9814822491c05c5af29167 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Wed, 3 Jan 2018 22:37:29 -0800
Subject: [PATCH 021/161] Fix objc compile

---
 src/core/ext/transport/chttp2/transport/flow_control.cc | 6 +++---
 src/core/ext/transport/chttp2/transport/flow_control.h  | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/flow_control.cc b/src/core/ext/transport/chttp2/transport/flow_control.cc
index b4864077bdd..3013db23f70 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.cc
+++ b/src/core/ext/transport/chttp2/transport/flow_control.cc
@@ -337,7 +337,7 @@ double TransportFlowControl::SmoothLogBdp(double value) {
 }
 
 FlowControlAction::Urgency TransportFlowControl::DeltaUrgency(
-    int32_t value, grpc_chttp2_setting_id setting_id) {
+    int64_t value, grpc_chttp2_setting_id setting_id) {
   int64_t delta =
       (int64_t)value - (int64_t)t_->settings[GRPC_LOCAL_SETTINGS][setting_id];
   // TODO(ncteisen): tune this
@@ -363,7 +363,7 @@ FlowControlAction TransportFlowControl::PeriodicUpdate() {
     action.set_send_initial_window_update(
         DeltaUrgency(target_initial_window_size_,
                      GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE),
-        target_initial_window_size_);
+        (uint32_t)target_initial_window_size_);
 
     // get bandwidth estimate and update max_frame accordingly.
     double bw_dbl = bdp_estimator_.EstimateBandwidth();
@@ -373,7 +373,7 @@ FlowControlAction TransportFlowControl::PeriodicUpdate() {
                 target_initial_window_size_),
         16384, 16777215);
     action.set_send_max_frame_size_update(
-        DeltaUrgency(frame_size, GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE),
+        DeltaUrgency((int64_t)frame_size, GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE),
         frame_size);
   }
   return UpdateAction(action);
diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h
index 789f755c386..38b9f50c8d4 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.h
+++ b/src/core/ext/transport/chttp2/transport/flow_control.h
@@ -304,7 +304,7 @@ class TransportFlowControl final : public TransportFlowControlBase {
  private:
   double TargetLogBdp();
   double SmoothLogBdp(double value);
-  FlowControlAction::Urgency DeltaUrgency(int32_t value,
+  FlowControlAction::Urgency DeltaUrgency(int64_t value,
                                           grpc_chttp2_setting_id setting_id);
 
   FlowControlAction UpdateAction(FlowControlAction action) {

From 91b56b81d554ce911dd18dee33f23e77b4d31c61 Mon Sep 17 00:00:00 2001
From: murgatroid99 <mlumish@google.com>
Date: Thu, 4 Jan 2018 10:57:59 -0800
Subject: [PATCH 022/161] Mark tcp errors as UNAVAILABLE in UV tcp code

---
 src/core/lib/iomgr/tcp_uv.cc | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/src/core/lib/iomgr/tcp_uv.cc b/src/core/lib/iomgr/tcp_uv.cc
index 40f4006203e..327bd3d1151 100644
--- a/src/core/lib/iomgr/tcp_uv.cc
+++ b/src/core/lib/iomgr/tcp_uv.cc
@@ -65,6 +65,17 @@ typedef struct {
   grpc_pollset* pollset;
 } grpc_tcp;
 
+static grpc_error* tcp_annotate_error(grpc_error* src_error, grpc_tcp* tcp) {
+  return grpc_error_set_str(
+      grpc_error_set_int(
+          src_error,
+          /* All tcp errors are marked with UNAVAILABLE so that application may
+           * choose to retry. */
+          GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE),
+      GRPC_ERROR_STR_TARGET_ADDRESS,
+      grpc_slice_from_copied_string(tcp->peer_string));
+}
+
 static void tcp_free(grpc_exec_ctx* exec_ctx, grpc_tcp* tcp) {
   grpc_resource_user_unref(exec_ctx, tcp->resource_user);
   gpr_free(tcp->handle);
@@ -166,7 +177,8 @@ static void read_callback(uv_stream_t* stream, ssize_t nread,
   // TODO(murgatroid99): figure out what the return value here means
   uv_read_stop(stream);
   if (nread == UV_EOF) {
-    error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("EOF");
+    error =
+        tcp_annotate_error(GRPC_ERROR_CREATE_FROM_STATIC_STRING("EOF"), tcp);
     grpc_slice_buffer_reset_and_unref_internal(&exec_ctx, tcp->read_slices);
   } else if (nread > 0) {
     // Successful read
@@ -181,7 +193,8 @@ static void read_callback(uv_stream_t* stream, ssize_t nread,
     }
   } else {
     // nread < 0: Error
-    error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Read failed");
+    error = tcp_annotate_error(
+        GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Read failed"), tcp);
     grpc_slice_buffer_reset_and_unref_internal(&exec_ctx, tcp->read_slices);
   }
   call_read_cb(&exec_ctx, tcp, error);
@@ -200,7 +213,9 @@ static void tcp_read_allocation_done(grpc_exec_ctx* exec_ctx, void* tcpp,
     status =
         uv_read_start((uv_stream_t*)tcp->handle, alloc_uv_buf, read_callback);
     if (status != 0) {
-      error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Read failed at start");
+      error = tcp_annotate_error(
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Read failed at start"),
+          tcp);
       error = grpc_error_set_str(
           error, GRPC_ERROR_STR_OS_ERROR,
           grpc_slice_from_static_string(uv_strerror(status)));
@@ -241,7 +256,8 @@ static void write_callback(uv_write_t* req, int status) {
   if (status == 0) {
     error = GRPC_ERROR_NONE;
   } else {
-    error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Write failed");
+    error = tcp_annotate_error(
+        GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Write failed"), tcp);
   }
   if (grpc_tcp_trace.enabled()) {
     const char* str = grpc_error_string(error);
@@ -275,9 +291,10 @@ static void uv_endpoint_write(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep,
   }
 
   if (tcp->shutting_down) {
-    GRPC_CLOSURE_SCHED(
-        exec_ctx, cb,
-        GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP socket is shutting down"));
+    GRPC_CLOSURE_SCHED(exec_ctx, cb,
+                       tcp_annotate_error(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                                              "TCP socket is shutting down"),
+                                          tcp));
     return;
   }
 

From 6535cfd14001a433f1af5da8476b9b1247fce689 Mon Sep 17 00:00:00 2001
From: Ken Payson <kpayson@google.com>
Date: Thu, 4 Jan 2018 12:56:52 -0800
Subject: [PATCH 023/161] Set error status correctly on server side

---
 CMakeLists.txt                                |   2 +
 Makefile                                      |   2 +
 gRPC-Core.podspec                             |   1 +
 grpc.gyp                                      |   2 +
 src/core/lib/surface/call.cc                  |   5 +-
 test/core/end2end/end2end_nosec_tests.cc      |   8 +
 test/core/end2end/end2end_tests.cc            |   8 +
 test/core/end2end/gen_build_yaml.py           |   1 +
 test/core/end2end/generate_tests.bzl          |   1 +
 test/core/end2end/tests/filter_status_code.cc | 353 ++++++++
 .../generated/sources_and_headers.json        |   2 +
 tools/run_tests/generated/tests.json          | 839 +++++++++++++++++-
 12 files changed, 1197 insertions(+), 27 deletions(-)
 create mode 100644 test/core/end2end/tests/filter_status_code.cc

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8eff902f6b0..eed12052685 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4618,6 +4618,7 @@ add_library(end2end_tests
   test/core/end2end/tests/filter_call_init_fails.cc
   test/core/end2end/tests/filter_causes_close.cc
   test/core/end2end/tests/filter_latency.cc
+  test/core/end2end/tests/filter_status_code.cc
   test/core/end2end/tests/graceful_server_shutdown.cc
   test/core/end2end/tests/high_initial_seqno.cc
   test/core/end2end/tests/hpack_size.cc
@@ -4719,6 +4720,7 @@ add_library(end2end_nosec_tests
   test/core/end2end/tests/filter_call_init_fails.cc
   test/core/end2end/tests/filter_causes_close.cc
   test/core/end2end/tests/filter_latency.cc
+  test/core/end2end/tests/filter_status_code.cc
   test/core/end2end/tests/graceful_server_shutdown.cc
   test/core/end2end/tests/high_initial_seqno.cc
   test/core/end2end/tests/hpack_size.cc
diff --git a/Makefile b/Makefile
index e51882c3640..c962a128735 100644
--- a/Makefile
+++ b/Makefile
@@ -8557,6 +8557,7 @@ LIBEND2END_TESTS_SRC = \
     test/core/end2end/tests/filter_call_init_fails.cc \
     test/core/end2end/tests/filter_causes_close.cc \
     test/core/end2end/tests/filter_latency.cc \
+    test/core/end2end/tests/filter_status_code.cc \
     test/core/end2end/tests/graceful_server_shutdown.cc \
     test/core/end2end/tests/high_initial_seqno.cc \
     test/core/end2end/tests/hpack_size.cc \
@@ -8655,6 +8656,7 @@ LIBEND2END_NOSEC_TESTS_SRC = \
     test/core/end2end/tests/filter_call_init_fails.cc \
     test/core/end2end/tests/filter_causes_close.cc \
     test/core/end2end/tests/filter_latency.cc \
+    test/core/end2end/tests/filter_status_code.cc \
     test/core/end2end/tests/graceful_server_shutdown.cc \
     test/core/end2end/tests/high_initial_seqno.cc \
     test/core/end2end/tests/hpack_size.cc \
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index 708c3436abd..c127660dd50 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -1038,6 +1038,7 @@ Pod::Spec.new do |s|
                       'test/core/end2end/tests/filter_call_init_fails.cc',
                       'test/core/end2end/tests/filter_causes_close.cc',
                       'test/core/end2end/tests/filter_latency.cc',
+                      'test/core/end2end/tests/filter_status_code.cc',
                       'test/core/end2end/tests/graceful_server_shutdown.cc',
                       'test/core/end2end/tests/high_initial_seqno.cc',
                       'test/core/end2end/tests/hpack_size.cc',
diff --git a/grpc.gyp b/grpc.gyp
index c34206b1a5d..9f6cd528b71 100644
--- a/grpc.gyp
+++ b/grpc.gyp
@@ -2378,6 +2378,7 @@
         'test/core/end2end/tests/filter_call_init_fails.cc',
         'test/core/end2end/tests/filter_causes_close.cc',
         'test/core/end2end/tests/filter_latency.cc',
+        'test/core/end2end/tests/filter_status_code.cc',
         'test/core/end2end/tests/graceful_server_shutdown.cc',
         'test/core/end2end/tests/high_initial_seqno.cc',
         'test/core/end2end/tests/hpack_size.cc',
@@ -2450,6 +2451,7 @@
         'test/core/end2end/tests/filter_call_init_fails.cc',
         'test/core/end2end/tests/filter_causes_close.cc',
         'test/core/end2end/tests/filter_latency.cc',
+        'test/core/end2end/tests/filter_status_code.cc',
         'test/core/end2end/tests/graceful_server_shutdown.cc',
         'test/core/end2end/tests/high_initial_seqno.cc',
         'test/core/end2end/tests/hpack_size.cc',
diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc
index a457aaa7a2f..d677576c145 100644
--- a/src/core/lib/surface/call.cc
+++ b/src/core/lib/surface/call.cc
@@ -1851,8 +1851,9 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
         {
           grpc_error* override_error = GRPC_ERROR_NONE;
           if (op->data.send_status_from_server.status != GRPC_STATUS_OK) {
-            override_error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                "Error from server send status");
+            override_error =
+                error_from_status(op->data.send_status_from_server.status,
+                                  "Returned non-ok status");
           }
           if (op->data.send_status_from_server.status_details != nullptr) {
             call->send_extra_metadata[1].md = grpc_mdelem_from_slices(
diff --git a/test/core/end2end/end2end_nosec_tests.cc b/test/core/end2end/end2end_nosec_tests.cc
index 3236feea56e..6318550ad86 100644
--- a/test/core/end2end/end2end_nosec_tests.cc
+++ b/test/core/end2end/end2end_nosec_tests.cc
@@ -68,6 +68,8 @@ extern void filter_causes_close(grpc_end2end_test_config config);
 extern void filter_causes_close_pre_init(void);
 extern void filter_latency(grpc_end2end_test_config config);
 extern void filter_latency_pre_init(void);
+extern void filter_status_code(grpc_end2end_test_config config);
+extern void filter_status_code_pre_init(void);
 extern void graceful_server_shutdown(grpc_end2end_test_config config);
 extern void graceful_server_shutdown_pre_init(void);
 extern void high_initial_seqno(grpc_end2end_test_config config);
@@ -170,6 +172,7 @@ void grpc_end2end_tests_pre_init(void) {
   filter_call_init_fails_pre_init();
   filter_causes_close_pre_init();
   filter_latency_pre_init();
+  filter_status_code_pre_init();
   graceful_server_shutdown_pre_init();
   high_initial_seqno_pre_init();
   hpack_size_pre_init();
@@ -237,6 +240,7 @@ void grpc_end2end_tests(int argc, char **argv,
     filter_call_init_fails(config);
     filter_causes_close(config);
     filter_latency(config);
+    filter_status_code(config);
     graceful_server_shutdown(config);
     high_initial_seqno(config);
     hpack_size(config);
@@ -356,6 +360,10 @@ void grpc_end2end_tests(int argc, char **argv,
       filter_latency(config);
       continue;
     }
+    if (0 == strcmp("filter_status_code", argv[i])) {
+      filter_status_code(config);
+      continue;
+    }
     if (0 == strcmp("graceful_server_shutdown", argv[i])) {
       graceful_server_shutdown(config);
       continue;
diff --git a/test/core/end2end/end2end_tests.cc b/test/core/end2end/end2end_tests.cc
index ca9443b642d..9d8dfd67238 100644
--- a/test/core/end2end/end2end_tests.cc
+++ b/test/core/end2end/end2end_tests.cc
@@ -70,6 +70,8 @@ extern void filter_causes_close(grpc_end2end_test_config config);
 extern void filter_causes_close_pre_init(void);
 extern void filter_latency(grpc_end2end_test_config config);
 extern void filter_latency_pre_init(void);
+extern void filter_status_code(grpc_end2end_test_config config);
+extern void filter_status_code_pre_init(void);
 extern void graceful_server_shutdown(grpc_end2end_test_config config);
 extern void graceful_server_shutdown_pre_init(void);
 extern void high_initial_seqno(grpc_end2end_test_config config);
@@ -173,6 +175,7 @@ void grpc_end2end_tests_pre_init(void) {
   filter_call_init_fails_pre_init();
   filter_causes_close_pre_init();
   filter_latency_pre_init();
+  filter_status_code_pre_init();
   graceful_server_shutdown_pre_init();
   high_initial_seqno_pre_init();
   hpack_size_pre_init();
@@ -241,6 +244,7 @@ void grpc_end2end_tests(int argc, char **argv,
     filter_call_init_fails(config);
     filter_causes_close(config);
     filter_latency(config);
+    filter_status_code(config);
     graceful_server_shutdown(config);
     high_initial_seqno(config);
     hpack_size(config);
@@ -364,6 +368,10 @@ void grpc_end2end_tests(int argc, char **argv,
       filter_latency(config);
       continue;
     }
+    if (0 == strcmp("filter_status_code", argv[i])) {
+      filter_status_code(config);
+      continue;
+    }
     if (0 == strcmp("graceful_server_shutdown", argv[i])) {
       graceful_server_shutdown(config);
       continue;
diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py
index 7c8e7f420ae..e7cf97b2d02 100755
--- a/test/core/end2end/gen_build_yaml.py
+++ b/test/core/end2end/gen_build_yaml.py
@@ -101,6 +101,7 @@ END2END_TESTS = {
     'filter_causes_close': default_test_options._replace(cpu_cost=LOWCPU),
     'filter_call_init_fails': default_test_options,
     'filter_latency': default_test_options._replace(cpu_cost=LOWCPU),
+    'filter_status_code': default_test_options._replace(cpu_cost=LOWCPU),
     'graceful_server_shutdown': default_test_options._replace(cpu_cost=LOWCPU,exclude_inproc=True),
     'hpack_size': default_test_options._replace(proxyable=False,
                                                 traceable=False,
diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl
index b9a42bdb885..1d759e1ecbf 100755
--- a/test/core/end2end/generate_tests.bzl
+++ b/test/core/end2end/generate_tests.bzl
@@ -146,6 +146,7 @@ END2END_TESTS = {
     'trailing_metadata': test_options(),
     'authority_not_supported': test_options(),
     'filter_latency': test_options(),
+    'filter_status_code': test_options(),
     'workaround_cronet_compression': test_options(),
     'write_buffering': test_options(needs_write_buffering=True),
     'write_buffering_at_end': test_options(needs_write_buffering=True),
diff --git a/test/core/end2end/tests/filter_status_code.cc b/test/core/end2end/tests/filter_status_code.cc
new file mode 100644
index 00000000000..261ddd93ec5
--- /dev/null
+++ b/test/core/end2end/tests/filter_status_code.cc
@@ -0,0 +1,353 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "test/core/end2end/end2end_tests.h"
+
+#include <limits.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <grpc/byte_buffer.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/time.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/surface/channel_init.h"
+#include "test/core/end2end/cq_verifier.h"
+
+static bool g_enable_filter = false;
+static gpr_mu g_mu;
+static bool g_client_code_recv;
+static bool g_server_code_recv;
+static gpr_cv g_client_code_cv;
+static gpr_cv g_server_code_cv;
+static grpc_status_code g_client_status_code;
+static grpc_status_code g_server_status_code;
+
+static void* tag(intptr_t t) { return (void*)t; }
+
+static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config,
+                                            const char* test_name,
+                                            grpc_channel_args* client_args,
+                                            grpc_channel_args* server_args) {
+  grpc_end2end_test_fixture f;
+  gpr_log(GPR_INFO, "Running test: %s/%s", test_name, config.name);
+  f = config.create_fixture(client_args, server_args);
+  config.init_server(&f, server_args);
+  config.init_client(&f, client_args);
+  return f;
+}
+
+static gpr_timespec n_seconds_from_now(int n) {
+  return grpc_timeout_seconds_to_deadline(n);
+}
+
+static gpr_timespec five_seconds_from_now(void) {
+  return n_seconds_from_now(5);
+}
+
+static void drain_cq(grpc_completion_queue* cq) {
+  grpc_event ev;
+  do {
+    ev = grpc_completion_queue_next(cq, five_seconds_from_now(), nullptr);
+  } while (ev.type != GRPC_QUEUE_SHUTDOWN);
+}
+
+static void shutdown_server(grpc_end2end_test_fixture* f) {
+  if (!f->server) return;
+  grpc_server_shutdown_and_notify(f->server, f->shutdown_cq, tag(1000));
+  GPR_ASSERT(grpc_completion_queue_pluck(f->shutdown_cq, tag(1000),
+                                         grpc_timeout_seconds_to_deadline(5),
+                                         nullptr)
+                 .type == GRPC_OP_COMPLETE);
+  grpc_server_destroy(f->server);
+  f->server = nullptr;
+}
+
+static void shutdown_client(grpc_end2end_test_fixture* f) {
+  if (!f->client) return;
+  grpc_channel_destroy(f->client);
+  f->client = nullptr;
+}
+
+static void end_test(grpc_end2end_test_fixture* f) {
+  shutdown_server(f);
+  shutdown_client(f);
+
+  grpc_completion_queue_shutdown(f->cq);
+  drain_cq(f->cq);
+  grpc_completion_queue_destroy(f->cq);
+  grpc_completion_queue_destroy(f->shutdown_cq);
+}
+
+// Simple request via a server filter that saves the reported status code.
+static void test_request(grpc_end2end_test_config config) {
+  grpc_call* c;
+  grpc_call* s;
+  grpc_end2end_test_fixture f =
+      begin_test(config, "filter_status_code", nullptr, nullptr);
+  cq_verifier* cqv = cq_verifier_create(f.cq);
+  grpc_op ops[6];
+  grpc_op* op;
+  grpc_metadata_array initial_metadata_recv;
+  grpc_metadata_array trailing_metadata_recv;
+  grpc_metadata_array request_metadata_recv;
+  grpc_call_details call_details;
+  grpc_status_code status;
+  grpc_call_error error;
+  grpc_slice details;
+  int was_cancelled = 2;
+
+  gpr_mu_lock(&g_mu);
+  g_client_status_code = GRPC_STATUS_OK;
+  g_server_status_code = GRPC_STATUS_OK;
+  gpr_mu_unlock(&g_mu);
+
+  gpr_timespec deadline = five_seconds_from_now();
+  c = grpc_channel_create_call(
+      f.client, nullptr, GRPC_PROPAGATE_DEFAULTS, f.cq,
+      grpc_slice_from_static_string("/foo"),
+      get_host_override_slice("foo.test.google.fr", config), deadline, nullptr);
+  GPR_ASSERT(c);
+
+  grpc_metadata_array_init(&initial_metadata_recv);
+  grpc_metadata_array_init(&trailing_metadata_recv);
+  grpc_metadata_array_init(&request_metadata_recv);
+  grpc_call_details_init(&call_details);
+
+  memset(ops, 0, sizeof(ops));
+  op = ops;
+  op->op = GRPC_OP_SEND_INITIAL_METADATA;
+  op->data.send_initial_metadata.count = 0;
+  op->data.send_initial_metadata.metadata = nullptr;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  op->op = GRPC_OP_RECV_INITIAL_METADATA;
+  op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
+  op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
+  op->data.recv_status_on_client.status = &status;
+  op->data.recv_status_on_client.status_details = &details;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), nullptr);
+  GPR_ASSERT(GRPC_CALL_OK == error);
+
+  error =
+      grpc_server_request_call(f.server, &s, &call_details,
+                               &request_metadata_recv, f.cq, f.cq, tag(101));
+  GPR_ASSERT(GRPC_CALL_OK == error);
+
+  CQ_EXPECT_COMPLETION(cqv, tag(101), 1);
+  cq_verify(cqv);
+
+  memset(ops, 0, sizeof(ops));
+  op = ops;
+  op->op = GRPC_OP_SEND_INITIAL_METADATA;
+  op->data.send_initial_metadata.count = 0;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
+  op->data.send_status_from_server.trailing_metadata_count = 0;
+  op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED;
+  grpc_slice status_string = grpc_slice_from_static_string("xyz");
+  op->data.send_status_from_server.status_details = &status_string;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
+  op->data.recv_close_on_server.cancelled = &was_cancelled;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), nullptr);
+  GPR_ASSERT(GRPC_CALL_OK == error);
+
+  CQ_EXPECT_COMPLETION(cqv, tag(102), 1);
+  CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
+  cq_verify(cqv);
+
+  GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED);
+  GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz"));
+
+  grpc_slice_unref(details);
+  grpc_metadata_array_destroy(&initial_metadata_recv);
+  grpc_metadata_array_destroy(&trailing_metadata_recv);
+  grpc_metadata_array_destroy(&request_metadata_recv);
+  grpc_call_details_destroy(&call_details);
+
+  grpc_call_unref(s);
+  grpc_call_unref(c);
+
+  cq_verifier_destroy(cqv);
+
+  end_test(&f);
+  config.tear_down_data(&f);
+
+  // Perform checks after test tear-down
+  // Guards against the case that there's outstanding channel-related work on a
+  // call prior to verification
+  // TODO(https://github.com/grpc/grpc/issues/13915) enable this for windows
+#ifndef GPR_WINDOWS
+  gpr_mu_lock(&g_mu);
+  if (!g_client_code_recv) {
+    GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu,
+                           grpc_timeout_seconds_to_deadline(3)));
+  }
+  if (!g_server_code_recv) {
+    GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu,
+                           grpc_timeout_seconds_to_deadline(3)));
+  }
+  GPR_ASSERT(g_client_status_code == GRPC_STATUS_UNIMPLEMENTED);
+  GPR_ASSERT(g_server_status_code == GRPC_STATUS_UNIMPLEMENTED);
+  gpr_mu_unlock(&g_mu);
+#endif  // GPR_WINDOWS
+}
+
+/*******************************************************************************
+ * Test status_code filter
+ */
+
+static grpc_error* init_call_elem(grpc_call_element* elem,
+                                  const grpc_call_element_args* args) {
+  return GRPC_ERROR_NONE;
+}
+
+static void client_destroy_call_elem(grpc_call_element* elem,
+                                     const grpc_call_final_info* final_info,
+                                     grpc_closure* ignored) {
+  gpr_mu_lock(&g_mu);
+  g_client_status_code = final_info->final_status;
+  g_client_code_recv = true;
+  gpr_cv_signal(&g_client_code_cv);
+  gpr_mu_unlock(&g_mu);
+}
+
+static void server_destroy_call_elem(grpc_call_element* elem,
+                                     const grpc_call_final_info* final_info,
+                                     grpc_closure* ignored) {
+  gpr_mu_lock(&g_mu);
+  g_server_status_code = final_info->final_status;
+  g_server_code_recv = true;
+  gpr_cv_signal(&g_server_code_cv);
+  gpr_mu_unlock(&g_mu);
+}
+
+static grpc_error* init_channel_elem(grpc_channel_element* elem,
+                                     grpc_channel_element_args* args) {
+  return GRPC_ERROR_NONE;
+}
+
+static void destroy_channel_elem(grpc_channel_element* elem) {}
+
+static const grpc_channel_filter test_client_filter = {
+    grpc_call_next_op,
+    grpc_channel_next_op,
+    0,
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
+    client_destroy_call_elem,
+    0,
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_channel_next_get_info,
+    "client_filter_status_code"};
+
+static const grpc_channel_filter test_server_filter = {
+    grpc_call_next_op,
+    grpc_channel_next_op,
+    0,
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
+    server_destroy_call_elem,
+    0,
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_channel_next_get_info,
+    "server_filter_status_code"};
+
+/*******************************************************************************
+ * Registration
+ */
+
+static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) {
+  grpc_channel_filter* filter = (grpc_channel_filter*)arg;
+  if (g_enable_filter) {
+    // Want to add the filter as close to the end as possible, to make
+    // sure that all of the filters work well together.  However, we
+    // can't add it at the very end, because the
+    // connected_channel/client_channel filter must be the last one.
+    // So we add it right before the last one.
+    grpc_channel_stack_builder_iterator* it =
+        grpc_channel_stack_builder_create_iterator_at_last(builder);
+    GPR_ASSERT(grpc_channel_stack_builder_move_prev(it));
+    const bool retval = grpc_channel_stack_builder_add_filter_before(
+        it, filter, nullptr, nullptr);
+    grpc_channel_stack_builder_iterator_destroy(it);
+    return retval;
+  } else {
+    return true;
+  }
+}
+
+static void init_plugin(void) {
+  gpr_mu_init(&g_mu);
+  gpr_cv_init(&g_client_code_cv);
+  gpr_cv_init(&g_server_code_cv);
+  g_client_code_recv = false;
+  g_server_code_recv = false;
+
+  grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX,
+                                   maybe_add_filter,
+                                   (void*)&test_client_filter);
+  grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX,
+                                   maybe_add_filter,
+                                   (void*)&test_client_filter);
+  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX,
+                                   maybe_add_filter,
+                                   (void*)&test_server_filter);
+}
+
+static void destroy_plugin(void) {
+  gpr_cv_destroy(&g_client_code_cv);
+  gpr_cv_destroy(&g_server_code_cv);
+  gpr_mu_destroy(&g_mu);
+}
+
+void filter_status_code(grpc_end2end_test_config config) {
+  g_enable_filter = true;
+  test_request(config);
+  g_enable_filter = false;
+}
+
+void filter_status_code_pre_init(void) {
+  grpc_register_plugin(init_plugin, destroy_plugin);
+}
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 0fc5a25afd3..d432bd0e537 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -7616,6 +7616,7 @@
       "test/core/end2end/tests/filter_call_init_fails.cc", 
       "test/core/end2end/tests/filter_causes_close.cc", 
       "test/core/end2end/tests/filter_latency.cc", 
+      "test/core/end2end/tests/filter_status_code.cc", 
       "test/core/end2end/tests/graceful_server_shutdown.cc", 
       "test/core/end2end/tests/high_initial_seqno.cc", 
       "test/core/end2end/tests/hpack_size.cc", 
@@ -7697,6 +7698,7 @@
       "test/core/end2end/tests/filter_call_init_fails.cc", 
       "test/core/end2end/tests/filter_causes_close.cc", 
       "test/core/end2end/tests/filter_latency.cc", 
+      "test/core/end2end/tests/filter_status_code.cc", 
       "test/core/end2end/tests/graceful_server_shutdown.cc", 
       "test/core/end2end/tests/high_initial_seqno.cc", 
       "test/core/end2end/tests/hpack_size.cc", 
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index 5cf371190c9..98517cba2e7 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -6797,6 +6797,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_census_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -8135,6 +8158,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_compress_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -9430,6 +9476,28 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_fakesec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -10636,6 +10704,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_fd_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -11903,6 +11994,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -13159,6 +13273,25 @@
       "linux"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+pipe_test", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -14343,6 +14476,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+trace_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -15635,6 +15791,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+workarounds_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -16991,6 +17170,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_http_proxy_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -18389,6 +18592,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_load_reporting_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -19745,6 +19971,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_oauth2_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -21067,7 +21317,7 @@
   }, 
   {
     "args": [
-      "graceful_server_shutdown"
+      "filter_status_code"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21091,7 +21341,7 @@
   }, 
   {
     "args": [
-      "high_initial_seqno"
+      "graceful_server_shutdown"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21115,14 +21365,14 @@
   }, 
   {
     "args": [
-      "idempotent_request"
+      "high_initial_seqno"
     ], 
     "ci_platforms": [
       "windows", 
       "linux", 
       "posix"
     ], 
-    "cpu_cost": 1.0, 
+    "cpu_cost": 0.1, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -21139,7 +21389,7 @@
   }, 
   {
     "args": [
-      "invoke_large_request"
+      "idempotent_request"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21163,7 +21413,7 @@
   }, 
   {
     "args": [
-      "large_metadata"
+      "invoke_large_request"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21187,7 +21437,7 @@
   }, 
   {
     "args": [
-      "load_reporting_hook"
+      "large_metadata"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21211,14 +21461,14 @@
   }, 
   {
     "args": [
-      "max_connection_age"
+      "load_reporting_hook"
     ], 
     "ci_platforms": [
       "windows", 
       "linux", 
       "posix"
     ], 
-    "cpu_cost": 0.1, 
+    "cpu_cost": 1.0, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -21235,7 +21485,31 @@
   }, 
   {
     "args": [
-      "max_message_length"
+      "max_connection_age"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_proxy_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
+  {
+    "args": [
+      "max_message_length"
     ], 
     "ci_platforms": [
       "windows", 
@@ -22169,6 +22443,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -23393,6 +23691,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair+trace_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -24577,6 +24899,32 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [
+      "msan"
+    ], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair_1byte_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -25923,6 +26271,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_ssl_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -27207,6 +27578,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_ssl_proxy_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -28364,6 +28759,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_uds_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -29560,6 +29978,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "inproc_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "high_initial_seqno"
@@ -30597,6 +31038,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_census_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -31912,6 +32376,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_compress_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -33112,7 +33599,7 @@
   }, 
   {
     "args": [
-      "graceful_server_shutdown"
+      "filter_status_code"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33135,7 +33622,7 @@
   }, 
   {
     "args": [
-      "high_initial_seqno"
+      "graceful_server_shutdown"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33158,7 +33645,7 @@
   }, 
   {
     "args": [
-      "hpack_size"
+      "high_initial_seqno"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33181,14 +33668,14 @@
   }, 
   {
     "args": [
-      "idempotent_request"
+      "hpack_size"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 1.0, 
+    "cpu_cost": 0.1, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33204,7 +33691,7 @@
   }, 
   {
     "args": [
-      "invoke_large_request"
+      "idempotent_request"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33227,14 +33714,14 @@
   }, 
   {
     "args": [
-      "keepalive_timeout"
+      "invoke_large_request"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 0.1, 
+    "cpu_cost": 1.0, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33250,14 +33737,14 @@
   }, 
   {
     "args": [
-      "large_metadata"
+      "keepalive_timeout"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 1.0, 
+    "cpu_cost": 0.1, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33273,7 +33760,7 @@
   }, 
   {
     "args": [
-      "load_reporting_hook"
+      "large_metadata"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33296,14 +33783,14 @@
   }, 
   {
     "args": [
-      "max_concurrent_streams"
+      "load_reporting_hook"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 0.1, 
+    "cpu_cost": 1.0, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33319,7 +33806,7 @@
   }, 
   {
     "args": [
-      "max_connection_age"
+      "max_concurrent_streams"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33342,7 +33829,30 @@
   }, 
   {
     "args": [
-      "max_message_length"
+      "max_connection_age"
+    ], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_fd_nosec_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
+  {
+    "args": [
+      "max_message_length"
     ], 
     "ci_platforms": [
       "linux", 
@@ -34354,6 +34864,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -35591,6 +36124,25 @@
       "linux"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+pipe_nosec_test", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -36752,6 +37304,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+trace_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -38021,6 +38596,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+workarounds_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -39353,6 +39951,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_http_proxy_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -40728,6 +41350,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_load_reporting_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -41988,6 +42633,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_proxy_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -43068,6 +43737,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -44268,6 +44961,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair+trace_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -45426,6 +46143,32 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [
+      "msan"
+    ], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair_1byte_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -46724,6 +47467,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_uds_nosec_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -47897,6 +48663,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "inproc_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "high_initial_seqno"

From a3997fa50f98ff5dd5baac9c53c2cb8543384e6e Mon Sep 17 00:00:00 2001
From: Alexander Polcyn <apolcyn@google.com>
Date: Thu, 14 Dec 2017 15:58:26 -0800
Subject: [PATCH 024/161] Fix failing ruby distrib tests on ruby 2.0 images

---
 test/distrib/ruby/distribtest.gemspec            | 2 ++
 test/distrib/ruby/run_distrib_test.sh            | 8 +++++++-
 tools/run_tests/artifacts/distribtest_targets.py | 7 ++++++-
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/test/distrib/ruby/distribtest.gemspec b/test/distrib/ruby/distribtest.gemspec
index d72892f46c3..f11f5218d5b 100644
--- a/test/distrib/ruby/distribtest.gemspec
+++ b/test/distrib/ruby/distribtest.gemspec
@@ -14,6 +14,8 @@ Gem::Specification.new do |s|
   s.platform      = Gem::Platform::RUBY
 
   s.add_dependency 'grpc', '>=0'
+  s.add_dependency 'public_suffix', '< 3.0'
+  s.add_dependency 'jwt', '< 2.0'
 
   s.add_development_dependency 'bundler', '~> 1.7'
 end
diff --git a/test/distrib/ruby/run_distrib_test.sh b/test/distrib/ruby/run_distrib_test.sh
index 0c214e38f25..d74f4cd76db 100755
--- a/test/distrib/ruby/run_distrib_test.sh
+++ b/test/distrib/ruby/run_distrib_test.sh
@@ -17,10 +17,16 @@ set -ex
 
 cd $(dirname $0)
 
+ARCH=$1
+PLATFORM=$2
 # Create an indexed local gem source with gRPC gems to test
 GEM_SOURCE=../../../gem_source
 mkdir -p ${GEM_SOURCE}/gems
-cp -r $EXTERNAL_GIT_ROOT/input_artifacts/*.gem ${GEM_SOURCE}/gems
+cp $EXTERNAL_GIT_ROOT/input_artifacts/grpc-*$ARCH-$PLATFORM.gem ${GEM_SOURCE}/gems
+if [[ "$(ls ${GEM_SOURCE}/gems | grep grpc | wc -l)" != 1 ]]; then
+  echo "Sanity check failed. Copied over more than one grpc gem into the gem source directory."
+  exit 1
+fi;
 gem install builder
 gem generate_index --directory ${GEM_SOURCE}
 
diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py
index 9959651b6cd..bf3d7a55e48 100644
--- a/tools/run_tests/artifacts/distribtest_targets.py
+++ b/tools/run_tests/artifacts/distribtest_targets.py
@@ -163,6 +163,10 @@ class RubyDistribTest(object):
     return []
 
   def build_jobspec(self):
+    arch_to_gem_arch = {
+        'x64': 'x86_64',
+        'x86': 'x86',
+    }
     if not self.platform == 'linux':
       raise Exception("Not supported yet.")
 
@@ -170,7 +174,8 @@ class RubyDistribTest(object):
           'tools/dockerfile/distribtest/ruby_%s_%s' % (
               self.docker_suffix,
               self.arch),
-          'test/distrib/ruby/run_distrib_test.sh',
+          'test/distrib/ruby/run_distrib_test.sh %s %s' %
+          (arch_to_gem_arch[self.arch], self.platform),
           copy_rel_path='test/distrib')
 
   def __str__(self):

From c6ae9b562bc14780c1b4022e1851a9a843e5e145 Mon Sep 17 00:00:00 2001
From: Yuchen Zeng <zyc@google.com>
Date: Thu, 4 Jan 2018 16:29:06 -0800
Subject: [PATCH 025/161] Silence openssl 1.1.0 warnings

---
 Makefile                                  | 2 +-
 build.yaml                                | 1 +
 grpc.gyp                                  | 3 +++
 include/grpc/impl/codegen/port_platform.h | 8 ++++++++
 src/core/tsi/ssl_transport_security.cc    | 3 +++
 5 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index e51882c3640..9687e8277c2 100644
--- a/Makefile
+++ b/Makefile
@@ -327,7 +327,7 @@ CXXFLAGS += -std=c++11
 ifeq ($(SYSTEM),Darwin)
 CXXFLAGS += -stdlib=libc++
 endif
-CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1
+CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 -Wno-deprecated-declarations
 COREFLAGS += -fno-rtti -fno-exceptions
 LDFLAGS += -g
 
diff --git a/build.yaml b/build.yaml
index 42d72459811..fef7d6189f7 100644
--- a/build.yaml
+++ b/build.yaml
@@ -5004,6 +5004,7 @@ defaults:
   global:
     COREFLAGS: -fno-rtti -fno-exceptions
     CPPFLAGS: -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1
+      -Wno-deprecated-declarations
     LDFLAGS: -g
   zlib:
     CFLAGS: -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-implicit-function-declaration
diff --git a/grpc.gyp b/grpc.gyp
index c34206b1a5d..06da3a758fb 100644
--- a/grpc.gyp
+++ b/grpc.gyp
@@ -57,6 +57,7 @@
       '-Wno-long-long',
       '-Wno-unused-parameter',
       '-DOSATOMIC_USE_INLINED=1',
+      '-Wno-deprecated-declarations',
     ],
     'ldflags': [
       '-g',
@@ -134,6 +135,7 @@
             '-Wno-long-long',
             '-Wno-unused-parameter',
             '-DOSATOMIC_USE_INLINED=1',
+            '-Wno-deprecated-declarations',
           ],
           'OTHER_CPLUSPLUSFLAGS': [
             '-g',
@@ -143,6 +145,7 @@
             '-Wno-long-long',
             '-Wno-unused-parameter',
             '-DOSATOMIC_USE_INLINED=1',
+            '-Wno-deprecated-declarations',
             '-stdlib=libc++',
             '-std=c++11',
             '-Wno-error=deprecated-declarations'
diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h
index f4bc3eb3e03..1c03197db5d 100644
--- a/include/grpc/impl/codegen/port_platform.h
+++ b/include/grpc/impl/codegen/port_platform.h
@@ -421,6 +421,14 @@ typedef unsigned __int64 uint64_t;
 #endif
 #endif
 
+#ifndef GRPC_UNUSED
+#if defined(__GNUC__) && !defined(__MINGW32__)
+#define GRPC_UNUSED __attribute__((unused))
+#else
+#define GRPC_UNUSED
+#endif
+#endif
+
 #ifndef GPR_PRINT_FORMAT_CHECK
 #ifdef __GNUC__
 #define GPR_PRINT_FORMAT_CHECK(FORMAT_STR, ARGS) \
diff --git a/src/core/tsi/ssl_transport_security.cc b/src/core/tsi/ssl_transport_security.cc
index f35caef6402..229f7efd373 100644
--- a/src/core/tsi/ssl_transport_security.cc
+++ b/src/core/tsi/ssl_transport_security.cc
@@ -116,6 +116,9 @@ typedef struct {
 
 static gpr_once init_openssl_once = GPR_ONCE_INIT;
 static gpr_mu* openssl_mutexes = nullptr;
+static void openssl_locking_cb(int mode, int type, const char* file,
+                               int line) GRPC_UNUSED;
+static unsigned long openssl_thread_id_cb(void) GRPC_UNUSED;
 
 static void openssl_locking_cb(int mode, int type, const char* file, int line) {
   if (mode & CRYPTO_LOCK) {

From cb0486adf810d991d712e26f65ac67b41b505eb4 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Fri, 5 Jan 2018 13:06:56 +0100
Subject: [PATCH 026/161] prepare for SourceLink support

---
 src/csharp/Grpc.Auth/Grpc.Auth.csproj         |  4 ++--
 .../Grpc.Core.Testing.csproj                  |  4 ++--
 src/csharp/Grpc.Core/Grpc.Core.csproj         |  4 ++--
 .../Grpc.Core/SourceLink.csproj.include       | 19 +++++++++++++++++++
 .../Grpc.HealthCheck/Grpc.HealthCheck.csproj  |  4 ++--
 .../Grpc.Reflection/Grpc.Reflection.csproj    |  4 ++--
 6 files changed, 29 insertions(+), 10 deletions(-)
 create mode 100755 src/csharp/Grpc.Core/SourceLink.csproj.include

diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.csproj b/src/csharp/Grpc.Auth/Grpc.Auth.csproj
index bbcbd95be5f..5bbff389487 100755
--- a/src/csharp/Grpc.Auth/Grpc.Auth.csproj
+++ b/src/csharp/Grpc.Auth/Grpc.Auth.csproj
@@ -15,12 +15,12 @@
     <PackageTags>gRPC RPC Protocol HTTP/2 Auth OAuth2</PackageTags>
     <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl>
     <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl>
-    <IncludeSymbols>true</IncludeSymbols>
-    <IncludeSource>true</IncludeSource>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
 
+  <Import Project="..\Grpc.Core\SourceLink.csproj.include" />
+
   <ItemGroup>
     <Compile Include="..\Grpc.Core\Version.cs" />
   </ItemGroup>
diff --git a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj
index 4d6767fa985..40840d4da3e 100755
--- a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj
+++ b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj
@@ -15,12 +15,12 @@
     <PackageTags>gRPC test testing</PackageTags>
     <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl>
     <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl>
-    <IncludeSymbols>true</IncludeSymbols>
-    <IncludeSource>true</IncludeSource>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
 
+  <Import Project="..\Grpc.Core\SourceLink.csproj.include" />
+
   <ItemGroup>
     <Compile Include="..\Grpc.Core\Version.cs" />
   </ItemGroup>
diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj
index d9950b2f201..6d44be7ddd6 100755
--- a/src/csharp/Grpc.Core/Grpc.Core.csproj
+++ b/src/csharp/Grpc.Core/Grpc.Core.csproj
@@ -14,12 +14,12 @@
     <PackageTags>gRPC RPC Protocol HTTP/2</PackageTags>
     <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl>
     <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl>
-    <IncludeSymbols>true</IncludeSymbols>
-    <IncludeSource>true</IncludeSource>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
 
+  <Import Project="SourceLink.csproj.include" />
+
   <ItemGroup>
     <EmbeddedResource Include="..\..\..\etc\roots.pem" />
     <Content Include="..\nativelibs\csharp_ext_macos_x64\libgrpc_csharp_ext.dylib">
diff --git a/src/csharp/Grpc.Core/SourceLink.csproj.include b/src/csharp/Grpc.Core/SourceLink.csproj.include
new file mode 100755
index 00000000000..bdc90d94e6a
--- /dev/null
+++ b/src/csharp/Grpc.Core/SourceLink.csproj.include
@@ -0,0 +1,19 @@
+<!-- Ensure that debugging of the resulting NuGet packages work (we're using SourceLink). -->
+<Project>
+
+  <ItemGroup Label="dotnet pack instructions">
+    <Content Include="$(OutputPath)netstandard1.5\$(PackageId).pdb">
+      <Pack>true</Pack>
+      <PackagePath>lib/netstandard1.5</PackagePath>
+    </Content>
+    <Content Include="$(OutputPath)net45\$(PackageId).pdb">
+      <Pack>true</Pack>
+      <PackagePath>lib/net45</PackagePath>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="SourceLink.Test" Version="2.7.3" PrivateAssets="all" />
+  </ItemGroup>
+
+</Project>
diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj
index 681719d124a..da61253455a 100755
--- a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj
+++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj
@@ -14,12 +14,12 @@
     <PackageTags>gRPC health check</PackageTags>
     <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl>
     <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl>
-    <IncludeSymbols>true</IncludeSymbols>
-    <IncludeSource>true</IncludeSource>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
 
+  <Import Project="..\Grpc.Core\SourceLink.csproj.include" />
+
   <ItemGroup>
     <Compile Include="..\Grpc.Core\Version.cs" />
   </ItemGroup>
diff --git a/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj b/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj
index 704eea5c17a..862ecda5fd9 100755
--- a/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj
+++ b/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj
@@ -14,12 +14,12 @@
     <PackageTags>gRPC reflection</PackageTags>
     <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl>
     <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl>
-    <IncludeSymbols>true</IncludeSymbols>
-    <IncludeSource>true</IncludeSource>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
 
+  <Import Project="..\Grpc.Core\SourceLink.csproj.include" />
+
   <ItemGroup>
     <Compile Include="..\Grpc.Core\Version.cs" />
   </ItemGroup>

From ebebb02d3f3a2fb801e90c7f964578568efe2877 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Fri, 5 Jan 2018 13:50:15 +0100
Subject: [PATCH 027/161] try embedding sources for a start

---
 src/csharp/Grpc.Core/SourceLink.csproj.include | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/csharp/Grpc.Core/SourceLink.csproj.include b/src/csharp/Grpc.Core/SourceLink.csproj.include
index bdc90d94e6a..02ae79fb893 100755
--- a/src/csharp/Grpc.Core/SourceLink.csproj.include
+++ b/src/csharp/Grpc.Core/SourceLink.csproj.include
@@ -13,7 +13,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="SourceLink.Test" Version="2.7.3" PrivateAssets="all" />
+    <PackageReference Include="SourceLink.Embed.AllSourceFiles" Version="2.7.3" PrivateAssets="all" />
   </ItemGroup>
 
 </Project>

From 91bab371297b1e53491548bbd9f3588dea93374d Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Fri, 5 Jan 2018 09:59:34 -0800
Subject: [PATCH 028/161] When building with bazel on a Mac, workaround
 bazelbuild/bazel#4341

---
 include/grpc/impl/codegen/port_platform.h | 13 +++++++++++++
 tools/bazel.rc                            |  1 +
 2 files changed, 14 insertions(+)

diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h
index f4bc3eb3e03..e6bee73ef15 100644
--- a/include/grpc/impl/codegen/port_platform.h
+++ b/include/grpc/impl/codegen/port_platform.h
@@ -195,12 +195,25 @@
 #define GPR_PTHREAD_TLS 1
 #else /* __MAC_OS_X_VERSION_MIN_REQUIRED < __MAC_10_7 */
 #define GPR_CPU_POSIX 1
+/* TODO(vjpai): there is a reported issue in bazel build for Mac where __thread
+   in a header is currently not working (bazelbuild/bazel#4341). Remove
+   the following conditional and use GPR_GCC_TLS when that is fixed */
+#ifndef GRPC_BAZEL_BUILD
 #define GPR_GCC_TLS 1
+#else /* GRPC_BAZEL_BUILD */
+#define GPR_PTHREAD_TLS 1
+#endif /* GRPC_BAZEL_BUILD */
 #define GPR_APPLE_PTHREAD_NAME 1
 #endif
 #else /* __MAC_OS_X_VERSION_MIN_REQUIRED */
 #define GPR_CPU_POSIX 1
+/* TODO(vjpai): Remove the following conditional and use only GPR_GCC_TLS
+   when bazelbuild/bazel#4341 is fixed */
+#ifndef GRPC_BAZEL_BUILD
 #define GPR_GCC_TLS 1
+#else /* GRPC_BAZEL_BUILD */
+#define GPR_PTHREAD_TLS 1
+#endif /* GRPC_BAZEL_BUILD */
 #endif
 #define GPR_POSIX_CRASH_HANDLER 1
 #endif
diff --git a/tools/bazel.rc b/tools/bazel.rc
index c554f039713..8af2fc981d7 100644
--- a/tools/bazel.rc
+++ b/tools/bazel.rc
@@ -1,4 +1,5 @@
 build --client_env=CC=clang
+build --copt -DGRPC_BAZEL_BUILD
 
 build:asan --strip=never
 build:asan --copt -fsanitize-coverage=edge

From 522df1681a566ec4772d62ba286a6ca03765ec2a Mon Sep 17 00:00:00 2001
From: Alexander Polcyn <apolcyn@google.com>
Date: Fri, 5 Jan 2018 10:33:57 -0800
Subject: [PATCH 029/161] add ruby 2.5 to package build on mac

---
 Rakefile                                      | 2 +-
 tools/distrib/build_ruby_environment_macos.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Rakefile b/Rakefile
index d76b9ff657c..c8a8631c928 100755
--- a/Rakefile
+++ b/Rakefile
@@ -113,7 +113,7 @@ task 'gem:native' do
   if RUBY_PLATFORM =~ /darwin/
     FileUtils.touch 'grpc_c.32.ruby'
     FileUtils.touch 'grpc_c.64.ruby'
-    system "rake cross native gem RUBY_CC_VERSION=2.4.0:2.3.0:2.2.2:2.1.5:2.0.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}"
+    system "rake cross native gem RUBY_CC_VERSION=2.5.0:2.4.0:2.3.0:2.2.2:2.1.6:2.0.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}"
   else
     Rake::Task['dlls'].execute
     docker_for_windows "gem update --system && bundle && rake cross native gem RUBY_CC_VERSION=2.4.0:2.3.0:2.2.2:2.1.5:2.0.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}"
diff --git a/tools/distrib/build_ruby_environment_macos.sh b/tools/distrib/build_ruby_environment_macos.sh
index fe0c5a4d70b..af367402554 100644
--- a/tools/distrib/build_ruby_environment_macos.sh
+++ b/tools/distrib/build_ruby_environment_macos.sh
@@ -47,7 +47,7 @@ EOF
 
 MAKE="make -j8"
 
-for v in 2.4.0 2.3.0 2.2.2 2.1.5 2.0.0-p645 ; do
+for v in 2.5.0 2.4.0 2.3.0 2.2.2 2.1.6 2.0.0-p645 ; do
   ccache -c
   rake -f $CROSS_RUBY cross-ruby VERSION=$v HOST=x86_64-darwin11
 done

From 77ad62e0fe119135e8897037aba5249d8729cc56 Mon Sep 17 00:00:00 2001
From: Ken Payson <kpayson@google.com>
Date: Wed, 6 Dec 2017 14:32:34 -0800
Subject: [PATCH 030/161] Fix a Python spinlock bug

---
 src/core/ext/transport/chttp2/transport/writing.cc |  6 +++---
 src/core/lib/iomgr/ev_poll_posix.cc                | 11 ++++++++++-
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc
index 043ca9bb83e..7d7dce34640 100644
--- a/src/core/ext/transport/chttp2/transport/writing.cc
+++ b/src/core/ext/transport/chttp2/transport/writing.cc
@@ -138,9 +138,9 @@ static void report_stall(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
                          const char* staller) {
   gpr_log(
       GPR_DEBUG,
-      "%s:%p stream %d stalled by %s [fc:pending=%" PRIdPTR ":flowed=%" PRId64
-      ":peer_initwin=%d:t_win=%" PRId64 ":s_win=%d:s_delta=%" PRId64 "]",
-      t->peer_string, t, s->id, staller, s->flow_controlled_buffer.length,
+      "%s:%p stream %d stalled by %s [fc:pending=%" PRIdPTR "pending-compressed=%" PRId64
+      ":flowed=%" PRId64 ":peer_initwin=%d:t_win=%" PRId64 ":s_win=%d:s_delta=%" PRId64 "]",
+      t->peer_string, t, s->id, staller, s->flow_controlled_buffer.length, s->compressed_data_buffer.length,
       s->flow_controlled_bytes_flowed,
       t->settings[GRPC_ACKED_SETTINGS]
                  [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE],
diff --git a/src/core/lib/iomgr/ev_poll_posix.cc b/src/core/lib/iomgr/ev_poll_posix.cc
index 006e3ddd2f3..0ec97990964 100644
--- a/src/core/lib/iomgr/ev_poll_posix.cc
+++ b/src/core/lib/iomgr/ev_poll_posix.cc
@@ -71,6 +71,7 @@ struct grpc_fd {
   int shutdown;
   int closed;
   int released;
+  gpr_atm pollhup;
   grpc_error* shutdown_error;
 
   /* The watcher list.
@@ -335,6 +336,7 @@ static grpc_fd* fd_create(int fd, const char* name) {
   r->on_done_closure = nullptr;
   r->closed = 0;
   r->released = 0;
+  gpr_atm_no_barrier_store(&r->pollhup, 0);
   r->read_notifier_pollset = nullptr;
 
   char* name2;
@@ -950,7 +952,8 @@ static grpc_error* pollset_work(grpc_pollset* pollset,
       pfds[0].events = POLLIN;
       pfds[0].revents = 0;
       for (i = 0; i < pollset->fd_count; i++) {
-        if (fd_is_orphaned(pollset->fds[i])) {
+        if (fd_is_orphaned(pollset->fds[i]) ||
+            gpr_atm_no_barrier_load(&pollset->fds[i]->pollhup) == 1) {
           GRPC_FD_UNREF(pollset->fds[i], "multipoller");
         } else {
           pollset->fds[fd_count++] = pollset->fds[i];
@@ -1017,6 +1020,12 @@ static grpc_error* pollset_work(grpc_pollset* pollset,
                       pfds[i].fd, (pfds[i].revents & POLLIN_CHECK) != 0,
                       (pfds[i].revents & POLLOUT_CHECK) != 0, pfds[i].revents);
             }
+            /* This is a mitigation to prevent poll() from spinning on a
+             ** POLLHUP https://github.com/grpc/grpc/pull/13665
+             */
+            if (pfds[i].revents & POLLHUP) {
+              gpr_atm_no_barrier_store(&watchers[i].fd->pollhup, 1);
+            }
             fd_end_poll(&watchers[i], pfds[i].revents & POLLIN_CHECK,
                         pfds[i].revents & POLLOUT_CHECK, pollset);
           }

From f40794953e361b2b5094fe825532498bee2f8ebd Mon Sep 17 00:00:00 2001
From: Alexander Polcyn <apolcyn@google.com>
Date: Fri, 5 Jan 2018 11:29:56 -0800
Subject: [PATCH 031/161] Manual merge conflict resolve: remove exec_ctx param
 from tcp_free

---
 src/core/lib/iomgr/tcp_uv.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/core/lib/iomgr/tcp_uv.cc b/src/core/lib/iomgr/tcp_uv.cc
index c227f5a5150..baa49d5cc5d 100644
--- a/src/core/lib/iomgr/tcp_uv.cc
+++ b/src/core/lib/iomgr/tcp_uv.cc
@@ -76,7 +76,7 @@ static grpc_error* tcp_annotate_error(grpc_error* src_error, grpc_tcp* tcp) {
       grpc_slice_from_copied_string(tcp->peer_string));
 }
 
-static void tcp_free(grpc_exec_ctx* exec_ctx, grpc_tcp* tcp) {
+static void tcp_free(grpc_tcp* tcp) {
   grpc_resource_user_unref(tcp->resource_user);
   gpr_free(tcp->handle);
   gpr_free(tcp->peer_string);

From e374b88d491f58e2cf8dcfba778fc9fc88bc3a06 Mon Sep 17 00:00:00 2001
From: Ken Payson <kpayson@google.com>
Date: Fri, 5 Jan 2018 11:30:25 -0800
Subject: [PATCH 032/161] Add more control flow logging

---
 src/core/ext/transport/chttp2/transport/writing.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc
index 7d7dce34640..9a6f5e9bcf7 100644
--- a/src/core/ext/transport/chttp2/transport/writing.cc
+++ b/src/core/ext/transport/chttp2/transport/writing.cc
@@ -138,10 +138,11 @@ static void report_stall(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
                          const char* staller) {
   gpr_log(
       GPR_DEBUG,
-      "%s:%p stream %d stalled by %s [fc:pending=%" PRIdPTR "pending-compressed=%" PRId64
-      ":flowed=%" PRId64 ":peer_initwin=%d:t_win=%" PRId64 ":s_win=%d:s_delta=%" PRId64 "]",
-      t->peer_string, t, s->id, staller, s->flow_controlled_buffer.length, s->compressed_data_buffer.length,
-      s->flow_controlled_bytes_flowed,
+      "%s:%p stream %d stalled by %s [fc:pending=%" PRIdPTR
+      ":pending-compressed=%" PRIdPTR ":flowed=%" PRId64
+      ":peer_initwin=%d:t_win=%" PRId64 ":s_win=%d:s_delta=%" PRId64 "]",
+      t->peer_string, t, s->id, staller, s->flow_controlled_buffer.length,
+      s->compressed_data_buffer.length, s->flow_controlled_bytes_flowed,
       t->settings[GRPC_ACKED_SETTINGS]
                  [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE],
       t->flow_control->remote_window(),

From 01c2334998b6f1f46df43776b458e59ab7fa32b4 Mon Sep 17 00:00:00 2001
From: murgatroid99 <mlumish@google.com>
Date: Fri, 5 Jan 2018 13:34:52 -0800
Subject: [PATCH 033/161] Update version to v1.8.4

---
 BUILD                                             | 2 +-
 CMakeLists.txt                                    | 2 +-
 Makefile                                          | 4 ++--
 build.yaml                                        | 2 +-
 gRPC-Core.podspec                                 | 2 +-
 gRPC-ProtoRPC.podspec                             | 2 +-
 gRPC-RxLibrary.podspec                            | 2 +-
 gRPC.podspec                                      | 2 +-
 package.xml                                       | 4 ++--
 src/cpp/common/version_cc.cc                      | 2 +-
 src/csharp/Grpc.Core/Version.csproj.include       | 2 +-
 src/csharp/Grpc.Core/VersionInfo.cs               | 4 ++--
 src/csharp/build_packages_dotnetcli.bat           | 2 +-
 src/csharp/build_packages_dotnetcli.sh            | 4 ++--
 src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +-
 src/objective-c/GRPCClient/private/version.h      | 2 +-
 src/objective-c/tests/version.h                   | 2 +-
 src/php/composer.json                             | 2 +-
 src/php/ext/grpc/version.h                        | 2 +-
 src/python/grpcio/grpc/_grpcio_metadata.py        | 2 +-
 src/python/grpcio/grpc_version.py                 | 2 +-
 src/python/grpcio_health_checking/grpc_version.py | 2 +-
 src/python/grpcio_reflection/grpc_version.py      | 2 +-
 src/python/grpcio_testing/grpc_version.py         | 2 +-
 src/python/grpcio_tests/grpc_version.py           | 2 +-
 src/ruby/lib/grpc/version.rb                      | 2 +-
 src/ruby/tools/version.rb                         | 2 +-
 tools/distrib/python/grpcio_tools/grpc_version.py | 2 +-
 tools/doxygen/Doxyfile.c++                        | 2 +-
 tools/doxygen/Doxyfile.c++.internal               | 2 +-
 30 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/BUILD b/BUILD
index ef345e50b8b..a81f1de4ffe 100644
--- a/BUILD
+++ b/BUILD
@@ -43,7 +43,7 @@ g_stands_for = "generous"
 
 core_version = "5.0.0"
 
-version = "1.8.3"
+version = "1.8.4"
 
 GPR_PUBLIC_HDRS = [
     "include/grpc/support/alloc.h",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 461b4d26036..1404729c2c8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -24,7 +24,7 @@
 cmake_minimum_required(VERSION 2.8)
 
 set(PACKAGE_NAME      "grpc")
-set(PACKAGE_VERSION   "1.8.3")
+set(PACKAGE_VERSION   "1.8.4")
 set(PACKAGE_STRING    "${PACKAGE_NAME} ${PACKAGE_VERSION}")
 set(PACKAGE_TARNAME   "${PACKAGE_NAME}-${PACKAGE_VERSION}")
 set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/")
diff --git a/Makefile b/Makefile
index 3d8f77ab6a5..f39e661c9fa 100644
--- a/Makefile
+++ b/Makefile
@@ -412,8 +412,8 @@ Q = @
 endif
 
 CORE_VERSION = 5.0.0
-CPP_VERSION = 1.8.3
-CSHARP_VERSION = 1.8.3
+CPP_VERSION = 1.8.4
+CSHARP_VERSION = 1.8.4
 
 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES))
 CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS)
diff --git a/build.yaml b/build.yaml
index 5cce4e66cc8..d9a854200cc 100644
--- a/build.yaml
+++ b/build.yaml
@@ -14,7 +14,7 @@ settings:
   '#10': See the expand_version.py for all the quirks here
   core_version: 5.0.0
   g_stands_for: generous
-  version: 1.8.3
+  version: 1.8.4
 filegroups:
 - name: census
   public_headers:
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index 030fcee2705..c453d3c39cf 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -22,7 +22,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-Core'
-  version = '1.8.3'
+  version = '1.8.4'
   s.version  = version
   s.summary  = 'Core cross-platform gRPC library, written in C'
   s.homepage = 'https://grpc.io'
diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec
index d8bfae0a6ec..3dc3a9d39e5 100644
--- a/gRPC-ProtoRPC.podspec
+++ b/gRPC-ProtoRPC.podspec
@@ -21,7 +21,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-ProtoRPC'
-  version = '1.8.3'
+  version = '1.8.4'
   s.version  = version
   s.summary  = 'RPC library for Protocol Buffers, based on gRPC'
   s.homepage = 'https://grpc.io'
diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec
index d7182abc136..035556bb360 100644
--- a/gRPC-RxLibrary.podspec
+++ b/gRPC-RxLibrary.podspec
@@ -21,7 +21,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-RxLibrary'
-  version = '1.8.3'
+  version = '1.8.4'
   s.version  = version
   s.summary  = 'Reactive Extensions library for iOS/OSX.'
   s.homepage = 'https://grpc.io'
diff --git a/gRPC.podspec b/gRPC.podspec
index a784ea840c7..267775db6b3 100644
--- a/gRPC.podspec
+++ b/gRPC.podspec
@@ -20,7 +20,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC'
-  version = '1.8.3'
+  version = '1.8.4'
   s.version  = version
   s.summary  = 'gRPC client library for iOS/OSX'
   s.homepage = 'https://grpc.io'
diff --git a/package.xml b/package.xml
index 67dff8f7b95..6eef86ffe25 100644
--- a/package.xml
+++ b/package.xml
@@ -13,8 +13,8 @@
  <date>2017-08-24</date>
  <time>16:06:07</time>
  <version>
-  <release>1.8.3</release>
-  <api>1.8.3</api>
+  <release>1.8.4</release>
+  <api>1.8.4</api>
  </version>
  <stability>
   <release>beta</release>
diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc
index 8eb5a2e3b88..1318528bf6e 100644
--- a/src/cpp/common/version_cc.cc
+++ b/src/cpp/common/version_cc.cc
@@ -22,5 +22,5 @@
 #include <grpc++/grpc++.h>
 
 namespace grpc {
-grpc::string Version() { return "1.8.3"; }
+grpc::string Version() { return "1.8.4"; }
 }  // namespace grpc
diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include
index 3eeda74547a..a1b2520fa52 100755
--- a/src/csharp/Grpc.Core/Version.csproj.include
+++ b/src/csharp/Grpc.Core/Version.csproj.include
@@ -1,7 +1,7 @@
 <!-- This file is generated -->
 <Project>
   <PropertyGroup>
-    <GrpcCsharpVersion>1.8.3</GrpcCsharpVersion>
+    <GrpcCsharpVersion>1.8.4</GrpcCsharpVersion>
     <GoogleProtobufVersion>3.3.0</GoogleProtobufVersion>
   </PropertyGroup>
 </Project>
diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs
index 7cf012ca94d..d85590473b6 100644
--- a/src/csharp/Grpc.Core/VersionInfo.cs
+++ b/src/csharp/Grpc.Core/VersionInfo.cs
@@ -33,11 +33,11 @@ namespace Grpc.Core
         /// <summary>
         /// Current <c>AssemblyFileVersion</c> of gRPC C# assemblies
         /// </summary>
-        public const string CurrentAssemblyFileVersion = "1.8.3.0";
+        public const string CurrentAssemblyFileVersion = "1.8.4.0";
 
         /// <summary>
         /// Current version of gRPC C#
         /// </summary>
-        public const string CurrentVersion = "1.8.3";
+        public const string CurrentVersion = "1.8.4";
     }
 }
diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat
index 86657c92a56..91190265539 100755
--- a/src/csharp/build_packages_dotnetcli.bat
+++ b/src/csharp/build_packages_dotnetcli.bat
@@ -13,7 +13,7 @@
 @rem limitations under the License.
 
 @rem Current package versions
-set VERSION=1.8.3
+set VERSION=1.8.4
 
 @rem Adjust the location of nuget.exe
 set NUGET=C:\nuget\nuget.exe
diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh
index 33c34a013cd..7f2931a3e36 100755
--- a/src/csharp/build_packages_dotnetcli.sh
+++ b/src/csharp/build_packages_dotnetcli.sh
@@ -39,7 +39,7 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts
 dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts
 dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts
 
-nuget pack Grpc.nuspec -Version "1.8.3" -OutputDirectory ../../artifacts
-nuget pack Grpc.Tools.nuspec -Version "1.8.3" -OutputDirectory ../../artifacts
+nuget pack Grpc.nuspec -Version "1.8.4" -OutputDirectory ../../artifacts
+nuget pack Grpc.Tools.nuspec -Version "1.8.4" -OutputDirectory ../../artifacts
 
 (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg)
diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec
index fc4859e018b..3cce237a3ea 100644
--- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec
+++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec
@@ -42,7 +42,7 @@ Pod::Spec.new do |s|
   # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed
   # before them.
   s.name     = '!ProtoCompiler-gRPCPlugin'
-  v = '1.8.3'
+  v = '1.8.4'
   s.version  = v
   s.summary  = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.'
   s.description = <<-DESC
diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h
index 3513a7d6f44..e4a036e963f 100644
--- a/src/objective-c/GRPCClient/private/version.h
+++ b/src/objective-c/GRPCClient/private/version.h
@@ -23,4 +23,4 @@
 // `tools/buildgen/generate_projects.sh`.
 
 
-#define GRPC_OBJC_VERSION_STRING @"1.8.3"
+#define GRPC_OBJC_VERSION_STRING @"1.8.4"
diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h
index 63a0db9bdb9..92836af560a 100644
--- a/src/objective-c/tests/version.h
+++ b/src/objective-c/tests/version.h
@@ -23,5 +23,5 @@
 // `tools/buildgen/generate_projects.sh`.
 
 
-#define GRPC_OBJC_VERSION_STRING @"1.8.3"
+#define GRPC_OBJC_VERSION_STRING @"1.8.4"
 #define GRPC_C_VERSION_STRING @"5.0.0"
diff --git a/src/php/composer.json b/src/php/composer.json
index 7c21402f07b..370286df49a 100644
--- a/src/php/composer.json
+++ b/src/php/composer.json
@@ -2,7 +2,7 @@
   "name": "grpc/grpc-dev",
   "description": "gRPC library for PHP - for Developement use only",
   "license": "Apache-2.0",
-  "version": "1.8.3",
+  "version": "1.8.4",
   "require": {
     "php": ">=5.5.0",
     "google/protobuf": "^v3.3.0"
diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h
index 48ce15fe70a..a16b8b91c29 100644
--- a/src/php/ext/grpc/version.h
+++ b/src/php/ext/grpc/version.h
@@ -20,6 +20,6 @@
 #ifndef VERSION_H
 #define VERSION_H
 
-#define PHP_GRPC_VERSION "1.8.3"
+#define PHP_GRPC_VERSION "1.8.4"
 
 #endif /* VERSION_H */
diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py
index 2f403d1d64f..19eaab26f04 100644
--- a/src/python/grpcio/grpc/_grpcio_metadata.py
+++ b/src/python/grpcio/grpc/_grpcio_metadata.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!!
 
-__version__ = """1.8.3"""
+__version__ = """1.8.4"""
diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py
index b6da32edc1f..f5e9365e40c 100644
--- a/src/python/grpcio/grpc_version.py
+++ b/src/python/grpcio/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!!
 
-VERSION='1.8.3'
+VERSION='1.8.4'
diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py
index f38db221e47..98365031e25 100644
--- a/src/python/grpcio_health_checking/grpc_version.py
+++ b/src/python/grpcio_health_checking/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!!
 
-VERSION='1.8.3'
+VERSION='1.8.4'
diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py
index aa51f09d0d2..4987d75eff9 100644
--- a/src/python/grpcio_reflection/grpc_version.py
+++ b/src/python/grpcio_reflection/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!!
 
-VERSION='1.8.3'
+VERSION='1.8.4'
diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py
index 560630b62d7..2ca3d6991e6 100644
--- a/src/python/grpcio_testing/grpc_version.py
+++ b/src/python/grpcio_testing/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!!
 
-VERSION='1.8.3'
+VERSION='1.8.4'
diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py
index 8f998544ea5..deabf095b47 100644
--- a/src/python/grpcio_tests/grpc_version.py
+++ b/src/python/grpcio_tests/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!!
 
-VERSION='1.8.3'
+VERSION='1.8.4'
diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb
index 4308231da0a..5ec7cc8efac 100644
--- a/src/ruby/lib/grpc/version.rb
+++ b/src/ruby/lib/grpc/version.rb
@@ -14,5 +14,5 @@
 
 # GRPC contains the General RPC module.
 module GRPC
-  VERSION = '1.8.3'
+  VERSION = '1.8.4'
 end
diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb
index 453a88935b7..c8853135bc8 100644
--- a/src/ruby/tools/version.rb
+++ b/src/ruby/tools/version.rb
@@ -14,6 +14,6 @@
 
 module GRPC
   module Tools
-    VERSION = '1.8.3'
+    VERSION = '1.8.4'
   end
 end
diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py
index 2822a981528..33bbc9d06fd 100644
--- a/tools/distrib/python/grpcio_tools/grpc_version.py
+++ b/tools/distrib/python/grpcio_tools/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!!
 
-VERSION='1.8.3'
+VERSION='1.8.4'
diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++
index 36bc63eafc2..eaaef9ea788 100644
--- a/tools/doxygen/Doxyfile.c++
+++ b/tools/doxygen/Doxyfile.c++
@@ -40,7 +40,7 @@ PROJECT_NAME           = "GRPC C++"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.8.3
+PROJECT_NUMBER         = 1.8.4
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index 7ad182544c6..ca60951767a 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -40,7 +40,7 @@ PROJECT_NAME           = "GRPC C++"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.8.3
+PROJECT_NUMBER         = 1.8.4
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

From 44c4c34b32a753181727935e17ca204291ef9af4 Mon Sep 17 00:00:00 2001
From: Muxi Yan <muxi@users.noreply.github.com>
Date: Fri, 5 Jan 2018 17:25:54 -0800
Subject: [PATCH 034/161] Check copyright of Objective-C++ source files

---
 tools/distrib/check_copyright.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/distrib/check_copyright.py b/tools/distrib/check_copyright.py
index 8f782e07c25..0eb2cbe1a25 100755
--- a/tools/distrib/check_copyright.py
+++ b/tools/distrib/check_copyright.py
@@ -48,6 +48,7 @@ LICENSE_PREFIX = {
     '.cc': r'\s*(?://|\*)\s*',
     '.h': r'\s*(?://|\*)\s*',
     '.m': r'\s*\*\s*',
+    '.mm': r'\s*\*\s*',
     '.php': r'\s*\*\s*',
     '.js': r'\s*\*\s*',
     '.py': r'#\s*',

From a3bfddd9580fc52e248306b2157c3d2d5f950251 Mon Sep 17 00:00:00 2001
From: Ken Payson <kpayson@google.com>
Date: Fri, 5 Jan 2018 18:10:32 -0800
Subject: [PATCH 035/161] Fix bug with pollhup workaround

---
 src/core/lib/iomgr/ev_poll_posix.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/core/lib/iomgr/ev_poll_posix.cc b/src/core/lib/iomgr/ev_poll_posix.cc
index 0ec97990964..a914de04bde 100644
--- a/src/core/lib/iomgr/ev_poll_posix.cc
+++ b/src/core/lib/iomgr/ev_poll_posix.cc
@@ -464,7 +464,7 @@ static grpc_error* fd_shutdown_error(grpc_fd* fd) {
 
 static void notify_on_locked(grpc_fd* fd, grpc_closure** st,
                              grpc_closure* closure) {
-  if (fd->shutdown) {
+  if (fd->shutdown || gpr_atm_no_barrier_load(&fd->pollhup)) {
     GRPC_CLOSURE_SCHED(closure,
                        GRPC_ERROR_CREATE_FROM_STATIC_STRING("FD shutdown"));
   } else if (*st == CLOSURE_NOT_READY) {

From 53bfe69f707e3729cd5845091a1282771b7e45ee Mon Sep 17 00:00:00 2001
From: David Garcia Quintas <dgq@google.com>
Date: Thu, 4 Jan 2018 16:39:08 -0800
Subject: [PATCH 036/161] Connected subchannel refactoring

---
 BUILD                                         |   9 +-
 .../filters/client_channel/client_channel.cc  |   6 +-
 .../ext/filters/client_channel/lb_policy.h    |   2 +-
 .../lb_policy/pick_first/pick_first.cc        |  25 +-
 .../lb_policy/round_robin/round_robin.cc      |  15 +-
 .../ext/filters/client_channel/subchannel.cc  | 250 ++++++++++--------
 .../ext/filters/client_channel/subchannel.h   |  56 ++--
 src/core/lib/debug/trace.h                    |   5 +-
 src/core/lib/support/ref_counted.h            |   4 +-
 test/cpp/end2end/client_lb_end2end_test.cc    |  36 +++
 10 files changed, 241 insertions(+), 167 deletions(-)

diff --git a/BUILD b/BUILD
index dba6592f172..290f4c56213 100644
--- a/BUILD
+++ b/BUILD
@@ -544,24 +544,24 @@ grpc_cc_library(
 
 grpc_cc_library(
     name = "debug_location",
-    public_hdrs = ["src/core/lib/support/debug_location.h"],
     language = "c++",
+    public_hdrs = ["src/core/lib/support/debug_location.h"],
 )
 
 grpc_cc_library(
     name = "ref_counted",
-    public_hdrs = ["src/core/lib/support/ref_counted.h"],
     language = "c++",
+    public_hdrs = ["src/core/lib/support/ref_counted.h"],
     deps = [
-        "grpc_trace",
         "debug_location",
+        "grpc_trace",
     ],
 )
 
 grpc_cc_library(
     name = "ref_counted_ptr",
-    public_hdrs = ["src/core/lib/support/ref_counted_ptr.h"],
     language = "c++",
+    public_hdrs = ["src/core/lib/support/ref_counted_ptr.h"],
 )
 
 grpc_cc_library(
@@ -919,6 +919,7 @@ grpc_cc_library(
     deps = [
         "grpc_base",
         "grpc_deadline_filter",
+        "ref_counted",
     ],
 )
 
diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc
index e99022a91b8..4f3b774212f 100644
--- a/src/core/ext/filters/client_channel/client_channel.cc
+++ b/src/core/ext/filters/client_channel/client_channel.cc
@@ -1004,7 +1004,7 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
                                           grpc_error* error) {
   channel_data* chand = (channel_data*)elem->channel_data;
   call_data* calld = (call_data*)elem->call_data;
-  const grpc_connected_subchannel_call_args call_args = {
+  const grpc_connected_subchannel::CallArgs call_args = {
       calld->pollent,                  // pollent
       calld->path,                     // path
       calld->call_start_time,          // start_time
@@ -1013,8 +1013,8 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
       calld->subchannel_call_context,  // context
       calld->call_combiner             // call_combiner
   };
-  grpc_error* new_error = grpc_connected_subchannel_create_call(
-      calld->connected_subchannel, &call_args, &calld->subchannel_call);
+  grpc_error* new_error = calld->connected_subchannel->CreateCall(
+      &call_args, &calld->subchannel_call);
   if (grpc_client_channel_trace.enabled()) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: create subchannel_call=%p: error=%s",
             chand, calld, calld->subchannel_call, grpc_error_string(new_error));
diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h
index 3572c97ed14..628127106b1 100644
--- a/src/core/ext/filters/client_channel/lb_policy.h
+++ b/src/core/ext/filters/client_channel/lb_policy.h
@@ -164,7 +164,7 @@ int grpc_lb_policy_pick_locked(grpc_lb_policy* policy,
                                grpc_call_context_element* context,
                                void** user_data, grpc_closure* on_complete);
 
-/** Perform a connected subchannel ping (see \a grpc_connected_subchannel_ping)
+/** Perform a connected subchannel ping (see \a grpc_connected_subchannel::Ping)
     against one of the connected subchannels managed by \a policy. */
 void grpc_lb_policy_ping_one_locked(grpc_lb_policy* policy,
                                     grpc_closure* on_initiate,
diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
index 0861261359f..a3b05aacaf6 100644
--- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
@@ -225,8 +225,7 @@ static void pf_ping_one_locked(grpc_lb_policy* pol, grpc_closure* on_initiate,
                                grpc_closure* on_ack) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
   if (p->selected) {
-    grpc_connected_subchannel_ping(p->selected->connected_subchannel,
-                                   on_initiate, on_ack);
+    p->selected->connected_subchannel->Ping(on_initiate, on_ack);
   } else {
     GRPC_CLOSURE_SCHED(on_initiate,
                        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Not connected"));
@@ -413,6 +412,18 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
           &p->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
           GRPC_ERROR_REF(error), "selected_not_ready+switch_to_update");
     } else {
+      if (sd->curr_connectivity_state < GRPC_CHANNEL_TRANSIENT_FAILURE) {
+        // Renew notification.
+        grpc_lb_subchannel_data_start_connectivity_watch(sd);
+      } else {  // in transient failure or shutdown. Rely on re-resolution to
+                // recover.
+        p->selected = nullptr;
+        grpc_lb_subchannel_data_stop_connectivity_watch(sd);
+        grpc_lb_subchannel_list_unref_for_connectivity_watch(
+            sd->subchannel_list, "pf_selected_shutdown");
+        grpc_lb_subchannel_data_unref_subchannel(
+            sd, "pf_selected_shutdown");  // Unrefs connected subchannel
+      }
       // TODO(juanlishen): we re-resolve when the selected subchannel goes to
       // TRANSIENT_FAILURE because we used to shut down in this case before
       // re-resolution is introduced. But we need to investigate whether we
@@ -432,16 +443,6 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
                                     sd->curr_connectivity_state,
                                     GRPC_ERROR_REF(error), "selected_changed");
       }
-      if (sd->curr_connectivity_state != GRPC_CHANNEL_SHUTDOWN) {
-        // Renew notification.
-        grpc_lb_subchannel_data_start_connectivity_watch(sd);
-      } else {
-        p->selected = nullptr;
-        grpc_lb_subchannel_data_stop_connectivity_watch(sd);
-        grpc_lb_subchannel_list_unref_for_connectivity_watch(
-            sd->subchannel_list, "pf_selected_shutdown");
-        grpc_lb_subchannel_data_unref_subchannel(sd, "pf_selected_shutdown");
-      }
     }
     return;
   }
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
index b0c84017dfb..0836dad2f6b 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
@@ -442,8 +442,19 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
   // Update state counters and new overall state.
   update_state_counters_locked(sd);
   update_lb_connectivity_status_locked(sd, GRPC_ERROR_REF(error));
+  // If the sd's new state is TRANSIENT_FAILURE, unref the *connected*
+  // subchannel, if any.
+  if (sd->curr_connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+    if (sd->connected_subchannel != nullptr) {
+      GRPC_CONNECTED_SUBCHANNEL_UNREF(sd->connected_subchannel,
+                                      "connected_subchannel_transient_failure");
+      sd->connected_subchannel = nullptr;
+    }
+    // Renew notification.
+    grpc_lb_subchannel_data_start_connectivity_watch(sd);
+  }
   // If the sd's new state is SHUTDOWN, unref the subchannel.
-  if (sd->curr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
+  else if (sd->curr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
     grpc_lb_subchannel_data_stop_connectivity_watch(sd);
     grpc_lb_subchannel_data_unref_subchannel(sd, "rr_connectivity_shutdown");
     grpc_lb_subchannel_list_unref_for_connectivity_watch(
@@ -540,7 +551,7 @@ static void rr_ping_one_locked(grpc_lb_policy* pol, grpc_closure* on_initiate,
         &p->subchannel_list->subchannels[next_ready_index];
     grpc_connected_subchannel* target = GRPC_CONNECTED_SUBCHANNEL_REF(
         selected->connected_subchannel, "rr_ping");
-    grpc_connected_subchannel_ping(target, on_initiate, on_ack);
+    target->Ping(on_initiate, on_ack);
     GRPC_CONNECTED_SUBCHANNEL_UNREF(target, "rr_ping");
   } else {
     GRPC_CLOSURE_SCHED(on_initiate, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc
index f07394d29b2..25615b6326f 100644
--- a/src/core/ext/filters/client_channel/subchannel.cc
+++ b/src/core/ext/filters/client_channel/subchannel.cc
@@ -41,6 +41,7 @@
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/support/debug_location.h"
 #include "src/core/lib/support/manual_constructor.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/channel_init.h"
@@ -96,7 +97,7 @@ struct grpc_subchannel {
   grpc_connect_out_args connecting_result;
 
   /** callback for connection finishing */
-  grpc_closure connected;
+  grpc_closure on_connected;
 
   /** callback for our alarm */
   grpc_closure on_alarm;
@@ -139,11 +140,10 @@ struct grpc_subchannel_call {
 };
 
 #define SUBCHANNEL_CALL_TO_CALL_STACK(call) ((grpc_call_stack*)((call) + 1))
-#define CHANNEL_STACK_FROM_CONNECTION(con) ((grpc_channel_stack*)(con))
 #define CALLSTACK_TO_SUBCHANNEL_CALL(callstack) \
   (((grpc_subchannel_call*)(callstack)) - 1)
 
-static void subchannel_connected(void* subchannel, grpc_error* error);
+static void on_subchannel_connected(void* subchannel, grpc_error* error);
 
 #ifndef NDEBUG
 #define REF_REASON reason
@@ -161,20 +161,20 @@ static void subchannel_connected(void* subchannel, grpc_error* error);
  */
 
 static void connection_destroy(void* arg, grpc_error* error) {
-  grpc_connected_subchannel* c = (grpc_connected_subchannel*)arg;
-  grpc_channel_stack_destroy(CHANNEL_STACK_FROM_CONNECTION(c));
-  gpr_free(c);
+  grpc_channel_stack* stk = (grpc_channel_stack*)arg;
+  grpc_channel_stack_destroy(stk);
+  gpr_free(stk);
 }
 
 grpc_connected_subchannel* grpc_connected_subchannel_ref(
     grpc_connected_subchannel* c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-  GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON);
+  c->Ref(DEBUG_LOCATION, REF_REASON);
   return c;
 }
 
 void grpc_connected_subchannel_unref(
     grpc_connected_subchannel* c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-  GRPC_CHANNEL_STACK_UNREF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON);
+  c->Unref(DEBUG_LOCATION, REF_REASON);
 }
 
 /*
@@ -241,16 +241,15 @@ grpc_subchannel* grpc_subchannel_ref_from_weak_ref(
 }
 
 static void disconnect(grpc_subchannel* c) {
-  grpc_connected_subchannel* con;
   grpc_subchannel_index_unregister(c->key, c);
   gpr_mu_lock(&c->mu);
   GPR_ASSERT(!c->disconnected);
   c->disconnected = true;
   grpc_connector_shutdown(c->connector, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                                             "Subchannel disconnected"));
-  con = GET_CONNECTED_SUBCHANNEL(c, no_barrier);
+  grpc_connected_subchannel* con = GET_CONNECTED_SUBCHANNEL(c, no_barrier);
   if (con != nullptr) {
-    GRPC_CONNECTED_SUBCHANNEL_UNREF(con, "connection");
+    GRPC_CONNECTED_SUBCHANNEL_UNREF(con, "disconnect");
     gpr_atm_no_barrier_store(&c->connected_subchannel, (gpr_atm)0xdeadbeef);
   }
   gpr_mu_unlock(&c->mu);
@@ -372,7 +371,7 @@ grpc_subchannel* grpc_subchannel_create(grpc_connector* connector,
   if (new_args != nullptr) grpc_channel_args_destroy(new_args);
   c->root_external_state_watcher.next = c->root_external_state_watcher.prev =
       &c->root_external_state_watcher;
-  GRPC_CLOSURE_INIT(&c->connected, subchannel_connected, c,
+  GRPC_CLOSURE_INIT(&c->on_connected, on_subchannel_connected, c,
                     grpc_schedule_on_exec_ctx);
   grpc_connectivity_state_init(&c->state_tracker, GRPC_CHANNEL_IDLE,
                                "subchannel");
@@ -395,7 +394,7 @@ static void continue_connect_locked(grpc_subchannel* c) {
   grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_CONNECTING,
                               GRPC_ERROR_NONE, "state_change");
   grpc_connector_connect(c->connector, &args, &c->connecting_result,
-                         &c->connected);
+                         &c->on_connected);
 }
 
 grpc_connectivity_state grpc_subchannel_check_connectivity(grpc_subchannel* c,
@@ -479,9 +478,10 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) {
     const grpc_millis time_til_next =
         c->next_attempt_deadline - grpc_core::ExecCtx::Get()->Now();
     if (time_til_next <= 0) {
-      gpr_log(GPR_INFO, "Retry immediately");
+      gpr_log(GPR_INFO, "Subchannel %p: Retry immediately", c);
     } else {
-      gpr_log(GPR_INFO, "Retry in %" PRIdPTR " milliseconds", time_til_next);
+      gpr_log(GPR_INFO, "Subchannel %p: Retry in %" PRIdPTR " milliseconds", c,
+              time_til_next);
     }
     GRPC_CLOSURE_INIT(&c->on_alarm, on_alarm, c, grpc_schedule_on_exec_ctx);
     grpc_timer_init(&c->alarm, c->next_attempt_deadline, &c->on_alarm);
@@ -525,75 +525,56 @@ void grpc_subchannel_notify_on_state_change(
   }
 }
 
-void grpc_connected_subchannel_process_transport_op(
-    grpc_connected_subchannel* con, grpc_transport_op* op) {
-  grpc_channel_stack* channel_stack = CHANNEL_STACK_FROM_CONNECTION(con);
-  grpc_channel_element* top_elem = grpc_channel_stack_element(channel_stack, 0);
-  top_elem->filter->start_transport_op(top_elem, op);
-}
-
-static void subchannel_on_child_state_changed(void* p, grpc_error* error) {
-  state_watcher* sw = (state_watcher*)p;
-  grpc_subchannel* c = sw->subchannel;
+static void on_connected_subchannel_connectivity_changed(void* p,
+                                                         grpc_error* error) {
+  state_watcher* connected_subchannel_watcher = (state_watcher*)p;
+  grpc_subchannel* c = connected_subchannel_watcher->subchannel;
   gpr_mu* mu = &c->mu;
 
   gpr_mu_lock(mu);
 
+  auto* con = GET_CONNECTED_SUBCHANNEL(c, no_barrier);
   /* if we failed just leave this closure */
-  if (sw->connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-    /* any errors on a subchannel ==> we're done, create a new one */
-    sw->connectivity_state = GRPC_CHANNEL_SHUTDOWN;
+  if (connected_subchannel_watcher->connectivity_state ==
+      GRPC_CHANNEL_TRANSIENT_FAILURE) {
+    if (!c->disconnected && con != nullptr) {
+      GRPC_CONNECTED_SUBCHANNEL_UNREF(con, "transient_failure");
+      gpr_atm_no_barrier_store(&c->connected_subchannel, (gpr_atm) nullptr);
+      grpc_connectivity_state_set(&c->state_tracker,
+                                  GRPC_CHANNEL_TRANSIENT_FAILURE,
+                                  GRPC_ERROR_REF(error), "reflect_child");
+      c->backoff_begun = false;
+      c->backoff->Reset();
+      if (grpc_trace_stream_refcount.enabled()) {
+        gpr_log(GPR_INFO,
+                "Connected subchannel %p of subchannel %p has gone into "
+                "TRANSIENT_FAILURE. Attempting to reconnect.",
+                con, c);
+      }
+      maybe_start_connecting_locked(c);
+      goto done;
+    } else {
+      connected_subchannel_watcher->connectivity_state = GRPC_CHANNEL_SHUTDOWN;
+    }
   }
-  grpc_connectivity_state_set(&c->state_tracker, sw->connectivity_state,
+  grpc_connectivity_state_set(&c->state_tracker,
+                              connected_subchannel_watcher->connectivity_state,
                               GRPC_ERROR_REF(error), "reflect_child");
-  if (sw->connectivity_state != GRPC_CHANNEL_SHUTDOWN) {
-    grpc_connected_subchannel_notify_on_state_change(
-        GET_CONNECTED_SUBCHANNEL(c, no_barrier), nullptr,
-        &sw->connectivity_state, &sw->closure);
+  if (connected_subchannel_watcher->connectivity_state <
+      GRPC_CHANNEL_TRANSIENT_FAILURE) {
     GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
-    sw = nullptr;
+    con->NotifyOnStateChange(nullptr,
+                             &connected_subchannel_watcher->connectivity_state,
+                             &connected_subchannel_watcher->closure);
+    connected_subchannel_watcher = nullptr;
   }
-
+done:
   gpr_mu_unlock(mu);
   GRPC_SUBCHANNEL_WEAK_UNREF(c, "state_watcher");
-  gpr_free(sw);
-}
-
-static void connected_subchannel_state_op(grpc_connected_subchannel* con,
-                                          grpc_pollset_set* interested_parties,
-                                          grpc_connectivity_state* state,
-                                          grpc_closure* closure) {
-  grpc_transport_op* op = grpc_make_transport_op(nullptr);
-  grpc_channel_element* elem;
-  op->connectivity_state = state;
-  op->on_connectivity_state_change = closure;
-  op->bind_pollset_set = interested_parties;
-  elem = grpc_channel_stack_element(CHANNEL_STACK_FROM_CONNECTION(con), 0);
-  elem->filter->start_transport_op(elem, op);
-}
-
-void grpc_connected_subchannel_notify_on_state_change(
-    grpc_connected_subchannel* con, grpc_pollset_set* interested_parties,
-    grpc_connectivity_state* state, grpc_closure* closure) {
-  connected_subchannel_state_op(con, interested_parties, state, closure);
-}
-
-void grpc_connected_subchannel_ping(grpc_connected_subchannel* con,
-                                    grpc_closure* on_initiate,
-                                    grpc_closure* on_ack) {
-  grpc_transport_op* op = grpc_make_transport_op(nullptr);
-  grpc_channel_element* elem;
-  op->send_ping.on_initiate = on_initiate;
-  op->send_ping.on_ack = on_ack;
-  elem = grpc_channel_stack_element(CHANNEL_STACK_FROM_CONNECTION(con), 0);
-  elem->filter->start_transport_op(elem, op);
+  gpr_free(connected_subchannel_watcher);
 }
 
 static bool publish_transport_locked(grpc_subchannel* c) {
-  grpc_connected_subchannel* con;
-  grpc_channel_stack* stk;
-  state_watcher* sw_subchannel;
-
   /* construct channel stack */
   grpc_channel_stack_builder* builder = grpc_channel_stack_builder_create();
   grpc_channel_stack_builder_set_channel_arguments(
@@ -605,8 +586,9 @@ static bool publish_transport_locked(grpc_subchannel* c) {
     grpc_channel_stack_builder_destroy(builder);
     return false;
   }
+  grpc_channel_stack* stk;
   grpc_error* error = grpc_channel_stack_builder_finish(
-      builder, 0, 1, connection_destroy, nullptr, (void**)&con);
+      builder, 0, 1, connection_destroy, nullptr, (void**)&stk);
   if (error != GRPC_ERROR_NONE) {
     grpc_transport_destroy(c->connecting_result.transport);
     gpr_log(GPR_ERROR, "error initializing subchannel stack: %s",
@@ -614,20 +596,21 @@ static bool publish_transport_locked(grpc_subchannel* c) {
     GRPC_ERROR_UNREF(error);
     return false;
   }
-  stk = CHANNEL_STACK_FROM_CONNECTION(con);
   memset(&c->connecting_result, 0, sizeof(c->connecting_result));
 
   /* initialize state watcher */
-  sw_subchannel = (state_watcher*)gpr_malloc(sizeof(*sw_subchannel));
-  sw_subchannel->subchannel = c;
-  sw_subchannel->connectivity_state = GRPC_CHANNEL_READY;
-  GRPC_CLOSURE_INIT(&sw_subchannel->closure, subchannel_on_child_state_changed,
-                    sw_subchannel, grpc_schedule_on_exec_ctx);
+  state_watcher* connected_subchannel_watcher =
+      (state_watcher*)gpr_zalloc(sizeof(*connected_subchannel_watcher));
+  connected_subchannel_watcher->subchannel = c;
+  connected_subchannel_watcher->connectivity_state = GRPC_CHANNEL_READY;
+  GRPC_CLOSURE_INIT(&connected_subchannel_watcher->closure,
+                    on_connected_subchannel_connectivity_changed,
+                    connected_subchannel_watcher, grpc_schedule_on_exec_ctx);
 
   if (c->disconnected) {
-    gpr_free(sw_subchannel);
+    gpr_free(connected_subchannel_watcher);
     grpc_channel_stack_destroy(stk);
-    gpr_free(con);
+    gpr_free(stk);
     return false;
   }
 
@@ -636,6 +619,8 @@ static bool publish_transport_locked(grpc_subchannel* c) {
                     I'd have expected the rel_cas below to be enough, but
                     seemingly it's not.
                     Re-evaluate if we really need this. */
+  grpc_connected_subchannel* con =
+      grpc_core::New<grpc_connected_subchannel>(stk);
   gpr_atm_full_barrier();
   GPR_ASSERT(gpr_atm_rel_cas(&c->connected_subchannel, 0, (gpr_atm)con));
 
@@ -643,9 +628,9 @@ static bool publish_transport_locked(grpc_subchannel* c) {
      ref for connecting is donated to the state watcher */
   GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
   GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting");
-  grpc_connected_subchannel_notify_on_state_change(
-      con, c->pollset_set, &sw_subchannel->connectivity_state,
-      &sw_subchannel->closure);
+  con->NotifyOnStateChange(c->pollset_set,
+                           &connected_subchannel_watcher->connectivity_state,
+                           &connected_subchannel_watcher->closure);
 
   /* signal completion */
   grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_READY,
@@ -653,11 +638,11 @@ static bool publish_transport_locked(grpc_subchannel* c) {
   return true;
 }
 
-static void subchannel_connected(void* arg, grpc_error* error) {
+static void on_subchannel_connected(void* arg, grpc_error* error) {
   grpc_subchannel* c = (grpc_subchannel*)arg;
   grpc_channel_args* delete_channel_args = c->connecting_result.channel_args;
 
-  GRPC_SUBCHANNEL_WEAK_REF(c, "connected");
+  GRPC_SUBCHANNEL_WEAK_REF(c, "on_subchannel_connected");
   gpr_mu_lock(&c->mu);
   c->connecting = false;
   if (c->connecting_result.transport != nullptr &&
@@ -736,36 +721,6 @@ const grpc_subchannel_key* grpc_subchannel_get_key(
   return subchannel->key;
 }
 
-grpc_error* grpc_connected_subchannel_create_call(
-    grpc_connected_subchannel* con,
-    const grpc_connected_subchannel_call_args* args,
-    grpc_subchannel_call** call) {
-  grpc_channel_stack* chanstk = CHANNEL_STACK_FROM_CONNECTION(con);
-  *call = (grpc_subchannel_call*)gpr_arena_alloc(
-      args->arena, sizeof(grpc_subchannel_call) + chanstk->call_stack_size);
-  grpc_call_stack* callstk = SUBCHANNEL_CALL_TO_CALL_STACK(*call);
-  (*call)->connection = GRPC_CONNECTED_SUBCHANNEL_REF(con, "subchannel_call");
-  const grpc_call_element_args call_args = {
-      callstk,            /* call_stack */
-      nullptr,            /* server_transport_data */
-      args->context,      /* context */
-      args->path,         /* path */
-      args->start_time,   /* start_time */
-      args->deadline,     /* deadline */
-      args->arena,        /* arena */
-      args->call_combiner /* call_combiner */
-  };
-  grpc_error* error = grpc_call_stack_init(chanstk, 1, subchannel_call_destroy,
-                                           *call, &call_args);
-  if (error != GRPC_ERROR_NONE) {
-    const char* error_string = grpc_error_string(error);
-    gpr_log(GPR_ERROR, "error: %s", error_string);
-    return error;
-  }
-  grpc_call_stack_set_pollset_or_pollset_set(callstk, args->pollent);
-  return GRPC_ERROR_NONE;
-}
-
 grpc_call_stack* grpc_subchannel_call_get_call_stack(
     grpc_subchannel_call* subchannel_call) {
   return SUBCHANNEL_CALL_TO_CALL_STACK(subchannel_call);
@@ -801,3 +756,70 @@ grpc_arg grpc_create_subchannel_address_arg(const grpc_resolved_address* addr) {
       (char*)GRPC_ARG_SUBCHANNEL_ADDRESS,
       addr->len > 0 ? grpc_sockaddr_to_uri(addr) : gpr_strdup(""));
 }
+
+grpc_connected_subchannel::grpc_connected_subchannel(
+    grpc_channel_stack* channel_stack)
+    : grpc_core::RefCountedWithTracing(&grpc_trace_stream_refcount),
+      channel_stack_(channel_stack) {}
+
+grpc_connected_subchannel* grpc_connected_subchannel::Ref(
+    const grpc_core::DebugLocation& location, const char* reason) {
+  GRPC_CHANNEL_STACK_REF(channel_stack_, REF_REASON);
+  grpc_core::RefCountedWithTracing::Ref(location, reason);
+  return this;
+}
+void grpc_connected_subchannel::Unref(const grpc_core::DebugLocation& location,
+                                      const char* reason) {
+  GRPC_CHANNEL_STACK_UNREF(channel_stack_, REF_REASON);
+  grpc_core::RefCountedWithTracing::Unref(location, reason);
+}
+
+void grpc_connected_subchannel::NotifyOnStateChange(
+    grpc_pollset_set* interested_parties, grpc_connectivity_state* state,
+    grpc_closure* closure) {
+  grpc_transport_op* op = grpc_make_transport_op(nullptr);
+  grpc_channel_element* elem;
+  op->connectivity_state = state;
+  op->on_connectivity_state_change = closure;
+  op->bind_pollset_set = interested_parties;
+  elem = grpc_channel_stack_element(channel_stack_, 0);
+  elem->filter->start_transport_op(elem, op);
+}
+
+void grpc_connected_subchannel::Ping(grpc_closure* on_initiate,
+                                     grpc_closure* on_ack) {
+  grpc_transport_op* op = grpc_make_transport_op(nullptr);
+  grpc_channel_element* elem;
+  op->send_ping.on_initiate = on_initiate;
+  op->send_ping.on_ack = on_ack;
+  elem = grpc_channel_stack_element(channel_stack_, 0);
+  elem->filter->start_transport_op(elem, op);
+}
+
+grpc_error* grpc_connected_subchannel::CreateCall(const CallArgs* args,
+                                                  grpc_subchannel_call** call) {
+  *call = (grpc_subchannel_call*)gpr_arena_alloc(
+      args->arena,
+      sizeof(grpc_subchannel_call) + channel_stack_->call_stack_size);
+  grpc_call_stack* callstk = SUBCHANNEL_CALL_TO_CALL_STACK(*call);
+  (*call)->connection = Ref(DEBUG_LOCATION, "subchannel_call");
+  const grpc_call_element_args call_args = {
+      callstk,            /* call_stack */
+      nullptr,            /* server_transport_data */
+      args->context,      /* context */
+      args->path,         /* path */
+      args->start_time,   /* start_time */
+      args->deadline,     /* deadline */
+      args->arena,        /* arena */
+      args->call_combiner /* call_combiner */
+  };
+  grpc_error* error = grpc_call_stack_init(
+      channel_stack_, 1, subchannel_call_destroy, *call, &call_args);
+  if (error != GRPC_ERROR_NONE) {
+    const char* error_string = grpc_error_string(error);
+    gpr_log(GPR_ERROR, "error: %s", error_string);
+    return error;
+  }
+  grpc_call_stack_set_pollset_or_pollset_set(callstk, args->pollent);
+  return GRPC_ERROR_NONE;
+}
diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h
index 9d34fff07a8..d9a850daaea 100644
--- a/src/core/ext/filters/client_channel/subchannel.h
+++ b/src/core/ext/filters/client_channel/subchannel.h
@@ -23,6 +23,7 @@
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/support/arena.h"
+#include "src/core/lib/support/ref_counted.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/metadata.h"
 
@@ -32,7 +33,6 @@
 /** A (sub-)channel that knows how to connect to exactly one target
     address. Provides a target for load balancing. */
 typedef struct grpc_subchannel grpc_subchannel;
-typedef struct grpc_connected_subchannel grpc_connected_subchannel;
 typedef struct grpc_subchannel_call grpc_subchannel_call;
 typedef struct grpc_subchannel_args grpc_subchannel_args;
 typedef struct grpc_subchannel_key grpc_subchannel_key;
@@ -73,6 +73,34 @@ typedef struct grpc_subchannel_key grpc_subchannel_key;
 #define GRPC_SUBCHANNEL_REF_EXTRA_ARGS
 #endif
 
+class grpc_connected_subchannel : public grpc_core::RefCountedWithTracing {
+ public:
+  struct CallArgs {
+    grpc_polling_entity* pollent;
+    grpc_slice path;
+    gpr_timespec start_time;
+    grpc_millis deadline;
+    gpr_arena* arena;
+    grpc_call_context_element* context;
+    grpc_call_combiner* call_combiner;
+  };
+
+  grpc_connected_subchannel(grpc_channel_stack* channel_stack);
+  grpc_connected_subchannel* Ref(const grpc_core::DebugLocation& location,
+                                 const char* reason);
+  void Unref(const grpc_core::DebugLocation& location, const char* reason);
+  grpc_channel_stack* channel_stack() { return channel_stack_; }
+  void NotifyOnStateChange(grpc_pollset_set* interested_parties,
+                           grpc_connectivity_state* state,
+                           grpc_closure* closure);
+  void Ping(grpc_closure* on_initiate, grpc_closure* on_ack);
+
+  grpc_error* CreateCall(const CallArgs* args, grpc_subchannel_call** call);
+
+ private:
+  grpc_channel_stack* channel_stack_;
+};
+
 grpc_subchannel* grpc_subchannel_ref(
     grpc_subchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 grpc_subchannel* grpc_subchannel_ref_from_weak_ref(
@@ -92,26 +120,6 @@ void grpc_subchannel_call_ref(
 void grpc_subchannel_call_unref(
     grpc_subchannel_call* call GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 
-/** construct a subchannel call */
-typedef struct {
-  grpc_polling_entity* pollent;
-  grpc_slice path;
-  gpr_timespec start_time;
-  grpc_millis deadline;
-  gpr_arena* arena;
-  grpc_call_context_element* context;
-  grpc_call_combiner* call_combiner;
-} grpc_connected_subchannel_call_args;
-
-grpc_error* grpc_connected_subchannel_create_call(
-    grpc_connected_subchannel* connected_subchannel,
-    const grpc_connected_subchannel_call_args* args,
-    grpc_subchannel_call** subchannel_call);
-
-/** process a transport level op */
-void grpc_connected_subchannel_process_transport_op(
-    grpc_connected_subchannel* subchannel, grpc_transport_op* op);
-
 /** poll the current connectivity state of a channel */
 grpc_connectivity_state grpc_subchannel_check_connectivity(
     grpc_subchannel* channel, grpc_error** error);
@@ -121,12 +129,6 @@ grpc_connectivity_state grpc_subchannel_check_connectivity(
 void grpc_subchannel_notify_on_state_change(
     grpc_subchannel* channel, grpc_pollset_set* interested_parties,
     grpc_connectivity_state* state, grpc_closure* notify);
-void grpc_connected_subchannel_notify_on_state_change(
-    grpc_connected_subchannel* channel, grpc_pollset_set* interested_parties,
-    grpc_connectivity_state* state, grpc_closure* notify);
-void grpc_connected_subchannel_ping(grpc_connected_subchannel* channel,
-                                    grpc_closure* on_initiate,
-                                    grpc_closure* on_ack);
 
 /** retrieve the grpc_connected_subchannel - or NULL if called before
     the subchannel becomes connected */
diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h
index 69ddd802222..6ee1b49f747 100644
--- a/src/core/lib/debug/trace.h
+++ b/src/core/lib/debug/trace.h
@@ -88,9 +88,10 @@ class TraceFlag {
 #ifndef NDEBUG
 typedef TraceFlag DebugOnlyTraceFlag;
 #else
-class DebugOnlyTraceFlag {
+class DebugOnlyTraceFlag : public TraceFlag {
  public:
-  DebugOnlyTraceFlag(bool default_enabled, const char* name) {}
+  DebugOnlyTraceFlag(bool default_enabled, const char* name)
+      : TraceFlag(default_enabled, name) {}
   bool enabled() { return false; }
 
  private:
diff --git a/src/core/lib/support/ref_counted.h b/src/core/lib/support/ref_counted.h
index 4c662f91190..f2182baea19 100644
--- a/src/core/lib/support/ref_counted.h
+++ b/src/core/lib/support/ref_counted.h
@@ -23,6 +23,7 @@
 #include <grpc/support/sync.h>
 
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/support/abstract.h"
 #include "src/core/lib/support/debug_location.h"
 #include "src/core/lib/support/memory.h"
 
@@ -97,6 +98,7 @@ class RefCountedWithTracing {
   // Not copyable nor movable.
   RefCountedWithTracing(const RefCountedWithTracing&) = delete;
   RefCountedWithTracing& operator=(const RefCountedWithTracing&) = delete;
+  GRPC_ABSTRACT_BASE_CLASS
 
  protected:
   // Allow Delete() to access destructor.
@@ -110,8 +112,6 @@ class RefCountedWithTracing {
     gpr_ref_init(&refs_, 1);
   }
 
-  virtual ~RefCountedWithTracing() {}
-
  private:
   TraceFlag* trace_flag_ = nullptr;
   gpr_refcount refs_;
diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc
index c6e9577f0ce..5a7e52e9e94 100644
--- a/test/cpp/end2end/client_lb_end2end_test.cc
+++ b/test/cpp/end2end/client_lb_end2end_test.cc
@@ -673,6 +673,42 @@ TEST_F(ClientLbEnd2endTest, RoundRobinReresolve) {
   GPR_ASSERT(gpr_time_cmp(deadline, now) > 0);
 }
 
+TEST_F(ClientLbEnd2endTest, RoundRobinSingleReconnect) {
+  const int kNumServers = 3;
+  StartServers(kNumServers);
+  const auto ports = GetServersPorts();
+  ResetStub(ports, "round_robin");
+  SetNextResolution(ports);
+  for (size_t i = 0; i < kNumServers; ++i) WaitForServer(i);
+  for (size_t i = 0; i < servers_.size(); ++i) {
+    CheckRpcSendOk();
+    EXPECT_EQ(1, servers_[i]->service_.request_count()) << "for backend #" << i;
+  }
+  // One request should have gone to each server.
+  for (size_t i = 0; i < servers_.size(); ++i) {
+    EXPECT_EQ(1, servers_[i]->service_.request_count());
+  }
+  const auto pre_death = servers_[0]->service_.request_count();
+  // Kill the first server.
+  servers_[0]->Shutdown(true);
+  // Client request still succeed. May need retrying if RR had returned a pick
+  // before noticing the change in the server's connectivity.
+  while (!SendRpc())
+    ;  // Retry until success.
+  // Send a bunch of RPCs that should succeed.
+  for (int i = 0; i < 10 * kNumServers; ++i) CheckRpcSendOk();
+  const auto post_death = servers_[0]->service_.request_count();
+  // No requests have gone to the deceased server.
+  EXPECT_EQ(pre_death, post_death);
+  // Bring the first server back up.
+  servers_[0].reset(new ServerData(server_host_, ports[0]));
+  // Requests should start arriving at the first server either right away (if
+  // the server managed to start before the RR policy retried the subchannel) or
+  // after the subchannel retry delay otherwise (RR's subchannel retried before
+  // the server was fully back up).
+  WaitForServer(0);
+}
+
 }  // namespace
 }  // namespace testing
 }  // namespace grpc

From 5dd32268be62114e8a7c81d60c0dc2633fb83081 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Tue, 14 Nov 2017 19:04:02 -0800
Subject: [PATCH 037/161] Switch C++ sync server to use gpr_thd rather than
 std::thread and provide resource exhaustion mechanism

---
 .../grpc++/impl/codegen/completion_queue.h    |   6 +-
 .../grpc++/impl/codegen/method_handler_impl.h |  12 +-
 include/grpc++/impl/codegen/server_context.h  |   6 +-
 include/grpc++/server.h                       |  18 +-
 include/grpc++/server_builder.h               |  19 +++
 src/cpp/client/secure_credentials.cc          |  14 +-
 src/cpp/server/create_default_thread_pool.cc  |   2 +-
 src/cpp/server/dynamic_thread_pool.cc         |  54 ++++--
 src/cpp/server/dynamic_thread_pool.h          |  20 ++-
 src/cpp/server/secure_server_credentials.cc   |   7 +-
 src/cpp/server/server_builder.cc              |   7 +-
 src/cpp/server/server_cc.cc                   |  46 +++--
 src/cpp/server/thread_pool_interface.h        |   4 +-
 src/cpp/thread_manager/thread_manager.cc      |  54 ++++--
 src/cpp/thread_manager/thread_manager.h       |  28 +++-
 test/cpp/end2end/thread_stress_test.cc        | 157 +++++++++++-------
 test/cpp/thread_manager/BUILD                 |  31 ++++
 .../cpp/thread_manager/thread_manager_test.cc |   8 +-
 18 files changed, 361 insertions(+), 132 deletions(-)
 create mode 100644 test/cpp/thread_manager/BUILD

diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h
index b8a78625782..452eac6646b 100644
--- a/include/grpc++/impl/codegen/completion_queue.h
+++ b/include/grpc++/impl/codegen/completion_queue.h
@@ -78,7 +78,8 @@ template <class ServiceType, class RequestType, class ResponseType>
 class ServerStreamingHandler;
 template <class ServiceType, class RequestType, class ResponseType>
 class BidiStreamingHandler;
-class UnknownMethodHandler;
+template <StatusCode code>
+class ErrorMethodHandler;
 template <class Streamer, bool WriteNeeded>
 class TemplatedBidiStreamingHandler;
 template <class InputMessage, class OutputMessage>
@@ -221,7 +222,8 @@ class CompletionQueue : private GrpcLibraryCodegen {
   friend class ::grpc::internal::ServerStreamingHandler;
   template <class Streamer, bool WriteNeeded>
   friend class ::grpc::internal::TemplatedBidiStreamingHandler;
-  friend class ::grpc::internal::UnknownMethodHandler;
+  template <StatusCode code>
+  friend class ::grpc::internal::ErrorMethodHandler;
   friend class ::grpc::Server;
   friend class ::grpc::ServerContext;
   friend class ::grpc::ServerInterface;
diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h
index c0af4ca1307..d98ab7938cf 100644
--- a/include/grpc++/impl/codegen/method_handler_impl.h
+++ b/include/grpc++/impl/codegen/method_handler_impl.h
@@ -242,12 +242,14 @@ class SplitServerStreamingHandler
             ServerSplitStreamer<RequestType, ResponseType>, false>(func) {}
 };
 
-/// Handle unknown method by returning UNIMPLEMENTED error.
-class UnknownMethodHandler : public MethodHandler {
+/// General method handler class for errors that prevent real method use
+/// e.g., handle unknown method by returning UNIMPLEMENTED error.
+template <StatusCode code>
+class ErrorMethodHandler : public MethodHandler {
  public:
   template <class T>
   static void FillOps(ServerContext* context, T* ops) {
-    Status status(StatusCode::UNIMPLEMENTED, "");
+    Status status(code, "");
     if (!context->sent_initial_metadata_) {
       ops->SendInitialMetadata(context->initial_metadata_,
                                context->initial_metadata_flags());
@@ -267,6 +269,10 @@ class UnknownMethodHandler : public MethodHandler {
   }
 };
 
+typedef ErrorMethodHandler<StatusCode::UNIMPLEMENTED> UnknownMethodHandler;
+typedef ErrorMethodHandler<StatusCode::RESOURCE_EXHAUSTED>
+    ResourceExhaustedHandler;
+
 }  // namespace internal
 }  // namespace grpc
 
diff --git a/include/grpc++/impl/codegen/server_context.h b/include/grpc++/impl/codegen/server_context.h
index a2d6967bf84..9f20335a2a2 100644
--- a/include/grpc++/impl/codegen/server_context.h
+++ b/include/grpc++/impl/codegen/server_context.h
@@ -63,7 +63,8 @@ template <class ServiceType, class RequestType, class ResponseType>
 class ServerStreamingHandler;
 template <class ServiceType, class RequestType, class ResponseType>
 class BidiStreamingHandler;
-class UnknownMethodHandler;
+template <StatusCode code>
+class ErrorMethodHandler;
 template <class Streamer, bool WriteNeeded>
 class TemplatedBidiStreamingHandler;
 class Call;
@@ -255,7 +256,8 @@ class ServerContext {
   friend class ::grpc::internal::ServerStreamingHandler;
   template <class Streamer, bool WriteNeeded>
   friend class ::grpc::internal::TemplatedBidiStreamingHandler;
-  friend class ::grpc::internal::UnknownMethodHandler;
+  template <StatusCode code>
+  friend class ::grpc::internal::ErrorMethodHandler;
   friend class ::grpc::ClientContext;
 
   /// Prevent copying.
diff --git a/include/grpc++/server.h b/include/grpc++/server.h
index 01c4a60d216..456603e4e7b 100644
--- a/include/grpc++/server.h
+++ b/include/grpc++/server.h
@@ -35,6 +35,7 @@
 #include <grpc++/support/config.h>
 #include <grpc++/support/status.h>
 #include <grpc/compression.h>
+#include <grpc/support/thd.h>
 
 struct grpc_server;
 
@@ -138,10 +139,17 @@ class Server final : public ServerInterface, private GrpcLibraryCodegen {
   ///
   /// \param sync_cq_timeout_msec The timeout to use when calling AsyncNext() on
   /// server completion queues passed via sync_server_cqs param.
+  ///
+  /// \param thread_creator The thread creation function for the sync
+  /// server. Typically gpr_thd_new
   Server(int max_message_size, ChannelArguments* args,
          std::shared_ptr<std::vector<std::unique_ptr<ServerCompletionQueue>>>
              sync_server_cqs,
-         int min_pollers, int max_pollers, int sync_cq_timeout_msec);
+         int min_pollers, int max_pollers, int sync_cq_timeout_msec,
+         std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
+                           const gpr_thd_options*)>
+             thread_creator,
+         std::function<void(gpr_thd_id)> thread_joiner);
 
   /// Register a service. This call does not take ownership of the service.
   /// The service must exist for the lifetime of the Server instance.
@@ -220,6 +228,14 @@ class Server final : public ServerInterface, private GrpcLibraryCodegen {
 
   std::unique_ptr<HealthCheckServiceInterface> health_check_service_;
   bool health_check_service_disabled_;
+
+  std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
+                    const gpr_thd_options*)>
+      thread_creator_;
+  std::function<void(gpr_thd_id)> thread_joiner_;
+
+  // A special handler for resource exhausted in sync case
+  std::unique_ptr<internal::MethodHandler> resource_exhausted_handler_;
 };
 
 }  // namespace grpc
diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h
index e2bae4b41fa..25bbacbbc7a 100644
--- a/include/grpc++/server_builder.h
+++ b/include/grpc++/server_builder.h
@@ -20,6 +20,7 @@
 #define GRPCXX_SERVER_BUILDER_H
 
 #include <climits>
+#include <functional>
 #include <map>
 #include <memory>
 #include <vector>
@@ -30,6 +31,7 @@
 #include <grpc++/support/config.h>
 #include <grpc/compression.h>
 #include <grpc/support/cpu.h>
+#include <grpc/support/thd.h>
 #include <grpc/support/useful.h>
 #include <grpc/support/workaround_list.h>
 
@@ -47,6 +49,7 @@ class Service;
 
 namespace testing {
 class ServerBuilderPluginTest;
+class ServerBuilderThreadCreatorOverrideTest;
 }  // namespace testing
 
 /// A builder class for the creation and startup of \a grpc::Server instances.
@@ -213,6 +216,17 @@ class ServerBuilder {
 
  private:
   friend class ::grpc::testing::ServerBuilderPluginTest;
+  friend class ::grpc::testing::ServerBuilderThreadCreatorOverrideTest;
+
+  ServerBuilder& SetThreadFunctions(
+      std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
+                        const gpr_thd_options*)>
+          thread_creator,
+      std::function<void(gpr_thd_id)> thread_joiner) {
+    thread_creator_ = thread_creator;
+    thread_joiner_ = thread_joiner;
+    return *this;
+  }
 
   struct Port {
     grpc::string addr;
@@ -272,6 +286,11 @@ class ServerBuilder {
     grpc_compression_algorithm algorithm;
   } maybe_default_compression_algorithm_;
   uint32_t enabled_compression_algorithms_bitset_;
+
+  std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
+                    const gpr_thd_options*)>
+      thread_creator_;
+  std::function<void(gpr_thd_id)> thread_joiner_;
 };
 
 }  // namespace grpc
diff --git a/src/cpp/client/secure_credentials.cc b/src/cpp/client/secure_credentials.cc
index 4fb128d98b1..94519d817bf 100644
--- a/src/cpp/client/secure_credentials.cc
+++ b/src/cpp/client/secure_credentials.cc
@@ -189,10 +189,16 @@ int MetadataCredentialsPluginWrapper::GetMetadata(
   }
   if (w->plugin_->IsBlocking()) {
     // Asynchronous return.
-    w->thread_pool_->Add(
-        std::bind(&MetadataCredentialsPluginWrapper::InvokePlugin, w, context,
-                  cb, user_data, nullptr, nullptr, nullptr, nullptr));
-    return 0;
+    if (w->thread_pool_->Add(std::bind(
+            &MetadataCredentialsPluginWrapper::InvokePlugin, w, context, cb,
+            user_data, nullptr, nullptr, nullptr, nullptr))) {
+      return 0;
+    } else {
+      *num_creds_md = 0;
+      *status = GRPC_STATUS_RESOURCE_EXHAUSTED;
+      *error_details = nullptr;
+      return true;
+    }
   } else {
     // Synchronous return.
     w->InvokePlugin(context, cb, user_data, creds_md, num_creds_md, status,
diff --git a/src/cpp/server/create_default_thread_pool.cc b/src/cpp/server/create_default_thread_pool.cc
index 8ca3e32c2fb..2d2abbe9d1c 100644
--- a/src/cpp/server/create_default_thread_pool.cc
+++ b/src/cpp/server/create_default_thread_pool.cc
@@ -28,7 +28,7 @@ namespace {
 ThreadPoolInterface* CreateDefaultThreadPoolImpl() {
   int cores = gpr_cpu_num_cores();
   if (!cores) cores = 4;
-  return new DynamicThreadPool(cores);
+  return new DynamicThreadPool(cores, gpr_thd_new, gpr_thd_join);
 }
 
 CreateThreadPoolFunc g_ctp_impl = CreateDefaultThreadPoolImpl;
diff --git a/src/cpp/server/dynamic_thread_pool.cc b/src/cpp/server/dynamic_thread_pool.cc
index 81c78fe739b..d0e62313f68 100644
--- a/src/cpp/server/dynamic_thread_pool.cc
+++ b/src/cpp/server/dynamic_thread_pool.cc
@@ -19,19 +19,32 @@
 #include "src/cpp/server/dynamic_thread_pool.h"
 
 #include <mutex>
-#include <thread>
 
 #include <grpc/support/log.h>
+#include <grpc/support/thd.h>
 
 namespace grpc {
 
-DynamicThreadPool::DynamicThread::DynamicThread(DynamicThreadPool* pool)
-    : pool_(pool),
-      thd_(new std::thread(&DynamicThreadPool::DynamicThread::ThreadFunc,
-                           this)) {}
+DynamicThreadPool::DynamicThread::DynamicThread(DynamicThreadPool* pool,
+                                                bool* valid)
+    : pool_(pool) {
+  gpr_thd_options opt = gpr_thd_options_default();
+  gpr_thd_options_set_joinable(&opt);
+
+  std::lock_guard<std::mutex> l(dt_mu_);
+  valid_ = *valid = pool->thread_creator_(
+      &thd_, "dynamic thread",
+      [](void* th) {
+        reinterpret_cast<DynamicThreadPool::DynamicThread*>(th)->ThreadFunc();
+      },
+      this, &opt);
+}
+
 DynamicThreadPool::DynamicThread::~DynamicThread() {
-  thd_->join();
-  thd_.reset();
+  std::lock_guard<std::mutex> l(dt_mu_);
+  if (valid_) {
+    pool_->thread_joiner_(thd_);
+  }
 }
 
 void DynamicThreadPool::DynamicThread::ThreadFunc() {
@@ -73,15 +86,26 @@ void DynamicThreadPool::ThreadFunc() {
   }
 }
 
-DynamicThreadPool::DynamicThreadPool(int reserve_threads)
+DynamicThreadPool::DynamicThreadPool(
+    int reserve_threads,
+    std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
+                      const gpr_thd_options*)>
+        thread_creator,
+    std::function<void(gpr_thd_id)> thread_joiner)
     : shutdown_(false),
       reserve_threads_(reserve_threads),
       nthreads_(0),
-      threads_waiting_(0) {
+      threads_waiting_(0),
+      thread_creator_(thread_creator),
+      thread_joiner_(thread_joiner) {
   for (int i = 0; i < reserve_threads_; i++) {
     std::lock_guard<std::mutex> lock(mu_);
     nthreads_++;
-    new DynamicThread(this);
+    bool valid;
+    auto* th = new DynamicThread(this, &valid);
+    if (!valid) {
+      delete th;
+    }
   }
 }
 
@@ -101,7 +125,7 @@ DynamicThreadPool::~DynamicThreadPool() {
   ReapThreads(&dead_threads_);
 }
 
-void DynamicThreadPool::Add(const std::function<void()>& callback) {
+bool DynamicThreadPool::Add(const std::function<void()>& callback) {
   std::lock_guard<std::mutex> lock(mu_);
   // Add works to the callbacks list
   callbacks_.push(callback);
@@ -109,7 +133,12 @@ void DynamicThreadPool::Add(const std::function<void()>& callback) {
   if (threads_waiting_ == 0) {
     // Kick off a new thread
     nthreads_++;
-    new DynamicThread(this);
+    bool valid;
+    auto* th = new DynamicThread(this, &valid);
+    if (!valid) {
+      delete th;
+      return false;
+    }
   } else {
     cv_.notify_one();
   }
@@ -117,6 +146,7 @@ void DynamicThreadPool::Add(const std::function<void()>& callback) {
   if (!dead_threads_.empty()) {
     ReapThreads(&dead_threads_);
   }
+  return true;
 }
 
 }  // namespace grpc
diff --git a/src/cpp/server/dynamic_thread_pool.h b/src/cpp/server/dynamic_thread_pool.h
index 9237c6e5cac..75d31cd9083 100644
--- a/src/cpp/server/dynamic_thread_pool.h
+++ b/src/cpp/server/dynamic_thread_pool.h
@@ -24,9 +24,9 @@
 #include <memory>
 #include <mutex>
 #include <queue>
-#include <thread>
 
 #include <grpc++/support/config.h>
+#include <grpc/support/thd.h>
 
 #include "src/cpp/server/thread_pool_interface.h"
 
@@ -34,20 +34,26 @@ namespace grpc {
 
 class DynamicThreadPool final : public ThreadPoolInterface {
  public:
-  explicit DynamicThreadPool(int reserve_threads);
+  DynamicThreadPool(int reserve_threads,
+                    std::function<int(gpr_thd_id*, const char*, void (*)(void*),
+                                      void*, const gpr_thd_options*)>
+                        thread_creator,
+                    std::function<void(gpr_thd_id)> thread_joiner);
   ~DynamicThreadPool();
 
-  void Add(const std::function<void()>& callback) override;
+  bool Add(const std::function<void()>& callback) override;
 
  private:
   class DynamicThread {
    public:
-    DynamicThread(DynamicThreadPool* pool);
+    DynamicThread(DynamicThreadPool* pool, bool* valid);
     ~DynamicThread();
 
    private:
     DynamicThreadPool* pool_;
-    std::unique_ptr<std::thread> thd_;
+    std::mutex dt_mu_;
+    gpr_thd_id thd_;
+    bool valid_;
     void ThreadFunc();
   };
   std::mutex mu_;
@@ -59,6 +65,10 @@ class DynamicThreadPool final : public ThreadPoolInterface {
   int nthreads_;
   int threads_waiting_;
   std::list<DynamicThread*> dead_threads_;
+  std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
+                    const gpr_thd_options*)>
+      thread_creator_;
+  std::function<void(gpr_thd_id)> thread_joiner_;
 
   void ThreadFunc();
   static void ReapThreads(std::list<DynamicThread*>* tlist);
diff --git a/src/cpp/server/secure_server_credentials.cc b/src/cpp/server/secure_server_credentials.cc
index 0fbe4ccd18b..fa08a6200f1 100644
--- a/src/cpp/server/secure_server_credentials.cc
+++ b/src/cpp/server/secure_server_credentials.cc
@@ -43,9 +43,14 @@ void AuthMetadataProcessorAyncWrapper::Process(
     return;
   }
   if (w->processor_->IsBlocking()) {
-    w->thread_pool_->Add(
+    bool added = w->thread_pool_->Add(
         std::bind(&AuthMetadataProcessorAyncWrapper::InvokeProcessor, w,
                   context, md, num_md, cb, user_data));
+    if (!added) {
+      // no thread available, so fail with temporary resource unavailability
+      cb(user_data, nullptr, 0, nullptr, 0, GRPC_STATUS_UNAVAILABLE, nullptr);
+      return;
+    }
   } else {
     // invoke directly.
     w->InvokeProcessor(context, md, num_md, cb, user_data);
diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc
index 200e477822c..d91ee7f4e3a 100644
--- a/src/cpp/server/server_builder.cc
+++ b/src/cpp/server/server_builder.cc
@@ -23,6 +23,7 @@
 #include <grpc++/server.h>
 #include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
+#include <grpc/support/thd.h>
 #include <grpc/support/useful.h>
 
 #include "src/cpp/server/thread_pool_interface.h"
@@ -43,7 +44,9 @@ ServerBuilder::ServerBuilder()
       max_send_message_size_(-1),
       sync_server_settings_(SyncServerSettings()),
       resource_quota_(nullptr),
-      generic_service_(nullptr) {
+      generic_service_(nullptr),
+      thread_creator_(gpr_thd_new),
+      thread_joiner_(gpr_thd_join) {
   gpr_once_init(&once_init_plugin_list, do_plugin_list_init);
   for (auto it = g_plugin_factory_list->begin();
        it != g_plugin_factory_list->end(); it++) {
@@ -262,7 +265,7 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
   std::unique_ptr<Server> server(new Server(
       max_receive_message_size_, &args, sync_server_cqs,
       sync_server_settings_.min_pollers, sync_server_settings_.max_pollers,
-      sync_server_settings_.cq_timeout_msec));
+      sync_server_settings_.cq_timeout_msec, thread_creator_, thread_joiner_));
 
   if (has_sync_methods) {
     // This is a Sync server
diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc
index 4f8f4e06fcc..6ab76a287e5 100644
--- a/src/cpp/server/server_cc.cc
+++ b/src/cpp/server/server_cc.cc
@@ -36,6 +36,7 @@
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/thd.h>
 
 #include "src/core/ext/transport/inproc/inproc_transport.h"
 #include "src/core/lib/profiling/timers.h"
@@ -196,7 +197,8 @@ class Server::SyncRequest final : public internal::CompletionQueueTag {
           ctx_(mrd->deadline_, &mrd->request_metadata_),
           has_request_payload_(mrd->has_request_payload_),
           request_payload_(mrd->request_payload_),
-          method_(mrd->method_) {
+          method_(mrd->method_),
+          server_(server) {
       ctx_.set_call(mrd->call_);
       ctx_.cq_ = &cq_;
       GPR_ASSERT(mrd->in_flight_);
@@ -210,10 +212,13 @@ class Server::SyncRequest final : public internal::CompletionQueueTag {
       }
     }
 
-    void Run(std::shared_ptr<GlobalCallbacks> global_callbacks) {
+    void Run(std::shared_ptr<GlobalCallbacks> global_callbacks,
+             bool resources) {
       ctx_.BeginCompletionOp(&call_);
       global_callbacks->PreSynchronousRequest(&ctx_);
-      method_->handler()->RunHandler(internal::MethodHandler::HandlerParameter(
+      auto* handler = resources ? method_->handler()
+                                : server_->resource_exhausted_handler_.get();
+      handler->RunHandler(internal::MethodHandler::HandlerParameter(
           &call_, &ctx_, request_payload_));
       global_callbacks->PostSynchronousRequest(&ctx_);
       request_payload_ = nullptr;
@@ -235,6 +240,7 @@ class Server::SyncRequest final : public internal::CompletionQueueTag {
     const bool has_request_payload_;
     grpc_byte_buffer* request_payload_;
     internal::RpcServiceMethod* const method_;
+    Server* server_;
   };
 
  private:
@@ -255,11 +261,15 @@ class Server::SyncRequest final : public internal::CompletionQueueTag {
 // appropriate RPC handlers
 class Server::SyncRequestThreadManager : public ThreadManager {
  public:
-  SyncRequestThreadManager(Server* server, CompletionQueue* server_cq,
-                           std::shared_ptr<GlobalCallbacks> global_callbacks,
-                           int min_pollers, int max_pollers,
-                           int cq_timeout_msec)
-      : ThreadManager(min_pollers, max_pollers),
+  SyncRequestThreadManager(
+      Server* server, CompletionQueue* server_cq,
+      std::shared_ptr<GlobalCallbacks> global_callbacks, int min_pollers,
+      int max_pollers, int cq_timeout_msec,
+      std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
+                        const gpr_thd_options*)>
+          thread_creator,
+      std::function<void(gpr_thd_id)> thread_joiner)
+      : ThreadManager(min_pollers, max_pollers, thread_creator, thread_joiner),
         server_(server),
         server_cq_(server_cq),
         cq_timeout_msec_(cq_timeout_msec),
@@ -285,7 +295,7 @@ class Server::SyncRequestThreadManager : public ThreadManager {
     GPR_UNREACHABLE_CODE(return TIMEOUT);
   }
 
-  void DoWork(void* tag, bool ok) override {
+  void DoWork(void* tag, bool ok, bool resources) override {
     SyncRequest* sync_req = static_cast<SyncRequest*>(tag);
 
     if (!sync_req) {
@@ -305,7 +315,7 @@ class Server::SyncRequestThreadManager : public ThreadManager {
       }
 
       GPR_TIMER_SCOPE("cd.Run()", 0);
-      cd.Run(global_callbacks_);
+      cd.Run(global_callbacks_, resources);
     }
     // TODO (sreek) If ok is false here (which it isn't in case of
     // grpc_request_registered_call), we should still re-queue the request
@@ -367,7 +377,11 @@ Server::Server(
     int max_receive_message_size, ChannelArguments* args,
     std::shared_ptr<std::vector<std::unique_ptr<ServerCompletionQueue>>>
         sync_server_cqs,
-    int min_pollers, int max_pollers, int sync_cq_timeout_msec)
+    int min_pollers, int max_pollers, int sync_cq_timeout_msec,
+    std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
+                      const gpr_thd_options*)>
+        thread_creator,
+    std::function<void(gpr_thd_id)> thread_joiner)
     : max_receive_message_size_(max_receive_message_size),
       sync_server_cqs_(sync_server_cqs),
       started_(false),
@@ -376,7 +390,9 @@ Server::Server(
       has_generic_service_(false),
       server_(nullptr),
       server_initializer_(new ServerInitializer(this)),
-      health_check_service_disabled_(false) {
+      health_check_service_disabled_(false),
+      thread_creator_(thread_creator),
+      thread_joiner_(thread_joiner) {
   g_gli_initializer.summon();
   gpr_once_init(&g_once_init_callbacks, InitGlobalCallbacks);
   global_callbacks_ = g_callbacks;
@@ -386,7 +402,7 @@ Server::Server(
        it++) {
     sync_req_mgrs_.emplace_back(new SyncRequestThreadManager(
         this, (*it).get(), global_callbacks_, min_pollers, max_pollers,
-        sync_cq_timeout_msec));
+        sync_cq_timeout_msec, thread_creator_, thread_joiner_));
   }
 
   grpc_channel_args channel_args;
@@ -549,6 +565,10 @@ void Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) {
     }
   }
 
+  if (!sync_server_cqs_->empty()) {
+    resource_exhausted_handler_.reset(new internal::ResourceExhaustedHandler);
+  }
+
   for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) {
     (*it)->Start();
   }
diff --git a/src/cpp/server/thread_pool_interface.h b/src/cpp/server/thread_pool_interface.h
index 028842a776f..656e6673f12 100644
--- a/src/cpp/server/thread_pool_interface.h
+++ b/src/cpp/server/thread_pool_interface.h
@@ -29,7 +29,9 @@ class ThreadPoolInterface {
   virtual ~ThreadPoolInterface() {}
 
   // Schedule the given callback for execution.
-  virtual void Add(const std::function<void()>& callback) = 0;
+  // Return true on success, false on failure
+  virtual bool Add(const std::function<void()>& callback)
+      GRPC_MUST_USE_RESULT = 0;
 };
 
 // Allows different codebases to use their own thread pool impls
diff --git a/src/cpp/thread_manager/thread_manager.cc b/src/cpp/thread_manager/thread_manager.cc
index 23264f1b5bc..107c60f4ebe 100644
--- a/src/cpp/thread_manager/thread_manager.cc
+++ b/src/cpp/thread_manager/thread_manager.cc
@@ -20,18 +20,26 @@
 
 #include <climits>
 #include <mutex>
-#include <thread>
 
 #include <grpc/support/log.h>
+#include <grpc/support/thd.h>
 
 namespace grpc {
 
-ThreadManager::WorkerThread::WorkerThread(ThreadManager* thd_mgr)
+ThreadManager::WorkerThread::WorkerThread(ThreadManager* thd_mgr, bool* valid)
     : thd_mgr_(thd_mgr) {
+  gpr_thd_options opt = gpr_thd_options_default();
+  gpr_thd_options_set_joinable(&opt);
+
   // Make thread creation exclusive with respect to its join happening in
   // ~WorkerThread().
   std::lock_guard<std::mutex> lock(wt_mu_);
-  thd_ = std::thread(&ThreadManager::WorkerThread::Run, this);
+  *valid = valid_ = thd_mgr->thread_creator_(
+      &thd_, "worker thread",
+      [](void* th) {
+        reinterpret_cast<ThreadManager::WorkerThread*>(th)->Run();
+      },
+      this, &opt);
 }
 
 void ThreadManager::WorkerThread::Run() {
@@ -42,15 +50,24 @@ void ThreadManager::WorkerThread::Run() {
 ThreadManager::WorkerThread::~WorkerThread() {
   // Don't join until the thread is fully constructed.
   std::lock_guard<std::mutex> lock(wt_mu_);
-  thd_.join();
+  if (valid_) {
+    thd_mgr_->thread_joiner_(thd_);
+  }
 }
 
-ThreadManager::ThreadManager(int min_pollers, int max_pollers)
+ThreadManager::ThreadManager(
+    int min_pollers, int max_pollers,
+    std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
+                      const gpr_thd_options*)>
+        thread_creator,
+    std::function<void(gpr_thd_id)> thread_joiner)
     : shutdown_(false),
       num_pollers_(0),
       min_pollers_(min_pollers),
       max_pollers_(max_pollers == -1 ? INT_MAX : max_pollers),
-      num_threads_(0) {}
+      num_threads_(0),
+      thread_creator_(thread_creator),
+      thread_joiner_(thread_joiner) {}
 
 ThreadManager::~ThreadManager() {
   {
@@ -111,7 +128,9 @@ void ThreadManager::Initialize() {
 
   for (int i = 0; i < min_pollers_; i++) {
     // Create a new thread (which ends up calling the MainWorkLoop() function
-    new WorkerThread(this);
+    bool valid;
+    new WorkerThread(this, &valid);
+    GPR_ASSERT(valid);  // we need to have at least this minimum
   }
 }
 
@@ -138,18 +157,27 @@ void ThreadManager::MainWorkLoop() {
       case WORK_FOUND:
         // If we got work and there are now insufficient pollers, start a new
         // one
+        bool resources;
         if (!shutdown_ && num_pollers_ < min_pollers_) {
-          num_pollers_++;
-          num_threads_++;
+          bool valid;
           // Drop lock before spawning thread to avoid contention
           lock.unlock();
-          new WorkerThread(this);
+          auto* th = new WorkerThread(this, &valid);
+          lock.lock();
+          if (valid) {
+            num_pollers_++;
+            num_threads_++;
+          } else {
+            delete th;
+          }
+          resources = (num_pollers_ > 0);
         } else {
-          // Drop lock for consistency with above branch
-          lock.unlock();
+          resources = true;
         }
+        // Drop lock before any application work
+        lock.unlock();
         // Lock is always released at this point - do the application work
-        DoWork(tag, ok);
+        DoWork(tag, ok, resources);
         // Take the lock again to check post conditions
         lock.lock();
         // If we're shutdown, we should finish at this point.
diff --git a/src/cpp/thread_manager/thread_manager.h b/src/cpp/thread_manager/thread_manager.h
index a206e0bd8a5..4fa8a6c563b 100644
--- a/src/cpp/thread_manager/thread_manager.h
+++ b/src/cpp/thread_manager/thread_manager.h
@@ -23,15 +23,19 @@
 #include <list>
 #include <memory>
 #include <mutex>
-#include <thread>
 
 #include <grpc++/support/config.h>
+#include <grpc/support/thd.h>
 
 namespace grpc {
 
 class ThreadManager {
  public:
-  explicit ThreadManager(int min_pollers, int max_pollers);
+  ThreadManager(int min_pollers, int max_pollers,
+                std::function<int(gpr_thd_id*, const char*, void (*)(void*),
+                                  void*, const gpr_thd_options*)>
+                    thread_creator,
+                std::function<void(gpr_thd_id)> thread_joiner);
   virtual ~ThreadManager();
 
   // Initializes and Starts the Rpc Manager threads
@@ -50,6 +54,8 @@ class ThreadManager {
   //  - ThreadManager does not interpret the values of 'tag' and 'ok'
   //  - ThreadManager WILL call DoWork() and pass '*tag' and 'ok' as input to
   //    DoWork()
+  //  - ThreadManager will also pass DoWork a bool saying if there are actually
+  //    resources to do the work
   //
   // If the return value is SHUTDOWN:,
   //  - ThreadManager WILL NOT call DoWork() and terminates the thead
@@ -69,7 +75,7 @@ class ThreadManager {
   // The implementation of DoWork() should also do any setup needed to ensure
   // that the next call to PollForWork() (not necessarily by the current thread)
   // actually finds some work
-  virtual void DoWork(void* tag, bool ok) = 0;
+  virtual void DoWork(void* tag, bool ok, bool resources) = 0;
 
   // Mark the ThreadManager as shutdown and begin draining the work. This is a
   // non-blocking call and the caller should call Wait(), a blocking call which
@@ -84,15 +90,15 @@ class ThreadManager {
   virtual void Wait();
 
  private:
-  // Helper wrapper class around std::thread. This takes a ThreadManager object
-  // and starts a new std::thread to calls the Run() function.
+  // Helper wrapper class around thread. This takes a ThreadManager object
+  // and starts a new thread to calls the Run() function.
   //
   // The Run() function calls ThreadManager::MainWorkLoop() function and once
   // that completes, it marks the WorkerThread completed by calling
   // ThreadManager::MarkAsCompleted()
   class WorkerThread {
    public:
-    WorkerThread(ThreadManager* thd_mgr);
+    WorkerThread(ThreadManager* thd_mgr, bool* valid);
     ~WorkerThread();
 
    private:
@@ -102,7 +108,8 @@ class ThreadManager {
 
     ThreadManager* const thd_mgr_;
     std::mutex wt_mu_;
-    std::thread thd_;
+    gpr_thd_id thd_;
+    bool valid_;
   };
 
   // The main funtion in ThreadManager
@@ -129,6 +136,13 @@ class ThreadManager {
   // currently polling i.e num_pollers_)
   int num_threads_;
 
+  // Functions for creating/joining threads. Normally, these should
+  // be gpr_thd_new/gpr_thd_join but they are overridable
+  std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
+                    const gpr_thd_options*)>
+      thread_creator_;
+  std::function<void(gpr_thd_id)> thread_joiner_;
+
   std::mutex list_mu_;
   std::list<WorkerThread*> completed_threads_;
 };
diff --git a/test/cpp/end2end/thread_stress_test.cc b/test/cpp/end2end/thread_stress_test.cc
index 90b2eddbbb4..fd43c8f584c 100644
--- a/test/cpp/end2end/thread_stress_test.cc
+++ b/test/cpp/end2end/thread_stress_test.cc
@@ -26,6 +26,7 @@
 #include <grpc++/server_builder.h>
 #include <grpc++/server_context.h>
 #include <grpc/grpc.h>
+#include <grpc/support/atm.h>
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
 
@@ -52,63 +53,13 @@ namespace testing {
 
 class TestServiceImpl : public ::grpc::testing::EchoTestService::Service {
  public:
-  TestServiceImpl() : signal_client_(false) {}
+  TestServiceImpl() {}
 
   Status Echo(ServerContext* context, const EchoRequest* request,
               EchoResponse* response) override {
     response->set_message(request->message());
     return Status::OK;
   }
-
-  // Unimplemented is left unimplemented to test the returned error.
-
-  Status RequestStream(ServerContext* context,
-                       ServerReader<EchoRequest>* reader,
-                       EchoResponse* response) override {
-    EchoRequest request;
-    response->set_message("");
-    while (reader->Read(&request)) {
-      response->mutable_message()->append(request.message());
-    }
-    return Status::OK;
-  }
-
-  // Return 3 messages.
-  // TODO(yangg) make it generic by adding a parameter into EchoRequest
-  Status ResponseStream(ServerContext* context, const EchoRequest* request,
-                        ServerWriter<EchoResponse>* writer) override {
-    EchoResponse response;
-    response.set_message(request->message() + "0");
-    writer->Write(response);
-    response.set_message(request->message() + "1");
-    writer->Write(response);
-    response.set_message(request->message() + "2");
-    writer->Write(response);
-
-    return Status::OK;
-  }
-
-  Status BidiStream(
-      ServerContext* context,
-      ServerReaderWriter<EchoResponse, EchoRequest>* stream) override {
-    EchoRequest request;
-    EchoResponse response;
-    while (stream->Read(&request)) {
-      gpr_log(GPR_INFO, "recv msg %s", request.message().c_str());
-      response.set_message(request.message());
-      stream->Write(response);
-    }
-    return Status::OK;
-  }
-
-  bool signal_client() {
-    std::unique_lock<std::mutex> lock(mu_);
-    return signal_client_;
-  }
-
- private:
-  bool signal_client_;
-  std::mutex mu_;
 };
 
 template <class Service>
@@ -119,10 +70,15 @@ class CommonStressTest {
   virtual void SetUp() = 0;
   virtual void TearDown() = 0;
   virtual void ResetStub() = 0;
+  virtual bool AllowExhaustion() = 0;
   grpc::testing::EchoTestService::Stub* GetStub() { return stub_.get(); }
 
  protected:
   std::unique_ptr<grpc::testing::EchoTestService::Stub> stub_;
+  // Some tests use a custom thread creator. This should be declared before the
+  // server so that it's destructor happens after the server
+  std::unique_ptr<ServerBuilderThreadCreatorOverrideTest> creator_;
+
   std::unique_ptr<Server> server_;
 
   virtual void SetUpStart(ServerBuilder* builder, Service* service) = 0;
@@ -147,6 +103,7 @@ class CommonStressTestInsecure : public CommonStressTest<Service> {
         CreateChannel(server_address_.str(), InsecureChannelCredentials());
     this->stub_ = grpc::testing::EchoTestService::NewStub(channel);
   }
+  bool AllowExhaustion() override { return false; }
 
  protected:
   void SetUpStart(ServerBuilder* builder, Service* service) override {
@@ -162,7 +119,7 @@ class CommonStressTestInsecure : public CommonStressTest<Service> {
   std::ostringstream server_address_;
 };
 
-template <class Service>
+template <class Service, bool allow_resource_exhaustion>
 class CommonStressTestInproc : public CommonStressTest<Service> {
  public:
   void ResetStub() override {
@@ -170,6 +127,7 @@ class CommonStressTestInproc : public CommonStressTest<Service> {
     std::shared_ptr<Channel> channel = this->server_->InProcessChannel(args);
     this->stub_ = grpc::testing::EchoTestService::NewStub(channel);
   }
+  bool AllowExhaustion() override { return allow_resource_exhaustion; }
 
  protected:
   void SetUpStart(ServerBuilder* builder, Service* service) override {
@@ -194,6 +152,67 @@ class CommonStressTestSyncServer : public BaseClass {
   TestServiceImpl service_;
 };
 
+class ServerBuilderThreadCreatorOverrideTest {
+ public:
+  ServerBuilderThreadCreatorOverrideTest(ServerBuilder* builder, size_t limit)
+      : limit_(limit), threads_(0) {
+    builder->SetThreadFunctions(
+        [this](gpr_thd_id* id, const char* name, void (*f)(void*), void* arg,
+               const gpr_thd_options* options) -> int {
+          std::unique_lock<std::mutex> l(mu_);
+          if (threads_ < limit_) {
+            l.unlock();
+            if (gpr_thd_new(id, name, f, arg, options) != 0) {
+              l.lock();
+              threads_++;
+              return 1;
+            }
+          }
+          return 0;
+        },
+        [this](gpr_thd_id id) {
+          gpr_thd_join(id);
+          std::unique_lock<std::mutex> l(mu_);
+          threads_--;
+          if (threads_ == 0) {
+            done_.notify_one();
+          }
+        });
+  }
+  ~ServerBuilderThreadCreatorOverrideTest() {
+    // Don't allow destruction until all threads are really done and uncounted
+    std::unique_lock<std::mutex> l(mu_);
+    done_.wait(l, [this] { return (threads_ == 0); });
+  }
+
+ private:
+  size_t limit_;
+  size_t threads_;
+  std::mutex mu_;
+  std::condition_variable done_;
+};
+
+template <class BaseClass>
+class CommonStressTestSyncServerLowThreadCount : public BaseClass {
+ public:
+  void SetUp() override {
+    ServerBuilder builder;
+    this->SetUpStart(&builder, &service_);
+    builder.SetSyncServerOption(ServerBuilder::SyncServerOption::MIN_POLLERS,
+                                1);
+    this->creator_.reset(
+        new ServerBuilderThreadCreatorOverrideTest(&builder, 4));
+    this->SetUpEnd(&builder);
+  }
+  void TearDown() override {
+    this->TearDownStart();
+    this->TearDownEnd();
+  }
+
+ private:
+  TestServiceImpl service_;
+};
+
 template <class BaseClass>
 class CommonStressTestAsyncServer : public BaseClass {
  public:
@@ -294,7 +313,8 @@ class End2endTest : public ::testing::Test {
   Common common_;
 };
 
-static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs) {
+static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs,
+                    bool allow_exhaustion, gpr_atm* errors) {
   EchoRequest request;
   EchoResponse response;
   request.set_message("Hello");
@@ -302,33 +322,48 @@ static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs) {
   for (int i = 0; i < num_rpcs; ++i) {
     ClientContext context;
     Status s = stub->Echo(&context, request, &response);
-    EXPECT_EQ(response.message(), request.message());
+    EXPECT_TRUE(s.ok() || (allow_exhaustion &&
+                           s.error_code() == StatusCode::RESOURCE_EXHAUSTED));
     if (!s.ok()) {
-      gpr_log(GPR_ERROR, "RPC error: %d: %s", s.error_code(),
-              s.error_message().c_str());
+      if (!(allow_exhaustion &&
+            s.error_code() == StatusCode::RESOURCE_EXHAUSTED)) {
+        gpr_log(GPR_ERROR, "RPC error: %d: %s", s.error_code(),
+                s.error_message().c_str());
+      }
+      gpr_atm_no_barrier_fetch_add(errors, static_cast<gpr_atm>(1));
+    } else {
+      EXPECT_EQ(response.message(), request.message());
     }
-    ASSERT_TRUE(s.ok());
   }
 }
 
 typedef ::testing::Types<
     CommonStressTestSyncServer<CommonStressTestInsecure<TestServiceImpl>>,
-    CommonStressTestSyncServer<CommonStressTestInproc<TestServiceImpl>>,
+    CommonStressTestSyncServer<CommonStressTestInproc<TestServiceImpl, false>>,
+    CommonStressTestSyncServerLowThreadCount<
+        CommonStressTestInproc<TestServiceImpl, true>>,
     CommonStressTestAsyncServer<
         CommonStressTestInsecure<grpc::testing::EchoTestService::AsyncService>>,
-    CommonStressTestAsyncServer<
-        CommonStressTestInproc<grpc::testing::EchoTestService::AsyncService>>>
+    CommonStressTestAsyncServer<CommonStressTestInproc<
+        grpc::testing::EchoTestService::AsyncService, false>>>
     CommonTypes;
 TYPED_TEST_CASE(End2endTest, CommonTypes);
 TYPED_TEST(End2endTest, ThreadStress) {
   this->common_.ResetStub();
   std::vector<std::thread> threads;
+  gpr_atm errors;
+  gpr_atm_rel_store(&errors, static_cast<gpr_atm>(0));
   for (int i = 0; i < kNumThreads; ++i) {
-    threads.emplace_back(SendRpc, this->common_.GetStub(), kNumRpcs);
+    threads.emplace_back(SendRpc, this->common_.GetStub(), kNumRpcs,
+                         this->common_.AllowExhaustion(), &errors);
   }
   for (int i = 0; i < kNumThreads; ++i) {
     threads[i].join();
   }
+  uint64_t error_cnt = static_cast<uint64_t>(gpr_atm_no_barrier_load(&errors));
+  if (error_cnt != 0) {
+    gpr_log(GPR_INFO, "RPC error count: %" PRIu64, error_cnt);
+  }
 }
 
 template <class Common>
diff --git a/test/cpp/thread_manager/BUILD b/test/cpp/thread_manager/BUILD
new file mode 100644
index 00000000000..1f0878770b5
--- /dev/null
+++ b/test/cpp/thread_manager/BUILD
@@ -0,0 +1,31 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache v2
+
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_package")
+
+grpc_package(name = "test/cpp/thread_manager")
+
+grpc_cc_test(
+    name = "thread_manager_test",
+    srcs = ["thread_manager_test.cc"],
+    deps = [
+        "//:gpr",
+        "//:grpc",
+        "//:grpc++",
+        "//test/cpp/util:test_config",
+    ],
+)
+
diff --git a/test/cpp/thread_manager/thread_manager_test.cc b/test/cpp/thread_manager/thread_manager_test.cc
index 8282d466948..d3d31f9dd97 100644
--- a/test/cpp/thread_manager/thread_manager_test.cc
+++ b/test/cpp/thread_manager/thread_manager_test.cc
@@ -20,10 +20,10 @@
 #include <memory>
 #include <string>
 
-#include <gflags/gflags.h>
 #include <grpc++/grpc++.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
+#include <grpc/support/thd.h>
 
 #include "src/cpp/thread_manager/thread_manager.h"
 #include "test/cpp/util/test_config.h"
@@ -32,13 +32,13 @@ namespace grpc {
 class ThreadManagerTest final : public grpc::ThreadManager {
  public:
   ThreadManagerTest()
-      : ThreadManager(kMinPollers, kMaxPollers),
+      : ThreadManager(kMinPollers, kMaxPollers, gpr_thd_new, gpr_thd_join),
         num_do_work_(0),
         num_poll_for_work_(0),
         num_work_found_(0) {}
 
   grpc::ThreadManager::WorkStatus PollForWork(void** tag, bool* ok) override;
-  void DoWork(void* tag, bool ok) override;
+  void DoWork(void* tag, bool ok, bool resources) override;
   void PerformTest();
 
  private:
@@ -89,7 +89,7 @@ grpc::ThreadManager::WorkStatus ThreadManagerTest::PollForWork(void** tag,
   }
 }
 
-void ThreadManagerTest::DoWork(void* tag, bool ok) {
+void ThreadManagerTest::DoWork(void* tag, bool ok, bool resources) {
   gpr_atm_no_barrier_fetch_add(&num_do_work_, 1);
   SleepForMs(kDoWorkDurationMsec);  // Simulate doing work by sleeping
 }

From 8924344a54ab80861842a379d256e5c6b8a30cae Mon Sep 17 00:00:00 2001
From: Ken Payson <kpayson@google.com>
Date: Mon, 8 Jan 2018 10:39:00 -0800
Subject: [PATCH 038/161] Fix stall bug with pollhup workaround

---
 src/core/lib/iomgr/ev_poll_posix.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/core/lib/iomgr/ev_poll_posix.cc b/src/core/lib/iomgr/ev_poll_posix.cc
index 13468b30c0c..ad85b126dd4 100644
--- a/src/core/lib/iomgr/ev_poll_posix.cc
+++ b/src/core/lib/iomgr/ev_poll_posix.cc
@@ -473,7 +473,7 @@ static grpc_error* fd_shutdown_error(grpc_fd* fd) {
 
 static void notify_on_locked(grpc_exec_ctx* exec_ctx, grpc_fd* fd,
                              grpc_closure** st, grpc_closure* closure) {
-  if (fd->shutdown) {
+  if (fd->shutdown || gpr_atm_no_barrier_load(&fd->pollhup)) {
     GRPC_CLOSURE_SCHED(exec_ctx, closure,
                        GRPC_ERROR_CREATE_FROM_STATIC_STRING("FD shutdown"));
   } else if (*st == CLOSURE_NOT_READY) {

From 8065a5e73c54801ed0f6c2b6e5469c4e34a83f48 Mon Sep 17 00:00:00 2001
From: Alexander Polcyn <apolcyn@google.com>
Date: Wed, 3 Jan 2018 14:47:26 -0800
Subject: [PATCH 039/161] Use rake-compiler-dock's Dockerfile as a base in
 grpc's fork of it

---
 Rakefile                                      |   4 +-
 third_party/rake-compiler-dock/Dockerfile     | 200 +-----------------
 third_party/rake-compiler-dock/build.sh       |   7 -
 .../compat-with-bundler.diff                  | 105 ---------
 .../rake-compiler-0.9.5/without-exts.diff     |  14 --
 .../build/patches/ruby-1.8.7-p374/nop.patch   |   2 -
 .../patches/ruby-1.9.3/no_sendfile.patch      |  13 --
 .../build/patches/ruby-1.9.3/nop.patch        |   2 -
 .../patches/ruby-2.3.0/no_sendfile.patch      |  12 --
 .../patches/ruby-2.4.0/no_sendfile.patch      |  12 --
 third_party/rake-compiler-dock/build/runas    |  12 --
 third_party/rake-compiler-dock/build/sigfw.c  |  43 ----
 .../rake-compiler-dock/build/strip_wrapper    |  30 ---
 third_party/rake-compiler-dock/build/sudoers  |   1 -
 14 files changed, 3 insertions(+), 454 deletions(-)
 delete mode 100755 third_party/rake-compiler-dock/build.sh
 delete mode 100644 third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/compat-with-bundler.diff
 delete mode 100644 third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/without-exts.diff
 delete mode 100644 third_party/rake-compiler-dock/build/patches/ruby-1.8.7-p374/nop.patch
 delete mode 100644 third_party/rake-compiler-dock/build/patches/ruby-1.9.3/no_sendfile.patch
 delete mode 100644 third_party/rake-compiler-dock/build/patches/ruby-1.9.3/nop.patch
 delete mode 100644 third_party/rake-compiler-dock/build/patches/ruby-2.3.0/no_sendfile.patch
 delete mode 100644 third_party/rake-compiler-dock/build/patches/ruby-2.4.0/no_sendfile.patch
 delete mode 100755 third_party/rake-compiler-dock/build/runas
 delete mode 100644 third_party/rake-compiler-dock/build/sigfw.c
 delete mode 100755 third_party/rake-compiler-dock/build/strip_wrapper
 delete mode 100644 third_party/rake-compiler-dock/build/sudoers

diff --git a/Rakefile b/Rakefile
index d76b9ff657c..74c8b1fd487 100755
--- a/Rakefile
+++ b/Rakefile
@@ -113,10 +113,10 @@ task 'gem:native' do
   if RUBY_PLATFORM =~ /darwin/
     FileUtils.touch 'grpc_c.32.ruby'
     FileUtils.touch 'grpc_c.64.ruby'
-    system "rake cross native gem RUBY_CC_VERSION=2.4.0:2.3.0:2.2.2:2.1.5:2.0.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}"
+    system "rake cross native gem RUBY_CC_VERSION=2.5.0:2.4.0:2.3.0:2.2.2:2.1.6:2.0.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}"
   else
     Rake::Task['dlls'].execute
-    docker_for_windows "gem update --system && bundle && rake cross native gem RUBY_CC_VERSION=2.4.0:2.3.0:2.2.2:2.1.5:2.0.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}"
+    docker_for_windows "gem update --system && bundle && rake cross native gem RUBY_CC_VERSION=2.5.0:2.4.0:2.3.0:2.2.2:2.1.6:2.0.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}"
   end
 end
 
diff --git a/third_party/rake-compiler-dock/Dockerfile b/third_party/rake-compiler-dock/Dockerfile
index b4a5158535f..06c721c39ba 100644
--- a/third_party/rake-compiler-dock/Dockerfile
+++ b/third_party/rake-compiler-dock/Dockerfile
@@ -1,182 +1,4 @@
-FROM ubuntu:17.04
-
-RUN apt-get -y update && \
-    apt-get install -y curl git-core xz-utils build-essential wget unzip sudo gpg dirmngr
-
-# Add "rvm" as system group, to avoid conflicts with host GIDs typically starting with 1000
-RUN groupadd -r rvm && useradd -r -g rvm -G sudo -p "" --create-home rvm && \
-    echo "source /etc/profile.d/rvm.sh" >> /etc/rubybashrc
-
-USER root
-RUN apt-get -y update && \
-    apt-get install -y gcc-mingw-w64-x86-64 gcc-mingw-w64-i686 g++-mingw-w64-x86-64 g++-mingw-w64-i686 \
-        gcc-multilib moreutils
-USER rvm
-
-# install rvm, RVM 1.26.0+ has signed releases, source rvm for usage outside of package scripts
-RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 && \
-    (curl -L http://get.rvm.io | sudo bash -s stable) && \
-    bash -c " \
-        source /etc/rubybashrc && \
-        rvmsudo rvm cleanup all "
-
-# Regenerate the following using build.sh if the build folder changes.
-RUN echo \\
-H4sIANslcFgAA+08a3fbxo75uvwVqOxTSbFIidTDVhKnTW0n8d28ju2ku6fO5R2RQ4kVRXJnSMve\
-OPe3LzAk9fIrvk3UuylxUnk0DwADYDADzKiD1A/c5oNvCi2E7W7rgdlpdbe3Tfxumj2zpepzeGC2\
-W+1Oy7TaZvcBNpvYHbrflq0MUpkwAfCAxVHgXIQ39uPhcB3srBsGSv8ydSMu5DeiQQrudW7Wv9Xq\
-rejf6nZ7D6D1jfhZgr+4/ve5x9IgkZADD8/sMecxbO1CZZQksR2L6PwCqCjzsjerPdrbt1++PT6x\
-j97/8t/2u1fPTp6/PXq9Uv3h4Oj48O0bVXv4+tmLA1DVe3tFS0X7s6Xw14Vs/Ys0ZN9q9d/t/zsd\
-c3X9W1a3XP/rgI0fmgM/bA6YHGnaUERpzFwX9CFUNl8c7lfoz9Hb9+8q2mTs+gL0GJrJJG6OognX\
-UsnFcm89xdJ7VXoB4mzSoI2FBlWwZjAfCvqEOh4fHFU07eXb1we7mzViAZ48eQIV7owi+GfWXNeC\
-EHQJzVSKZhA5LGgKNua6E01iP+AC0dD4StNYqta0jHJakAFdlzzRFfFTjTzdL8+OX9oHbz7sNnni\
-4AoYXBAHwsmbdR2H/vzdu6Z8//eH3tRwvhGNO/Z/s2etnv+sTrtdrv91QPOhBg/hZORLwC19KNgE\
-Rix0Ay7h+PDF4ZsTwG/gRWLKhCvBTyCJsCpKRrj2cITDpTQIxWECiMMPEx663KVeAw64rwCT8O5w\
-H0zqRR33I2eMY0nsiSxQILmpn4yg4matOLCyNBLehvDKD9PzBiBpcLNjC6DhhizIWRbEZzYEsD4S\
-iJWFF3mnjM0T5JtjNw57iQj0PWBjVsyU+FfDXIgRWU6DhjU1bcMPnSB1OTzJ0Bmjp4t1F7KZXMRc\
-Xq2eMj+hWg1lA7Hvwi60HmvaWeS7Wsa3jRipsUYfhD2d1LVP6IV8r4YD6sofjf0goG+NnNv6Y+1z\
-hnPC/FANZWLoNMAZoT0/xPLZbx/rhEUmInUUYuYkfhRCyKd2VnxMRIhowpJUIme6iawBNB/CMU8g\
-jZWwMwQpygy1KmPu+N6FakBEkCEySEawgNmQzC60sgtL8ySiWOKTOLlAnwy1H5eHTZgc1x9fweYF\
-bChz4cHCbGqZPBqwgKYBNZLvw3qrrnpnYke1j2sKcy7YT0qypKDaj5kIVCuA4DjdEH49+K/Dk+OT\
-Zyfvj2sL7Z95IHk2diY4fs6ds7imxG5+RGbyUo4Q7UlEolahbpVlIhkKhRc1uu71n/n/mCXOiMtv\
-lAf48vjfsnpdk/x/r9Uu4/91wLL+6RSkm8aOsa3H7e3OV7KH++d/OttWp9T/OuB2/YdRbKi2P0Tj\
-rvyP2VqN/3rdtlWe/9YBru95GOsM8WDHmk4Uev6QNvrBvIxnDJefQ7drdreZZxi9XrvFBwxMWtTd\
-7z0++t5hZf0vxtB6y+gb3a+wB9zf//eoqvT/a4C79U9fWaJTeKYPUnWiN8hrfDmN2/2/2eq2V/L/\
-ltUt83/rgecimkDH9HbanU6n7zmdttUeWF6/3+U722aXse5Ox3L7nttvtzvwGuOtYx6DuQ2t1iP1\
-DyxUoUZoHsErJiT8JwsxkH4SYPnnoeD+/+r4EUo3Ep7h8qfaPkv4IzhmSQPaLfgbCxGD2YPWziP6\
-tw1baBEt7Tgd/M6d5BH89u7Zyd7Lj7CHMeSQq7DTQyuFCcdAFylN8lRDZqb+IOBZJqEwVU3DkB9i\
-wc/8CCO1lbFTDAkx3B9cwCiKxn44hChEfEhFqwj+P6mPe2FVDLLNsFoBhwWBoal0SZYxQXoTJOsg\
-Cw5LJS/oqkyICLH7BeSIJBSIGtgrATfiMqwmGvGA7AuMB7GzHwJTXLp0F+OLKJzwMDG0w4S4CrHJ\
-DyTNOYiYCxhF45cRo+CfAwtkBDETCUReHrlj4M2EC4E/EExcNLAw5trvEvVIeR3hRk4+HeRP3QMh\
-fV/KFLmlrtm9z6NmEw8Io3RgoJSXncSKy6D0TTMb39zJRL8icSQVRlPKUqA6VZ6IJD8TO8zFPhmL\
-QS7yhjYd+c6IBicXse8osS5OlJ8nJFrwkIuGmpqv8lFTSvfMjMaFOGCJF4mJdsYCmqOSmpOkCmHI\
-uctdkgfKc8ouchwkaBzKPFRood6mxOA9boCMMukThVzGyFcyUpg1ym4wB1XrkmXlE3zBJ8SlEaCo\
-GkVliDZ0xrMMV47I5xLtROWqtJnVL5gEsUbZjjOONv4u4AyNL4wS3phzpJw7uChNJ4mQMxQWgyGO\
-CMlw1HxlsXwQgeBuA097gPziTLF+qHJALi5YNyNPjEsDjiM0XkRVEbj6JjHE4yHowqsYmq7rGvGv\
-zKKJWuGh9FG3TI4NMYBLaHdhawn060ADU9EqzKQBZgctU3JB2R1Z26o3wDLB5QHPKvS6pi0dZW/m\
-YXBzW37UbbVbzOv3DMPrm52+2aejbq/TocndhlnD2dyK/eefQTd7/cYObGV/sOJv7IzBrCfIUZSi\
-ygbKY3kcfQLaY3E3Cxtq6dCVy6LNS0f4cTLv5UxcePIE2EDaeRdD8EAZmE22aXvorGvUjLpTNXUN\
-NH1OpHBTJK2pHwQ5O1IZnQdTrpaNIyIpIVv5aN5zBNglMbC/WsjST9Bi8oToT7XKxqeC6OfmAp1K\
-XXk3bWvOBRnc/TjYujcHOY2COswhipVhGXEqR7WHipKdcWvnTTXlS+r1+SgeuihI0nK71220+7Cl\
-/rbuqecNyjz7Ic9nxlSWk2xIqvx2NuvCkf1QjMrG2LkntlX/Gvawi54NoOjePuNiQd0btML8sBB2\
-Y0n35EYn44lHZfQGr7GPWpRDHnKh2CpEvoLmS0bmbKvqJZ0UnSuw+xR+u1Zdjbmx3Q43mltDu3vw\
-VQz5lCoflwQYxVk2XE2F1pb6ptSkk8iBri3VnPWb5rzImpp2UWFT34/kbS+TywVpFzSLfrfSJXXM\
-x5IOcOfkM2WRUZHvxOWCy0uNRCMhM5XFqKscF6q4hduZDJ/TdhfFPKwlRsgmuENVp9W66uZdLqrS\
-M6YCV+vCGSAnUz0NT8PKfIHPu1IHWwxutvSrQxQ7gjO3tuokPF/IZHVBLxTnK1QpgMwhE/x1yi6o\
-XhFdYUVKdPRlWWwzeTjxFS+mGMQNXolxyeMWM4YnUDGNfgV+/DEvPNmFxWc4i/K+RjFiK7lWNchO\
-RHd7Cd2jKOnNtXG7freuQ3GnDq6OMmQ6+KHW/HttZhz1wu5q1fpmE6menppkD6enVnUJw73kszjn\
-K7Sz2gFyPob66cNafkcoZ/SRdjaFav1uTPkN0B9FpM6Tp0Y9Fhh01KDI3CkzPK1J4dAe45+f1mfI\
-4zSRmZSWVEyKneIZaLU2Wzc54dXGRKQhmjyvYZEHwRK3fBHXwhdVnLvQe7mkG53ozCWtLqR7+5+7\
-nMo1fqgQztLUF4vqXNDpd+n019npNcz7HgtIUBWHhXjGz2I/mWJEpMSA51JUW9tq4hEpCs4qFPaE\
-efSSrxYMARnFL1G4hPCYZ1Hjo+sCveR36YxSDOhWYjx+pg+i8yLM687xzSNmxUd1LqirsfTcm6JJ\
-BwyPNG9VxL8oXMEn0RmdaUKZwKOjwZ4aumhgKz0WXyHe0e2a5X5Nr/2D472jw3cn9H4RvUh2yHJ/\
-qq024kLSQbOMrtEyaAVFU2mY2v+D5Pjd+T/KpkRpoqOVynsm/nK46/5n27JW8n9m22qV+b91wFLQ\
-rIIG9R5wfqBR7+owrL2xLQ+ae8xxdkwMmnuMWayzEjTfPDoLmm9uV0FzW7lN+rNNXrPw/fSyz6bn\
-f5+byo5lc+PT68M3L35Vr48/47eVl8bXRBeEI3MCFV1XWHbzYb+8P3y1/7kIFaq6zkM2CLguR5Qp\
-qc7rXV+qBh/9BgsCSoJV8xAFW4v1k4yvq3SC3EvmlbTMdquE+iM9XEEX/TwSDodfEXnkjC21NR7R\
-hmgaOw386BfvlFQmx0/u6XOuvf/tG+2v+RTkX7j/t6xeef+zDrhR/1/n6l/Bv3D/j19K/78OKO//\
-/9pwy/q3JQYu6qriD/qBu95/W2ZnZf1vd8v33+uBL17/O/2d7XbLMQzH4z2r2y7Wf3a+m/fOznPz\
-7+r81un16ei2VRToDIfnGObYXho6dOfoTSJXvXGOh+pBbkd90rtderULMmbT8AzkhaS7P7DtouSM\
-RIQB8ZBr8B9ZFCd5InEY5SLoFyuSc/XLFc8ZEQknkiOQfjiChOFHEA0tEFEauqeLCFJEgH+G+Mdl\
-GBViZM4DDE7tgIkhtz2XbvbwP448pWFWOF1NKU/GiT/hQB/DCQwnVLAFBrt4jrOHPFGt+d/Ic9kF\
-oAkGENPn6WqGO6ZkGRQrEuQoTTDIDNU7aDz7JcwZgxvYfkL5dW7HI1csRtILKL5kJOVGHtF8mLRR\
-SWdM7P5jE8vqlzkVrHHOlObszVyFFbgE6pAIW47+oSnNhjb2G3GcrFJyZfPV4ZuDN2/pN0WzUZXN\
-GYXv/qc2/5Zwjf+3jLbR+rPP/2Z5/l8L3Kj/9e3/rU6vfXX/L/M/a4Ev3v/5gLfdHdz/WbvdG2zf\
-b//f6Xfaav9XhR7t/7Md5tadojDBytKuM6vV1FU/lwlUzjev6wC7cH7B5ePs7ZQi6LAEnj4lBl3u\
-SYN+c2o/2zt4+1zTN/KL95fPPhzYxwdv9p8fvjoAU4O8A2jg+d/XNnXt+u/8Wf7fsjC4zPI/5fvf\
-tcCN+v9T/L9lYQfl/ztl/mctUPr/0v+7TZkIP7angsUxF1+dxh3+3zTN7ZXzX3sb3UC5/tcAGz+o\
-/7MG3bxRMoM2AE3bgGMyiOyeLQrVm6YAvChw1cN2mVAiQT0m/uALejw9iM4huxbLeyGKJFKv64Gp\
-9AouQF/CIB3OXxpMp1PjbDbeiMSwmfjOmCfNHdwRNG32ZiDhk5hYqc6r6Gua+IGsapoyXlzolY1P\
-6qXHgOHyZxNe22zVPxs4s4qmqYfL2OfZ0YsPhuD00ODTpXcJ3u4/m3/Xm/DZmLBYvQhhYkhP1+hZ\
-8y6c5JQN+qV7VRGqZo2GE0SSa9njkvfEieHE9Av8hmqlFynY+BuWG1T7UaNHIFr+dLRgJEsqLTMC\
-W/kra2To02XSiC7pFRiig8+alobYIikNhjKpgWKoAQ8LtPT2hp+jKzetbUXQiVyOxDZ/Mqg6+6V5\
-LgyDM2eUPZUhHiPhq3nPpyMmqnJ1isXssiHZtBRNovV9+cYSSiihhBJKKKGEEkoooYQSSiihhBJK\
-KKGEEkoooYQSSiihhBJKKKGEEkoooYQSSiihhH9H+D9i0BbqAHgAAA==\
-| base64 -d | tar xzC /tmp
-
-# Import patch files for ruby and gems
-RUN cp -r /tmp/build/patches /home/rvm/patches/
-ENV BASH_ENV /etc/rubybashrc
-
-# install rubies and fix permissions on
-RUN bash -c " \
-    export CFLAGS='-s -O3 -fno-fast-math -fPIC' && \
-    echo 'about to install patches for ruby 2.4.0 from:' && \
-    ls -r ~/patches && \
-    for v in 2.4.0 ; do \
-	rvm install \$v --patch \$(echo ~/patches/ruby-\$v/* | tr ' ' ','); \
-    done && \
-    rvm cleanup all && \
-    find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
-
-# Install rake-compiler and typical gems in all Rubies
-# do not generate documentation for gems
-RUN echo "gem: --no-ri --no-rdoc" >> ~/.gemrc && \
-    bash -c " \
-        rvm all do gem install bundler rake-compiler hoe mini_portile rubygems-tasks && \
-        rvm 2.4.0 do gem install mini_portile2 && \
-        find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
-
-RUN bash -c "gem env"
-RUN bash -c "gem list rake-compiler"
-
-# Install rake-compiler's cross rubies in global dir instead of /root
-RUN sudo mkdir -p /usr/local/rake-compiler && \
-    sudo chown rvm.rvm /usr/local/rake-compiler && \
-    ln -s /usr/local/rake-compiler ~/.rake-compiler
-
-# Patch rake-compiler to avoid build of ruby extensions
-RUN cd /usr/local/rvm/gems/ruby-2.4.0/gems/rake-compiler-0.9.5 && git apply /home/rvm/patches/rake-compiler-0.9.5/*.diff ; \
-    true
-
-RUN bash -c "rvm use 2.4.0 --default && \
-    export MAKE=\"make -j`nproc`\" CFLAGS='-s -O1 -fno-omit-frame-pointer -fno-fast-math' && \
-    rake-compiler cross-ruby VERSION=2.4.0 HOST=i686-w64-mingw32 && \
-    rake-compiler cross-ruby VERSION=2.4.0 HOST=x86_64-w64-mingw32 && \
-    rake-compiler cross-ruby VERSION=2.4.0 HOST=x86_64-linux-gnu && \
-    rake-compiler cross-ruby VERSION=2.3.0 HOST=i686-w64-mingw32 && \
-    rake-compiler cross-ruby VERSION=2.3.0 HOST=x86_64-w64-mingw32 && \
-    rake-compiler cross-ruby VERSION=2.3.0 HOST=x86_64-linux-gnu && \
-    rake-compiler cross-ruby VERSION=2.2.2 HOST=i686-w64-mingw32 && \
-    rake-compiler cross-ruby VERSION=2.2.2 HOST=x86_64-w64-mingw32 && \
-    rake-compiler cross-ruby VERSION=2.2.2 HOST=x86_64-linux-gnu && \
-    rake-compiler cross-ruby VERSION=2.1.5 HOST=i686-w64-mingw32 && \
-    rake-compiler cross-ruby VERSION=2.1.5 HOST=x86_64-w64-mingw32 && \
-    rake-compiler cross-ruby VERSION=2.1.5 HOST=x86_64-linux-gnu && \
-    rake-compiler cross-ruby VERSION=2.0.0-p645 HOST=i686-w64-mingw32 && \
-    rake-compiler cross-ruby VERSION=2.0.0-p645 HOST=x86_64-w64-mingw32 && \
-    rake-compiler cross-ruby VERSION=2.0.0-p645 HOST=x86_64-linux-gnu && \
-    rm -rf ~/.rake-compiler/tmp/builds ~/.rake-compiler/sources && \
-    find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
-
-RUN bash -c "rvm use 2.4.0 --default && \
-    export MAKE=\"make -j`nproc`\" CFLAGS='-m32 -s -O1 -fno-omit-frame-pointer -fno-fast-math' LDFLAGS='-m32' && \
-    rake-compiler cross-ruby VERSION=2.4.0 HOST=i686-linux-gnu && \
-    rake-compiler cross-ruby VERSION=2.3.0 HOST=i686-linux-gnu && \
-    rake-compiler cross-ruby VERSION=2.2.2 HOST=i686-linux-gnu && \
-    rake-compiler cross-ruby VERSION=2.1.5 HOST=i686-linux-gnu && \
-    rake-compiler cross-ruby VERSION=2.0.0-p645 HOST=i686-linux-gnu && \
-    rm -rf ~/.rake-compiler/tmp/builds ~/.rake-compiler/sources && \
-    find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
-
-RUN bash -c " \
-    rvm alias create 2.4 2.4.0 "
-
-USER root
-
-# Fix paths in rake-compiler/config.yml and add rvm and mingw-tools to the global bashrc
-RUN sed -i -- "s:/root/.rake-compiler:/usr/local/rake-compiler:g" /usr/local/rake-compiler/config.yml && \
-    echo "source /etc/profile.d/rvm.sh" >> /etc/bash.bashrc && \
-    echo "export PATH=\$PATH:/opt/mingw/mingw32/bin" >> /etc/bash.bashrc && \
-    echo "export PATH=\$PATH:/opt/mingw/mingw64/bin" >> /etc/bash.bashrc
-
-# Install wrappers for strip commands as a workaround for "Protocol error" in boot2docker.
-RUN cp /tmp/build/strip_wrapper /root/
-RUN sudo chmod +rx /root/strip_wrapper
-RUN mv /usr/bin/i686-w64-mingw32-strip /usr/bin/i686-w64-mingw32-strip.bin && \
-    mv /usr/bin/x86_64-w64-mingw32-strip /usr/bin/x86_64-w64-mingw32-strip.bin && \
-    ln /root/strip_wrapper /usr/bin/i686-w64-mingw32-strip && \
-    ln /root/strip_wrapper /usr/bin/x86_64-w64-mingw32-strip
+FROM larskanis/rake-compiler-dock:0.6.2
 
 RUN find / -name rbconfig.rb | while read f ; do sed -i 's/0x0501/0x0600/' $f ; done
 RUN find / -name win32.h | while read f ; do sed -i 's/gettimeofday/rb_gettimeofday/' $f ; done
@@ -184,26 +6,6 @@ RUN sed -i 's/defined.__MINGW64__.$/1/' /usr/local/rake-compiler/ruby/i686-w64-m
 RUN find / -name libwinpthread.dll.a | xargs rm
 RUN find / -name libwinpthread-1.dll | xargs rm
 RUN find / -name *msvcrt-ruby*.dll.a | while read f ; do n=`echo $f | sed s/.dll//` ; mv $f $n ; done
-RUN find /usr/local/rake-compiler/ruby -name libruby.so | xargs rm
-RUN find /usr/local/rake-compiler/ruby -name libruby-static.a | while read f ; do ar t $f | xargs ar d $f ; done
-RUN find /usr/local/rake-compiler/ruby -name libruby-static.a | while read f ; do mv $f `echo $f | sed s/-static//` ; done
-
-# Install SIGINT forwarder
-RUN cp /tmp/build/sigfw.c /root/
-RUN gcc $HOME/sigfw.c -o /usr/local/bin/sigfw
-
-# Install user mapper
-RUN cp /tmp/build/runas /usr/local/bin/
-
-# Install sudoers configuration
-RUN cp /tmp/build/sudoers /etc/sudoers.d/rake-compiler-dock
-
-# Fixup Ruby 2.4 'static' compilation issue.
-RUN echo '!<arch>' > /usr/local/rake-compiler/ruby/x86_64-linux-gnu/ruby-2.4.0/lib/libruby.a
-RUN echo '!<arch>' > /usr/local/rake-compiler/ruby/i686-linux-gnu/ruby-2.4.0/lib/libruby.a
-
-ENV RUBY_CC_VERSION 2.4.0:2.3.0:2.2.2:2.1.5:2.0.0
-
 RUN apt-get install -y g++-multilib
 
 CMD bash
diff --git a/third_party/rake-compiler-dock/build.sh b/third_party/rake-compiler-dock/build.sh
deleted file mode 100755
index ca01fa6d4db..00000000000
--- a/third_party/rake-compiler-dock/build.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-
-# Run this to produce the snipplet of data to insert in the Dockerfile.
-
-echo 'RUN echo \\'
-tar cz build | base64 | sed 's/$/\\/'
-echo '| base64 -d | tar xzC /tmp'
diff --git a/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/compat-with-bundler.diff b/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/compat-with-bundler.diff
deleted file mode 100644
index ea22bd928ec..00000000000
--- a/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/compat-with-bundler.diff
+++ /dev/null
@@ -1,105 +0,0 @@
-From 41f834449fc4323b2f995e8715aa5842d9fd9334 Mon Sep 17 00:00:00 2001
-From: Lars Kanis <lars@greiz-reinsdorf.de>
-Date: Sat, 30 Jan 2016 08:08:07 +0100
-Subject: [PATCH] Change the fake mechanism to be compatible with bundler.
-
-The previous fake mechanism worked by hooking onto the
-"require 'rbconfig'" call.
-This is problematic because bundler internally requires rbconfig, but doesn't
-work corretly in a faked environment.
-It then fails to load gems that are also part of the standard library, like
-json and rdoc.
-This results in issues like https://github.com/rake-compiler/rake-compiler-dock/issues/8
-
-The fake mechanism is now changed to hook onto the "require 'mkrb'" call,
-which is typically part of the extconf file, and it is where the faked platform
-values are actually needed.
-That way it is loaded after bundler/setup, so that the library paths are
-set according to the Gemfile.lock, to the native Linux libraries, before
-the fake environment is active.
-
-Please note, that the build directory of a given gem needs to be cleared,
-in order to get updated fake files. So do a "rm tmp pkg -rf".
----
- lib/rake/extensiontask.rb | 35 ++++++++++++++---------------------
- 1 file changed, 14 insertions(+), 21 deletions(-)
-
-diff --git a/lib/rake/extensiontask.rb b/lib/rake/extensiontask.rb
-index 030af96..f914919 100644
---- a/lib/rake/extensiontask.rb
-+++ b/lib/rake/extensiontask.rb
-@@ -169,8 +169,8 @@ Java extension should be preferred.
-         # now add the extconf script
-         cmd << abs_extconf.relative_path_from(abs_tmp_path)
- 
--        # rbconfig.rb will be present if we are cross compiling
--        if t.prerequisites.include?("#{tmp_path}/rbconfig.rb") then
-+        # fake.rb will be present if we are cross compiling
-+        if t.prerequisites.include?("#{tmp_path}/fake.rb") then
-           options.push(*cross_config_options(platf))
-         end
- 
-@@ -365,39 +365,30 @@ Java extension should be preferred.
-       # define compilation tasks for cross platform!
-       define_compile_tasks(for_platform, ruby_ver)
- 
--      # chain fake.rb, rbconfig.rb and mkmf.rb to Makefile generation
-+      # chain fake.rb and mkmf.rb to Makefile generation
-       file "#{tmp_path}/Makefile" => ["#{tmp_path}/fake.rb",
--                                      "#{tmp_path}/rbconfig.rb",
-                                       "#{tmp_path}/mkmf.rb"]
- 
--      # copy the file from the cross-ruby location
--      file "#{tmp_path}/rbconfig.rb" => [rbconfig_file] do |t|
-+      # copy the rbconfig from the cross-ruby location and
-+      # genearte fake.rb for different ruby versions
-+      file "#{tmp_path}/fake.rb" => [rbconfig_file] do |t|
-         File.open(t.name, 'w') do |f|
--          f.write "require 'fake.rb'\n\n"
-+          f.write fake_rb(for_platform, ruby_ver)
-           f.write File.read(t.prerequisites.first)
-         end
-       end
- 
-       # copy mkmf from cross-ruby location
-       file "#{tmp_path}/mkmf.rb" => [mkmf_file] do |t|
--        cp t.prerequisites.first, t.name
--        if ruby_ver < "1.9" && "1.9" <= RUBY_VERSION
--          File.open(t.name, 'r+t') do |f|
--            content = f.read
-+        File.open(t.name, 'w') do |f|
-+          content = File.read(t.prerequisites.first)
-+          content.sub!(/^(require ')rbconfig(')$/, '\\1fake\\2')
-+          if ruby_ver < "1.9" && "1.9" <= RUBY_VERSION
-             content.sub!(/^(      break )\*(defaults)$/, '\\1\\2.first')
-             content.sub!(/^(    return )\*(defaults)$/, '\\1\\2.first')
-             content.sub!(/^(  mfile\.)print( configuration\(srcprefix\))$/, '\\1puts\\2')
--            f.rewind
--            f.write content
--            f.truncate(f.tell)
-           end
--        end
--      end
--
--      # genearte fake.rb for different ruby versions
--      file "#{tmp_path}/fake.rb" do |t|
--        File.open(t.name, 'w') do |f|
--          f.write fake_rb(for_platform, ruby_ver)
-+          f.write content
-         end
-       end
- 
-@@ -495,8 +486,10 @@ Java extension should be preferred.
-         # "cannot load such file -- win32/resolv" when it is required later on.
-         # See also: https://github.com/tjschuck/rake-compiler-dev-box/issues/5
-         require 'resolv'
-+        require 'rbconfig'
- 
-         class Object
-+          remove_const :RbConfig
-           remove_const :RUBY_PLATFORM
-           remove_const :RUBY_VERSION
-           remove_const :RUBY_DESCRIPTION if defined?(RUBY_DESCRIPTION)
--- 
-2.5.0.windows.1
-
diff --git a/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/without-exts.diff b/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/without-exts.diff
deleted file mode 100644
index 07739d33ecc..00000000000
--- a/third_party/rake-compiler-dock/build/patches/rake-compiler-0.9.5/without-exts.diff
+++ /dev/null
@@ -1,14 +0,0 @@
-diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake
-index 6acc816..6aa2a49 100644
---- a/tasks/bin/cross-ruby.rake
-+++ b/tasks/bin/cross-ruby.rake
-@@ -135,8 +135,7 @@ file "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/Makefile" => ["#{USER
-     "--build=#{RUBY_BUILD}",
-     '--enable-shared',
-     '--disable-install-doc',
--    '--without-tk',
--    '--without-tcl'
-+    '--with-ext='
-   ]
-
-   # Force Winsock2 for Ruby 1.8, 1.9 defaults to it
diff --git a/third_party/rake-compiler-dock/build/patches/ruby-1.8.7-p374/nop.patch b/third_party/rake-compiler-dock/build/patches/ruby-1.8.7-p374/nop.patch
deleted file mode 100644
index fac8525da60..00000000000
--- a/third_party/rake-compiler-dock/build/patches/ruby-1.8.7-p374/nop.patch
+++ /dev/null
@@ -1,2 +0,0 @@
-diff --git a/configure b/configure
-index 55157af..6630eba 100755
diff --git a/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/no_sendfile.patch b/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/no_sendfile.patch
deleted file mode 100644
index d8f339e8140..00000000000
--- a/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/no_sendfile.patch
+++ /dev/null
@@ -1,13 +0,0 @@
-diff --git a/configure b/configure
-index 898730c..cfe6253 100755
---- a/configure
-+++ b/configure
-@@ -14695,7 +14695,7 @@ for ac_func in fmod killpg wait4 waitpid fork spawnv syscall __syscall chroot ge
- 	      setsid telldir seekdir fchmod cosh sinh tanh log2 round\
- 	      setuid setgid daemon select_large_fdset setenv unsetenv\
-               mktime timegm gmtime_r clock_gettime gettimeofday poll ppoll\
--              pread sendfile shutdown sigaltstack dl_iterate_phdr
-+              pread shutdown sigaltstack dl_iterate_phdr
- do :
-   as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
- ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
diff --git a/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/nop.patch b/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/nop.patch
deleted file mode 100644
index fac8525da60..00000000000
--- a/third_party/rake-compiler-dock/build/patches/ruby-1.9.3/nop.patch
+++ /dev/null
@@ -1,2 +0,0 @@
-diff --git a/configure b/configure
-index 55157af..6630eba 100755
diff --git a/third_party/rake-compiler-dock/build/patches/ruby-2.3.0/no_sendfile.patch b/third_party/rake-compiler-dock/build/patches/ruby-2.3.0/no_sendfile.patch
deleted file mode 100644
index 915fc7b790f..00000000000
--- a/third_party/rake-compiler-dock/build/patches/ruby-2.3.0/no_sendfile.patch
+++ /dev/null
@@ -1,12 +0,0 @@
-diff --git a/configure b/configure
-index ebe3d8c..a336b73 100755
---- a/configure
-+++ b/configure
-@@ -18943,7 +18943,6 @@ do :
-   ac_fn_c_check_func "$LINENO" "sendfile" "ac_cv_func_sendfile"
- if test "x$ac_cv_func_sendfile" = xyes; then :
-   cat >>confdefs.h <<_ACEOF
--#define HAVE_SENDFILE 1
- _ACEOF
- 
- fi
diff --git a/third_party/rake-compiler-dock/build/patches/ruby-2.4.0/no_sendfile.patch b/third_party/rake-compiler-dock/build/patches/ruby-2.4.0/no_sendfile.patch
deleted file mode 100644
index 915fc7b790f..00000000000
--- a/third_party/rake-compiler-dock/build/patches/ruby-2.4.0/no_sendfile.patch
+++ /dev/null
@@ -1,12 +0,0 @@
-diff --git a/configure b/configure
-index ebe3d8c..a336b73 100755
---- a/configure
-+++ b/configure
-@@ -18943,7 +18943,6 @@ do :
-   ac_fn_c_check_func "$LINENO" "sendfile" "ac_cv_func_sendfile"
- if test "x$ac_cv_func_sendfile" = xyes; then :
-   cat >>confdefs.h <<_ACEOF
--#define HAVE_SENDFILE 1
- _ACEOF
- 
- fi
diff --git a/third_party/rake-compiler-dock/build/runas b/third_party/rake-compiler-dock/build/runas
deleted file mode 100755
index b29ce31fcce..00000000000
--- a/third_party/rake-compiler-dock/build/runas
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-groupadd -g "$GID" "$GROUP"
-mkdir -p /tmp/home
-useradd -g "$GID" -u "$UID" -G rvm,sudo -p "" -b /tmp/home -m "$USER"
-
-HOME=$(bash <<< "echo ~$USER")
-ln -s /usr/local/rake-compiler "$HOME"/.rake-compiler
-
-sudo -u "$USER" --set-home \
-    BASH_ENV=/etc/rubybashrc \
-    -- "$@"
diff --git a/third_party/rake-compiler-dock/build/sigfw.c b/third_party/rake-compiler-dock/build/sigfw.c
deleted file mode 100644
index 291d76cec8c..00000000000
--- a/third_party/rake-compiler-dock/build/sigfw.c
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * This program handles SIGINT and forwards it to another process.
- * It is intended to be run as PID 1.
- *
- * Docker starts processes with "docker run" as PID 1.
- * On Linux, the default signal handler for PID 1 ignores any signals.
- * Therefore Ctrl-C aka SIGINT is ignored per default.
- */
-
-#include <signal.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-
-int pid = 0;
-
-void
-handle_sigint (int signum)
-{
-  if(pid)
-    kill(pid, SIGINT);
-}
-
-int main(int argc, char *argv[]){
-  struct sigaction new_action;
-  int status = -1;
-
-  /* Set up the structure to specify the new action. */
-  new_action.sa_handler = handle_sigint;
-  sigemptyset (&new_action.sa_mask);
-  new_action.sa_flags = 0;
-
-  sigaction (SIGINT, &new_action, (void*)0);
-
-  pid = fork();
-  if(pid){
-    wait(&status);
-    return WEXITSTATUS(status);
-  }else{
-    status = execvp(argv[1], &argv[1]);
-    perror("exec");
-    return status;
-  }
-}
diff --git a/third_party/rake-compiler-dock/build/strip_wrapper b/third_party/rake-compiler-dock/build/strip_wrapper
deleted file mode 100755
index 7f8a1346a1d..00000000000
--- a/third_party/rake-compiler-dock/build/strip_wrapper
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env ruby
-
-# Strip file on local folder instead of a Virtualbox shared folder
-# to work around this bug: https://www.virtualbox.org/ticket/8463
-
-require 'tempfile'
-require 'fileutils'
-
-strip = "#{File.basename($0)}.bin"
-
-files = ARGV.reject{|f| f=~/^-/ }.map do |arg|
-  tmp = Tempfile.new 'strip'
-  tmp.close
-  FileUtils.cp arg, tmp.path
-  [tmp, arg]
-end
-
-options = ARGV.select{|f| f=~/^-/ } + files.map{|t,o| t.path }
-
-unless system( strip, *options )
-  exit 127
-end
-code = $?.exitstatus
-
-files.each do |tmp, orig|
-  FileUtils.rm orig
-  FileUtils.cp tmp.path, orig
-end
-
-exit code
diff --git a/third_party/rake-compiler-dock/build/sudoers b/third_party/rake-compiler-dock/build/sudoers
deleted file mode 100644
index f9f9b97c951..00000000000
--- a/third_party/rake-compiler-dock/build/sudoers
+++ /dev/null
@@ -1 +0,0 @@
-Defaults        env_keep += "http_proxy https_proxy ftp_proxy RCD_HOST_RUBY_PLATFORM RCD_HOST_RUBY_VERSION RCD_IMAGE RUBY_CC_VERSION"

From 9423c274cdfc1e9e6869cd8b64ba86aa985f902d Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Mon, 8 Jan 2018 14:22:47 -0800
Subject: [PATCH 040/161] Need to include functional to use std::bind

---
 test/core/transport/chttp2/settings_timeout_test.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/core/transport/chttp2/settings_timeout_test.cc b/test/core/transport/chttp2/settings_timeout_test.cc
index 08473c72b68..d7d6ee75082 100644
--- a/test/core/transport/chttp2/settings_timeout_test.cc
+++ b/test/core/transport/chttp2/settings_timeout_test.cc
@@ -21,6 +21,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#include <functional>
 #include <memory>
 #include <thread>
 

From 2146b3aec114649319314ef4c5b18ca3affb640e Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Thu, 4 Jan 2018 14:57:27 -0800
Subject: [PATCH 041/161] Add go release versions to client_matrix.py

---
 tools/interop_matrix/client_matrix.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py
index f32bdf8b150..0d05e6bc024 100644
--- a/tools/interop_matrix/client_matrix.py
+++ b/tools/interop_matrix/client_matrix.py
@@ -103,6 +103,9 @@ LANG_RELEASE_MATRIX = {
         {
             'v1.8.1': None
         },
+        {
+            'v1.9.1': None
+        },
     ],
     'java': [
         {

From 8fc3715a17a1b764143d0b37652a07a45d1cdf01 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Mon, 18 Dec 2017 14:33:51 -0800
Subject: [PATCH 042/161] Catch exceptions from sync method handlers without
 crashing server

---
 CMakeLists.txt                                |  41 +++++
 Makefile                                      |  50 +++++-
 build.yaml                                    |  16 +-
 .../grpc++/impl/codegen/method_handler_impl.h |  35 +++-
 test/cpp/end2end/BUILD                        | 159 ++++++++++--------
 test/cpp/end2end/exception_test.cc            | 116 +++++++++++++
 .../generated/sources_and_headers.json        |  19 +++
 tools/run_tests/generated/tests.json          |  24 +++
 8 files changed, 383 insertions(+), 77 deletions(-)
 create mode 100644 test/cpp/end2end/exception_test.cc

diff --git a/CMakeLists.txt b/CMakeLists.txt
index eed12052685..2cec087850d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -525,6 +525,7 @@ add_dependencies(buildtests_cxx cxx_string_ref_test)
 add_dependencies(buildtests_cxx cxx_time_test)
 add_dependencies(buildtests_cxx end2end_test)
 add_dependencies(buildtests_cxx error_details_test)
+add_dependencies(buildtests_cxx exception_test)
 add_dependencies(buildtests_cxx filter_end2end_test)
 add_dependencies(buildtests_cxx generic_end2end_test)
 add_dependencies(buildtests_cxx golden_file_test)
@@ -10188,6 +10189,46 @@ target_link_libraries(error_details_test
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
+add_executable(exception_test
+  test/cpp/end2end/exception_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+
+target_include_directories(exception_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${BENCHMARK_ROOT_DIR}/include
+  PRIVATE ${ZLIB_ROOT_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CARES_INCLUDE_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(exception_test
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_test_util
+  grpc_test_util
+  grpc++
+  grpc
+  gpr_test_util
+  gpr
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
 add_executable(filter_end2end_test
   test/cpp/end2end/filter_end2end_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
diff --git a/Makefile b/Makefile
index c962a128735..339c52294a6 100644
--- a/Makefile
+++ b/Makefile
@@ -77,7 +77,6 @@ CC_opt = $(DEFAULT_CC)
 CXX_opt = $(DEFAULT_CXX)
 LD_opt = $(DEFAULT_CC)
 LDXX_opt = $(DEFAULT_CXX)
-CXXFLAGS_opt = -fno-exceptions
 CPPFLAGS_opt = -O2
 DEFINES_opt = NDEBUG
 
@@ -95,7 +94,6 @@ CC_dbg = $(DEFAULT_CC)
 CXX_dbg = $(DEFAULT_CXX)
 LD_dbg = $(DEFAULT_CC)
 LDXX_dbg = $(DEFAULT_CXX)
-CXXFLAGS_dbg = -fno-exceptions
 CPPFLAGS_dbg = -O0
 DEFINES_dbg = _DEBUG DEBUG
 
@@ -1126,6 +1124,7 @@ cxx_string_ref_test: $(BINDIR)/$(CONFIG)/cxx_string_ref_test
 cxx_time_test: $(BINDIR)/$(CONFIG)/cxx_time_test
 end2end_test: $(BINDIR)/$(CONFIG)/end2end_test
 error_details_test: $(BINDIR)/$(CONFIG)/error_details_test
+exception_test: $(BINDIR)/$(CONFIG)/exception_test
 filter_end2end_test: $(BINDIR)/$(CONFIG)/filter_end2end_test
 generic_end2end_test: $(BINDIR)/$(CONFIG)/generic_end2end_test
 golden_file_test: $(BINDIR)/$(CONFIG)/golden_file_test
@@ -1573,6 +1572,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/cxx_time_test \
   $(BINDIR)/$(CONFIG)/end2end_test \
   $(BINDIR)/$(CONFIG)/error_details_test \
+  $(BINDIR)/$(CONFIG)/exception_test \
   $(BINDIR)/$(CONFIG)/filter_end2end_test \
   $(BINDIR)/$(CONFIG)/generic_end2end_test \
   $(BINDIR)/$(CONFIG)/golden_file_test \
@@ -1702,6 +1702,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/cxx_time_test \
   $(BINDIR)/$(CONFIG)/end2end_test \
   $(BINDIR)/$(CONFIG)/error_details_test \
+  $(BINDIR)/$(CONFIG)/exception_test \
   $(BINDIR)/$(CONFIG)/filter_end2end_test \
   $(BINDIR)/$(CONFIG)/generic_end2end_test \
   $(BINDIR)/$(CONFIG)/golden_file_test \
@@ -2101,6 +2102,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/end2end_test || ( echo test end2end_test failed ; exit 1 )
 	$(E) "[RUN]     Testing error_details_test"
 	$(Q) $(BINDIR)/$(CONFIG)/error_details_test || ( echo test error_details_test failed ; exit 1 )
+	$(E) "[RUN]     Testing exception_test"
+	$(Q) $(BINDIR)/$(CONFIG)/exception_test || ( echo test exception_test failed ; exit 1 )
 	$(E) "[RUN]     Testing filter_end2end_test"
 	$(Q) $(BINDIR)/$(CONFIG)/filter_end2end_test || ( echo test filter_end2end_test failed ; exit 1 )
 	$(E) "[RUN]     Testing generic_end2end_test"
@@ -14981,6 +14984,49 @@ endif
 $(OBJDIR)/$(CONFIG)/test/cpp/util/error_details_test.o: $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc
 
 
+EXCEPTION_TEST_SRC = \
+    test/cpp/end2end/exception_test.cc \
+
+EXCEPTION_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(EXCEPTION_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/exception_test: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
+
+$(BINDIR)/$(CONFIG)/exception_test: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/exception_test: $(PROTOBUF_DEP) $(EXCEPTION_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(EXCEPTION_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/exception_test
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/cpp/end2end/exception_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_exception_test: $(EXCEPTION_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(EXCEPTION_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 FILTER_END2END_TEST_SRC = \
     test/cpp/end2end/filter_end2end_test.cc \
 
diff --git a/build.yaml b/build.yaml
index 42d72459811..8ab15b77414 100644
--- a/build.yaml
+++ b/build.yaml
@@ -4007,6 +4007,20 @@ targets:
   deps:
   - grpc++_error_details
   - grpc++
+- name: exception_test
+  gtest: true
+  build: test
+  language: c++
+  src:
+  - test/cpp/end2end/exception_test.cc
+  deps:
+  - grpc++_test_util
+  - grpc_test_util
+  - grpc++
+  - grpc
+  - gpr_test_util
+  - gpr
+  uses_polling: false
 - name: filter_end2end_test
   gtest: true
   build: test
@@ -4913,7 +4927,6 @@ configs:
     DEFINES: NDEBUG
   dbg:
     CPPFLAGS: -O0
-    CXXFLAGS: -fno-exceptions
     DEFINES: _DEBUG DEBUG
   gcov:
     CC: gcc
@@ -4956,7 +4969,6 @@ configs:
     LDFLAGS: -rdynamic
   opt:
     CPPFLAGS: -O2
-    CXXFLAGS: -fno-exceptions
     DEFINES: NDEBUG
   stapprof:
     CPPFLAGS: -O2 -DGRPC_STAP_PROFILER
diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h
index c0af4ca1307..b72dceb1b4c 100644
--- a/include/grpc++/impl/codegen/method_handler_impl.h
+++ b/include/grpc++/impl/codegen/method_handler_impl.h
@@ -27,6 +27,25 @@
 namespace grpc {
 
 namespace internal {
+
+// Invoke the method handler, fill in the status, and
+// return whether or not we finished safely (without an exception).
+// Note that exception handling is 0-cost in most compiler/library
+// implementations (except when an exception is actually thrown),
+// so this process doesn't require additional overhead in the common case.
+// Additionally, we don't need to return if we caught an exception or not;
+// the handling is the same in either case.
+template <class F>
+Status CatchingFunctionHandler(F&& callable) {
+  try {
+    return callable();
+  } catch (const std::exception& e) {
+    return Status(StatusCode::UNKNOWN, e.what());
+  } catch (...) {
+    return Status(StatusCode::UNKNOWN, "Exception in method handler");
+  }
+}
+
 /// A wrapper class of an application provided rpc method handler.
 template <class ServiceType, class RequestType, class ResponseType>
 class RpcMethodHandler : public MethodHandler {
@@ -43,7 +62,9 @@ class RpcMethodHandler : public MethodHandler {
         param.request.bbuf_ptr(), &req);
     ResponseType rsp;
     if (status.ok()) {
-      status = func_(service_, param.server_context, &req, &rsp);
+      status = CatchingFunctionHandler([this, &param, &req, &rsp] {
+        return func_(service_, param.server_context, &req, &rsp);
+      });
     }
 
     GPR_CODEGEN_ASSERT(!param.server_context->sent_initial_metadata_);
@@ -86,7 +107,9 @@ class ClientStreamingHandler : public MethodHandler {
   void RunHandler(const HandlerParameter& param) final {
     ServerReader<RequestType> reader(param.call, param.server_context);
     ResponseType rsp;
-    Status status = func_(service_, param.server_context, &reader, &rsp);
+    Status status = CatchingFunctionHandler([this, &param, &reader, &rsp] {
+      return func_(service_, param.server_context, &reader, &rsp);
+    });
 
     GPR_CODEGEN_ASSERT(!param.server_context->sent_initial_metadata_);
     CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
@@ -130,7 +153,9 @@ class ServerStreamingHandler : public MethodHandler {
 
     if (status.ok()) {
       ServerWriter<ResponseType> writer(param.call, param.server_context);
-      status = func_(service_, param.server_context, &req, &writer);
+      status = CatchingFunctionHandler([this, &param, &req, &writer] {
+        return func_(service_, param.server_context, &req, &writer);
+      });
     }
 
     CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> ops;
@@ -172,7 +197,9 @@ class TemplatedBidiStreamingHandler : public MethodHandler {
 
   void RunHandler(const HandlerParameter& param) final {
     Streamer stream(param.call, param.server_context);
-    Status status = func_(param.server_context, &stream);
+    Status status = CatchingFunctionHandler([this, &param, &stream] {
+      return func_(param.server_context, &stream);
+    });
 
     CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> ops;
     if (!param.server_context->sent_initial_metadata_) {
diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD
index fa77c30aca6..8894c68b95d 100644
--- a/test/cpp/end2end/BUILD
+++ b/test/cpp/end2end/BUILD
@@ -16,25 +16,31 @@ licenses(["notice"])  # Apache v2
 
 load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_package", "grpc_cc_binary")
 
-grpc_package(name = "test/cpp/end2end", visibility = "public") # Allows external users to implement end2end tests.
+grpc_package(
+    name = "test/cpp/end2end",
+    visibility = "public",
+)  # Allows external users to implement end2end tests.
 
 grpc_cc_library(
     name = "test_service_impl",
     testonly = True,
     srcs = ["test_service_impl.cc"],
     hdrs = ["test_service_impl.h"],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         "//src/proto/grpc/testing:echo_proto",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "async_end2end_test",
     srcs = ["async_end2end_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         "//:gpr",
         "//:grpc",
@@ -47,14 +53,17 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "client_crash_test",
     srcs = ["client_crash_test.cc"],
+    data = [
+        ":client_crash_test_server",
+    ],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         "//:gpr",
         "//:grpc",
@@ -66,18 +75,16 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    data = [
-        ":client_crash_test_server",
-    ],
-    external_deps = [
-        "gtest",
-    ],
 )
 
 grpc_cc_binary(
     name = "client_crash_test_server",
     testonly = True,
     srcs = ["client_crash_test_server.cc"],
+    external_deps = [
+        "gflags",
+        "gtest",
+    ],
     deps = [
         "//:gpr",
         "//:grpc",
@@ -89,16 +96,15 @@ grpc_cc_binary(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gflags",
-        "gtest",
-    ],
 )
 
 grpc_cc_library(
     name = "end2end_test_lib",
-    srcs = ["end2end_test.cc"],
     testonly = True,
+    srcs = ["end2end_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         ":test_service_impl",
         "//:gpr",
@@ -111,40 +117,58 @@ grpc_cc_library(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "end2end_test",
     deps = [
-        ":end2end_test_lib"
+        ":end2end_test_lib",
     ],
 )
 
 grpc_cc_test(
-    name = "filter_end2end_test",
-    srcs = ["filter_end2end_test.cc"],
+    name = "exception_test",
+    srcs = ["exception_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         "//:gpr",
         "//:grpc",
         "//:grpc++",
         "//src/proto/grpc/testing:echo_messages_proto",
         "//src/proto/grpc/testing:echo_proto",
-        "//src/proto/grpc/testing/duplicate:echo_duplicate_proto",
         "//test/core/util:gpr_test_util",
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
+)
+
+grpc_cc_test(
+    name = "filter_end2end_test",
+    srcs = ["filter_end2end_test.cc"],
     external_deps = [
         "gtest",
     ],
+    deps = [
+        "//:gpr",
+        "//:grpc",
+        "//:grpc++",
+        "//src/proto/grpc/testing:echo_messages_proto",
+        "//src/proto/grpc/testing:echo_proto",
+        "//src/proto/grpc/testing/duplicate:echo_duplicate_proto",
+        "//test/core/util:gpr_test_util",
+        "//test/core/util:grpc_test_util",
+        "//test/cpp/util:test_util",
+    ],
 )
 
 grpc_cc_test(
     name = "generic_end2end_test",
     srcs = ["generic_end2end_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         "//:gpr",
         "//:grpc",
@@ -156,14 +180,14 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "hybrid_end2end_test",
     srcs = ["hybrid_end2end_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         ":test_service_impl",
         "//:gpr",
@@ -176,14 +200,15 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "mock_test",
     srcs = ["mock_test.cc"],
+    external_deps = [
+        "gmock",
+        "gtest",
+    ],
     deps = [
         "//:gpr",
         "//:grpc",
@@ -196,15 +221,14 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gmock",
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "client_lb_end2end_test",
     srcs = ["client_lb_end2end_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         ":test_service_impl",
         "//:gpr",
@@ -217,37 +241,38 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "grpclb_end2end_test",
     srcs = ["grpclb_end2end_test.cc"],
+    external_deps = [
+        "gmock",
+        "gtest",
+    ],
     deps = [
         ":test_service_impl",
         "//:gpr",
         "//:grpc",
         "//:grpc++",
+        "//:grpc_resolver_fake",
         "//src/proto/grpc/lb/v1:load_balancer_proto",
         "//src/proto/grpc/testing:echo_messages_proto",
         "//src/proto/grpc/testing:echo_proto",
         "//src/proto/grpc/testing/duplicate:echo_duplicate_proto",
-        "//:grpc_resolver_fake",
         "//test/core/util:gpr_test_util",
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gmock",
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "proto_server_reflection_test",
     srcs = ["proto_server_reflection_test.cc"],
+    external_deps = [
+        "gtest",
+        "gflags",
+    ],
     deps = [
         ":test_service_impl",
         "//:gpr",
@@ -262,15 +287,14 @@ grpc_cc_test(
         "//test/cpp/util:grpc++_proto_reflection_desc_db",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-        "gflags",
-    ],
 )
 
 grpc_cc_test(
     name = "server_builder_plugin_test",
     srcs = ["server_builder_plugin_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         ":test_service_impl",
         "//:gpr",
@@ -283,14 +307,17 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "server_crash_test",
     srcs = ["server_crash_test.cc"],
+    data = [
+        ":server_crash_test_client",
+    ],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         "//:gpr",
         "//:grpc",
@@ -302,18 +329,16 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-    ],
-    data = [
-        ":server_crash_test_client",
-    ],
 )
 
 grpc_cc_binary(
     name = "server_crash_test_client",
     testonly = True,
     srcs = ["server_crash_test_client.cc"],
+    external_deps = [
+        "gflags",
+        "gtest",
+    ],
     deps = [
         "//:gpr",
         "//:grpc",
@@ -325,15 +350,14 @@ grpc_cc_binary(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gflags",
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "shutdown_test",
     srcs = ["shutdown_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         "//:gpr",
         "//:grpc",
@@ -345,14 +369,14 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "streaming_throughput_test",
     srcs = ["streaming_throughput_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         "//:gpr",
         "//:grpc",
@@ -364,14 +388,14 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-    ],
 )
 
 grpc_cc_test(
     name = "thread_stress_test",
     srcs = ["thread_stress_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
     deps = [
         "//:gpr",
         "//:grpc",
@@ -383,7 +407,4 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
-    external_deps = [
-        "gtest",
-    ],
 )
diff --git a/test/cpp/end2end/exception_test.cc b/test/cpp/end2end/exception_test.cc
new file mode 100644
index 00000000000..6545ffa5304
--- /dev/null
+++ b/test/cpp/end2end/exception_test.cc
@@ -0,0 +1,116 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <exception>
+#include <memory>
+
+#include <grpc++/channel.h>
+#include <grpc++/client_context.h>
+#include <grpc++/server.h>
+#include <grpc++/server_builder.h>
+#include <grpc++/server_context.h>
+
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/core/util/test_config.h"
+
+#include <gtest/gtest.h>
+
+namespace grpc {
+namespace testing {
+
+const char* kErrorMessage = "This service caused an exception";
+
+class ExceptingServiceImpl : public ::grpc::testing::EchoTestService::Service {
+ public:
+  Status Echo(ServerContext* server_context, const EchoRequest* request,
+              EchoResponse* response) override {
+    throw - 1;
+  }
+  Status RequestStream(ServerContext* context,
+                       ServerReader<EchoRequest>* reader,
+                       EchoResponse* response) override {
+    throw ServiceException();
+  }
+
+ private:
+  class ServiceException final : public std::exception {
+   public:
+    ServiceException() {}
+
+   private:
+    const char* what() const noexcept override { return kErrorMessage; }
+  };
+};
+
+class ExceptionTest : public ::testing::Test {
+ protected:
+  ExceptionTest() {}
+
+  void SetUp() override {
+    ServerBuilder builder;
+    builder.RegisterService(&service_);
+    server_ = builder.BuildAndStart();
+  }
+
+  void TearDown() override { server_->Shutdown(); }
+
+  void ResetStub() {
+    channel_ = server_->InProcessChannel(ChannelArguments());
+    stub_ = grpc::testing::EchoTestService::NewStub(channel_);
+  }
+
+  std::shared_ptr<Channel> channel_;
+  std::unique_ptr<grpc::testing::EchoTestService::Stub> stub_;
+  std::unique_ptr<Server> server_;
+  ExceptingServiceImpl service_;
+};
+
+TEST_F(ExceptionTest, Unary) {
+  ResetStub();
+  EchoRequest request;
+  EchoResponse response;
+  request.set_message("test");
+  ClientContext context;
+
+  Status s = stub_->Echo(&context, request, &response);
+  EXPECT_FALSE(s.ok());
+  EXPECT_EQ(s.error_code(), StatusCode::UNKNOWN);
+}
+
+TEST_F(ExceptionTest, RequestStream) {
+  ResetStub();
+  EchoResponse response;
+  ClientContext context;
+
+  auto stream = stub_->RequestStream(&context, &response);
+  stream->WritesDone();
+  Status s = stream->Finish();
+
+  EXPECT_FALSE(s.ok());
+  EXPECT_EQ(s.error_code(), StatusCode::UNKNOWN);
+  EXPECT_EQ(s.error_message(), kErrorMessage);
+}
+
+}  // namespace testing
+}  // namespace grpc
+
+int main(int argc, char** argv) {
+  grpc_test_init(argc, argv);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index d432bd0e537..137e36a432e 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -3160,6 +3160,25 @@
     "third_party": false, 
     "type": "target"
   }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc++", 
+      "grpc++_test_util", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c++", 
+    "name": "exception_test", 
+    "src": [
+      "test/cpp/end2end/exception_test.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
   {
     "deps": [
       "gpr", 
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index 98517cba2e7..bb672bea7da 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -3695,6 +3695,30 @@
     ], 
     "uses_polling": true
   }, 
+  {
+    "args": [], 
+    "benchmark": false, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": true, 
+    "language": "c++", 
+    "name": "exception_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "uses_polling": false
+  }, 
   {
     "args": [], 
     "benchmark": false, 

From 9427eabf3bf9f1de14a3fabdc72e090dac7601e1 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Tue, 19 Dec 2017 08:28:08 -0800
Subject: [PATCH 043/161] uses_polling shouldn't be false for this test

---
 build.yaml                           | 1 -
 tools/run_tests/generated/tests.json | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/build.yaml b/build.yaml
index 8ab15b77414..e217504dd51 100644
--- a/build.yaml
+++ b/build.yaml
@@ -4020,7 +4020,6 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
-  uses_polling: false
 - name: filter_end2end_test
   gtest: true
   build: test
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index bb672bea7da..72d1f612e6c 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -3717,7 +3717,7 @@
       "posix", 
       "windows"
     ], 
-    "uses_polling": false
+    "uses_polling": true
   }, 
   {
     "args": [], 

From 9809ce38e9f79b4e9a0b1ec1c076cce0beee1e98 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Tue, 19 Dec 2017 09:09:52 -0800
Subject: [PATCH 044/161] Use appropriate preprocessor guards to allow building
 without exceptions

---
 BUILD                                          | 18 ++++++++++++++----
 bazel/grpc_build_system.bzl                    |  4 ++++
 .../grpc++/impl/codegen/method_handler_impl.h  | 10 +++++++---
 include/grpc/impl/codegen/port_platform.h      | 15 +++++++++++++++
 test/cpp/end2end/exception_test.cc             |  4 ++++
 5 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/BUILD b/BUILD
index dba6592f172..ebd198275ea 100644
--- a/BUILD
+++ b/BUILD
@@ -38,6 +38,16 @@ config_setting(
     values = {"define": "grpc_no_ares=true"},
 )
 
+config_setting(
+    name = "grpc_allow_exceptions",
+    values = {"define": "GRPC_ALLOW_EXCEPTIONS=1"},
+)
+
+config_setting(
+    name = "grpc_disallow_exceptions",
+    values = {"define": "GRPC_ALLOW_EXCEPTIONS=0"},
+)
+
 config_setting(
     name = "remote_execution",
     values = {"define": "GRPC_PORT_ISOLATED_RUNTIME=1"},
@@ -544,24 +554,24 @@ grpc_cc_library(
 
 grpc_cc_library(
     name = "debug_location",
-    public_hdrs = ["src/core/lib/support/debug_location.h"],
     language = "c++",
+    public_hdrs = ["src/core/lib/support/debug_location.h"],
 )
 
 grpc_cc_library(
     name = "ref_counted",
-    public_hdrs = ["src/core/lib/support/ref_counted.h"],
     language = "c++",
+    public_hdrs = ["src/core/lib/support/ref_counted.h"],
     deps = [
-        "grpc_trace",
         "debug_location",
+        "grpc_trace",
     ],
 )
 
 grpc_cc_library(
     name = "ref_counted_ptr",
-    public_hdrs = ["src/core/lib/support/ref_counted_ptr.h"],
     language = "c++",
+    public_hdrs = ["src/core/lib/support/ref_counted_ptr.h"],
 )
 
 grpc_cc_library(
diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
index d146ca9c2ca..d61eced2d96 100644
--- a/bazel/grpc_build_system.bzl
+++ b/bazel/grpc_build_system.bzl
@@ -60,6 +60,10 @@ def grpc_cc_library(name, srcs = [], public_hdrs = [], hdrs = [],
     defines = select({"//:grpc_no_ares": ["GRPC_ARES=0"],
                       "//conditions:default": [],}) +
               select({"//:remote_execution":  ["GRPC_PORT_ISOLATED_RUNTIME=1"],
+                      "//conditions:default": [],} +
+              select({"//:grpc_allow_exceptions":  ["GRPC_ALLOW_EXCEPTIONS=1"],
+                      "//:grpc_disallow_exceptions":
+                      ["GRPC_ALLOW_EXCEPTIONS=0"],
                       "//conditions:default": [],}),
     hdrs = _maybe_update_cc_library_hdrs(hdrs + public_hdrs),
     deps = deps + _get_external_deps(external_deps),
diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h
index b72dceb1b4c..41c287231fa 100644
--- a/include/grpc++/impl/codegen/method_handler_impl.h
+++ b/include/grpc++/impl/codegen/method_handler_impl.h
@@ -35,15 +35,19 @@ namespace internal {
 // so this process doesn't require additional overhead in the common case.
 // Additionally, we don't need to return if we caught an exception or not;
 // the handling is the same in either case.
-template <class F>
-Status CatchingFunctionHandler(F&& callable) {
+template <class Callable>
+Status CatchingFunctionHandler(Callable&& handler) {
+#if GRPC_ALLOW_EXCEPTIONS
   try {
-    return callable();
+    return handler();
   } catch (const std::exception& e) {
     return Status(StatusCode::UNKNOWN, e.what());
   } catch (...) {
     return Status(StatusCode::UNKNOWN, "Exception in method handler");
   }
+#else
+  return handler();
+#endif
 }
 
 /// A wrapper class of an application provided rpc method handler.
diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h
index e6bee73ef15..becb16b5b8e 100644
--- a/include/grpc/impl/codegen/port_platform.h
+++ b/include/grpc/impl/codegen/port_platform.h
@@ -477,6 +477,21 @@ typedef unsigned __int64 uint64_t;
 #endif /* GPR_ATTRIBUTE_NO_TSAN (2) */
 #endif /* GPR_ATTRIBUTE_NO_TSAN (1) */
 
+/* GRPC_ALLOW_EXCEPTIONS should be 0 or 1 if exceptions are allowed or not */
+#ifndef GRPC_ALLOW_EXCEPTIONS
+/* If not already set, set to 1 on Windows (style guide standard) but to
+ * 0 on non-Windows platforms unless the compiler defines __EXCEPTIONS */
+#ifdef GPR_WINDOWS
+#define GRPC_ALLOW_EXCEPTIONS 1
+#else
+#ifdef __EXCEPTIONS
+#define GRPC_ALLOW_EXCEPTIONS 1
+#else
+#define GRPC_ALLOW_EXCEPTIONS 0
+#endif
+#endif
+#endif
+
 #ifndef __STDC_FORMAT_MACROS
 #define __STDC_FORMAT_MACROS
 #endif
diff --git a/test/cpp/end2end/exception_test.cc b/test/cpp/end2end/exception_test.cc
index 6545ffa5304..7e0d5c7951d 100644
--- a/test/cpp/end2end/exception_test.cc
+++ b/test/cpp/end2end/exception_test.cc
@@ -24,6 +24,7 @@
 #include <grpc++/server.h>
 #include <grpc++/server_builder.h>
 #include <grpc++/server_context.h>
+#include <grpc/impl/codegen/port_platform.h>
 
 #include "src/proto/grpc/testing/echo.grpc.pb.h"
 #include "test/core/util/test_config.h"
@@ -35,6 +36,7 @@ namespace testing {
 
 const char* kErrorMessage = "This service caused an exception";
 
+#if GRPC_ALLOW_EXCEPTIONS
 class ExceptingServiceImpl : public ::grpc::testing::EchoTestService::Service {
  public:
   Status Echo(ServerContext* server_context, const EchoRequest* request,
@@ -106,6 +108,8 @@ TEST_F(ExceptionTest, RequestStream) {
   EXPECT_EQ(s.error_message(), kErrorMessage);
 }
 
+#endif
+
 }  // namespace testing
 }  // namespace grpc
 

From 37abc4ae7b295651d739bdc1f9071af49d4831b8 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Tue, 19 Dec 2017 09:20:50 -0800
Subject: [PATCH 045/161] Typo

---
 bazel/grpc_build_system.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
index d61eced2d96..33bfa74e103 100644
--- a/bazel/grpc_build_system.bzl
+++ b/bazel/grpc_build_system.bzl
@@ -60,7 +60,7 @@ def grpc_cc_library(name, srcs = [], public_hdrs = [], hdrs = [],
     defines = select({"//:grpc_no_ares": ["GRPC_ARES=0"],
                       "//conditions:default": [],}) +
               select({"//:remote_execution":  ["GRPC_PORT_ISOLATED_RUNTIME=1"],
-                      "//conditions:default": [],} +
+                      "//conditions:default": [],}) +
               select({"//:grpc_allow_exceptions":  ["GRPC_ALLOW_EXCEPTIONS=1"],
                       "//:grpc_disallow_exceptions":
                       ["GRPC_ALLOW_EXCEPTIONS=0"],

From ab0065478419c785e21012fbbeb2c28b5a37d3b3 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Tue, 19 Dec 2017 10:17:11 -0800
Subject: [PATCH 046/161] Tag new #else and #endif blocks with comments

---
 include/grpc++/impl/codegen/method_handler_impl.h |  4 ++--
 include/grpc/impl/codegen/port_platform.h         | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h
index 41c287231fa..ed6c146e6c2 100644
--- a/include/grpc++/impl/codegen/method_handler_impl.h
+++ b/include/grpc++/impl/codegen/method_handler_impl.h
@@ -45,9 +45,9 @@ Status CatchingFunctionHandler(Callable&& handler) {
   } catch (...) {
     return Status(StatusCode::UNKNOWN, "Exception in method handler");
   }
-#else
+#else   // GRPC_ALLOW_EXCEPTIONS
   return handler();
-#endif
+#endif  // GRPC_ALLOW_EXCEPTIONS
 }
 
 /// A wrapper class of an application provided rpc method handler.
diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h
index becb16b5b8e..a03c31dc9d0 100644
--- a/include/grpc/impl/codegen/port_platform.h
+++ b/include/grpc/impl/codegen/port_platform.h
@@ -483,14 +483,14 @@ typedef unsigned __int64 uint64_t;
  * 0 on non-Windows platforms unless the compiler defines __EXCEPTIONS */
 #ifdef GPR_WINDOWS
 #define GRPC_ALLOW_EXCEPTIONS 1
-#else
+#else /* GPR_WINDOWS */
 #ifdef __EXCEPTIONS
 #define GRPC_ALLOW_EXCEPTIONS 1
-#else
+#else /* __EXCEPTIONS */
 #define GRPC_ALLOW_EXCEPTIONS 0
-#endif
-#endif
-#endif
+#endif /* __EXCEPTIONS */
+#endif /* __GPR_WINDOWS */
+#endif /* GRPC_ALLOW_EXCEPTIONS */
 
 #ifndef __STDC_FORMAT_MACROS
 #define __STDC_FORMAT_MACROS

From b425fc79da44da1641b03cd6c35cf13b542c8e07 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Tue, 19 Dec 2017 10:20:12 -0800
Subject: [PATCH 047/161] Comment on cpp guard

---
 test/cpp/end2end/exception_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/cpp/end2end/exception_test.cc b/test/cpp/end2end/exception_test.cc
index 7e0d5c7951d..722276d149f 100644
--- a/test/cpp/end2end/exception_test.cc
+++ b/test/cpp/end2end/exception_test.cc
@@ -108,7 +108,7 @@ TEST_F(ExceptionTest, RequestStream) {
   EXPECT_EQ(s.error_message(), kErrorMessage);
 }
 
-#endif
+#endif  // GRPC_ALLOW_EXCEPTIONS
 
 }  // namespace testing
 }  // namespace grpc

From 9d2d84382e317e5c8a149c497bfe9408af12b038 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Mon, 8 Jan 2018 15:11:23 -0800
Subject: [PATCH 048/161] Add portability tests on Linux to make sure that
 build with -fno-exceptions continues to work

---
 Makefile                               | 25 +++++++++++++++++--------
 build.yaml                             |  4 ++++
 tools/run_tests/generated/configs.json |  5 ++++-
 tools/run_tests/run_tests_matrix.py    |  9 +++++++++
 4 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 339c52294a6..01b7a6879a4 100644
--- a/Makefile
+++ b/Makefile
@@ -142,14 +142,14 @@ LDXX_asan-noleaks = clang++
 CPPFLAGS_asan-noleaks = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS
 LDFLAGS_asan-noleaks = -fsanitize=address
 
-VALID_CONFIG_c++-compat = 1
-CC_c++-compat = $(DEFAULT_CC)
-CXX_c++-compat = $(DEFAULT_CXX)
-LD_c++-compat = $(DEFAULT_CC)
-LDXX_c++-compat = $(DEFAULT_CXX)
-CFLAGS_c++-compat = -Wc++-compat
-CPPFLAGS_c++-compat = -O0
-DEFINES_c++-compat = _DEBUG DEBUG
+VALID_CONFIG_noexcept = 1
+CC_noexcept = $(DEFAULT_CC)
+CXX_noexcept = $(DEFAULT_CXX)
+LD_noexcept = $(DEFAULT_CC)
+LDXX_noexcept = $(DEFAULT_CXX)
+CXXFLAGS_noexcept = -fno-exceptions
+CPPFLAGS_noexcept = -O2
+DEFINES_noexcept = NDEBUG
 
 VALID_CONFIG_ubsan = 1
 REQUIRE_CUSTOM_LIBRARIES_ubsan = 1
@@ -205,6 +205,15 @@ LDXX_lto = $(DEFAULT_CXX)
 CPPFLAGS_lto = -O2
 DEFINES_lto = NDEBUG
 
+VALID_CONFIG_c++-compat = 1
+CC_c++-compat = $(DEFAULT_CC)
+CXX_c++-compat = $(DEFAULT_CXX)
+LD_c++-compat = $(DEFAULT_CC)
+LDXX_c++-compat = $(DEFAULT_CXX)
+CFLAGS_c++-compat = -Wc++-compat
+CPPFLAGS_c++-compat = -O0
+DEFINES_c++-compat = _DEBUG DEBUG
+
 VALID_CONFIG_mutrace = 1
 CC_mutrace = $(DEFAULT_CC)
 CXX_mutrace = $(DEFAULT_CXX)
diff --git a/build.yaml b/build.yaml
index e217504dd51..238d9970abb 100644
--- a/build.yaml
+++ b/build.yaml
@@ -4966,6 +4966,10 @@ configs:
     CPPFLAGS: -O3 -fno-omit-frame-pointer
     DEFINES: NDEBUG
     LDFLAGS: -rdynamic
+  noexcept:
+    CPPFLAGS: -O2
+    CXXFLAGS: -fno-exceptions
+    DEFINES: NDEBUG
   opt:
     CPPFLAGS: -O2
     DEFINES: NDEBUG
diff --git a/tools/run_tests/generated/configs.json b/tools/run_tests/generated/configs.json
index fee8290efe0..a14340cb7bd 100644
--- a/tools/run_tests/generated/configs.json
+++ b/tools/run_tests/generated/configs.json
@@ -42,7 +42,7 @@
     }
   }, 
   {
-    "config": "c++-compat"
+    "config": "noexcept"
   }, 
   {
     "config": "ubsan", 
@@ -73,6 +73,9 @@
   {
     "config": "lto"
   }, 
+  {
+    "config": "c++-compat"
+  }, 
   {
     "config": "mutrace"
   }, 
diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index ac90bef5ac0..344035478b5 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -313,6 +313,15 @@ def _create_portability_test_jobs(extra_args=[],
         extra_envs={'GRPC_DNS_RESOLVER': 'ares'},
         timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
 
+    # C and C++ with no-exceptions on Linux
+    test_jobs += _generate_jobs(
+        languages=['c', 'c++'],
+        configs=['noexcept'],
+        platforms=['linux'],
+        labels=['portability', 'corelang'],
+        extra_args=extra_args,
+        timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+
     # TODO(zyc): Turn on this test after adding c-ares support on windows.
     # C with the c-ares DNS resolver on Windows
     # test_jobs += _generate_jobs(languages=['c'],

From 43e09a5f3e5b0d46862d7e05b7eb54f660eabb89 Mon Sep 17 00:00:00 2001
From: Ken Payson <kpayson@google.com>
Date: Fri, 1 Dec 2017 15:58:04 -0800
Subject: [PATCH 049/161] Initialize last sent ping time

---
 .../ext/transport/chttp2/transport/chttp2_transport.cc    | 1 +
 src/core/ext/transport/chttp2/transport/writing.cc        | 8 ++++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 3a2c4b6d1bd..63ac65ac788 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -549,6 +549,7 @@ static void init_transport(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
   /* No pings allowed before receiving a header or data frame. */
   t->ping_state.pings_before_data_required = 0;
   t->ping_state.is_delayed_ping_timer_set = false;
+  t->ping_state.last_ping_sent_time = GRPC_MILLIS_INF_PAST;
 
   t->ping_recv_state.last_ping_recv_time = GRPC_MILLIS_INF_PAST;
   t->ping_recv_state.ping_strikes = 0;
diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc
index 15869b8880c..204b5a77087 100644
--- a/src/core/ext/transport/chttp2/transport/writing.cc
+++ b/src/core/ext/transport/chttp2/transport/writing.cc
@@ -81,8 +81,11 @@ static void maybe_initiate_ping(grpc_exec_ctx* exec_ctx,
     /* not enough elapsed time between successive pings */
     if (grpc_http_trace.enabled() || grpc_bdp_estimator_trace.enabled()) {
       gpr_log(GPR_DEBUG,
-              "%s: Ping delayed [%p]: not enough time elapsed since last ping",
-              t->is_client ? "CLIENT" : "SERVER", t->peer_string);
+              "%s: Ping delayed [%p]: not enough time elapsed since last ping. "
+              " Last ping %f: Next ping %f: Now %f",
+              t->is_client ? "CLIENT" : "SERVER", t->peer_string,
+              (double)t->ping_state.last_ping_sent_time,
+              (double)next_allowed_ping, (double)now);
     }
     if (!t->ping_state.is_delayed_ping_timer_set) {
       t->ping_state.is_delayed_ping_timer_set = true;
@@ -91,6 +94,7 @@ static void maybe_initiate_ping(grpc_exec_ctx* exec_ctx,
     }
     return;
   }
+
   pq->inflight_id = t->ping_ctr;
   t->ping_ctr++;
   GRPC_CLOSURE_LIST_SCHED(exec_ctx, &pq->lists[GRPC_CHTTP2_PCL_INITIATE]);

From 642eba3e67fe4c32d1f7f5d4bb2324a3e67f4cf5 Mon Sep 17 00:00:00 2001
From: yang-g <yangg@google.com>
Date: Tue, 5 Dec 2017 17:34:48 -0800
Subject: [PATCH 050/161] Bug fix, use the last_sent ping time for next send

---
 .../ext/transport/chttp2/transport/writing.cc     | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc
index 204b5a77087..cfaa0d0f703 100644
--- a/src/core/ext/transport/chttp2/transport/writing.cc
+++ b/src/core/ext/transport/chttp2/transport/writing.cc
@@ -69,14 +69,15 @@ static void maybe_initiate_ping(grpc_exec_ctx* exec_ctx,
     return;
   }
   grpc_millis now = grpc_exec_ctx_now(exec_ctx);
+
+  grpc_millis next_allowed_ping_interval =
+      (t->keepalive_permit_without_calls == 0 &&
+       grpc_chttp2_stream_map_size(&t->stream_map) == 0)
+          ? 7200 * GPR_MS_PER_SEC
+          : t->ping_policy.min_sent_ping_interval_without_data;
   grpc_millis next_allowed_ping =
-      t->ping_state.last_ping_sent_time +
-      t->ping_policy.min_sent_ping_interval_without_data;
-  if (t->keepalive_permit_without_calls == 0 &&
-      grpc_chttp2_stream_map_size(&t->stream_map) == 0) {
-    next_allowed_ping =
-        t->ping_recv_state.last_ping_recv_time + 7200 * GPR_MS_PER_SEC;
-  }
+      t->ping_state.last_ping_sent_time + next_allowed_ping_interval;
+
   if (next_allowed_ping > now) {
     /* not enough elapsed time between successive pings */
     if (grpc_http_trace.enabled() || grpc_bdp_estimator_trace.enabled()) {

From 4ed35d17f531ad83b0553afc67adb99c6f3deb2f Mon Sep 17 00:00:00 2001
From: Juanli Shen <juanlishen@google.com>
Date: Mon, 8 Jan 2018 18:01:45 -0800
Subject: [PATCH 051/161] Unify the timer callback bools

---
 .../client_channel/lb_policy/grpclb/grpclb.cc | 49 +++++++++----------
 1 file changed, 23 insertions(+), 26 deletions(-)

diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index 3c64213fb93..8621b85d52d 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -365,11 +365,11 @@ typedef struct glb_lb_policy {
   /** are we already watching the LB channel's connectivity? */
   bool watching_lb_channel;
 
-  /** is \a lb_call_retry_timer active? */
-  bool retry_timer_active;
+  /** is the callback associated with \a lb_call_retry_timer pending? */
+  bool retry_timer_callback_pending;
 
-  /** is \a lb_fallback_timer active? */
-  bool fallback_timer_active;
+  /** is the callback associated with \a lb_fallback_timer pending? */
+  bool fallback_timer_callback_pending;
 
   /** called upon changes to the LB channel's connectivity. */
   grpc_closure lb_channel_on_connectivity_changed;
@@ -424,7 +424,7 @@ typedef struct glb_lb_policy {
   /* Interval and timer for next client load report. */
   grpc_millis client_stats_report_interval;
   grpc_timer client_load_report_timer;
-  bool client_load_report_timer_pending;
+  bool client_load_report_timer_callback_pending;
   bool last_client_load_report_counters_were_zero;
   /* Closure used for either the load report timer or the callback for
    * completion of sending the load report. */
@@ -1019,13 +1019,11 @@ static void glb_shutdown_locked(grpc_lb_policy* pol) {
     grpc_call_cancel(lb_call, nullptr);
     /* lb_on_server_status_received will pick up the cancel and clean up */
   }
-  if (glb_policy->retry_timer_active) {
+  if (glb_policy->retry_timer_callback_pending) {
     grpc_timer_cancel(&glb_policy->lb_call_retry_timer);
-    glb_policy->retry_timer_active = false;
   }
-  if (glb_policy->fallback_timer_active) {
+  if (glb_policy->fallback_timer_callback_pending) {
     grpc_timer_cancel(&glb_policy->lb_fallback_timer);
-    glb_policy->fallback_timer_active = false;
   }
 
   pending_pick* pp = glb_policy->pending_picks;
@@ -1154,14 +1152,15 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy);
 static void start_picking_locked(glb_lb_policy* glb_policy) {
   /* start a timer to fall back */
   if (glb_policy->lb_fallback_timeout_ms > 0 &&
-      glb_policy->serverlist == nullptr && !glb_policy->fallback_timer_active) {
+      glb_policy->serverlist == nullptr &&
+      !glb_policy->fallback_timer_callback_pending) {
     grpc_millis deadline =
         grpc_core::ExecCtx::Get()->Now() + glb_policy->lb_fallback_timeout_ms;
     GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer");
     GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked,
                       glb_policy,
                       grpc_combiner_scheduler(glb_policy->base.combiner));
-    glb_policy->fallback_timer_active = true;
+    glb_policy->fallback_timer_callback_pending = true;
     grpc_timer_init(&glb_policy->lb_fallback_timer, deadline,
                     &glb_policy->lb_on_fallback);
   }
@@ -1282,7 +1281,7 @@ static void glb_notify_on_state_change_locked(grpc_lb_policy* pol,
 
 static void lb_call_on_retry_timer_locked(void* arg, grpc_error* error) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)arg;
-  glb_policy->retry_timer_active = false;
+  glb_policy->retry_timer_callback_pending = false;
   if (!glb_policy->shutting_down && glb_policy->lb_call == nullptr &&
       error == GRPC_ERROR_NONE) {
     if (grpc_lb_glb_trace.enabled()) {
@@ -1295,7 +1294,7 @@ static void lb_call_on_retry_timer_locked(void* arg, grpc_error* error) {
 
 static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
   if (glb_policy->started_picking && glb_policy->updating_lb_call) {
-    if (glb_policy->retry_timer_active) {
+    if (glb_policy->retry_timer_callback_pending) {
       grpc_timer_cancel(&glb_policy->lb_call_retry_timer);
     }
     if (!glb_policy->shutting_down) start_picking_locked(glb_policy);
@@ -1310,10 +1309,10 @@ static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
       grpc_millis timeout = next_try - grpc_core::ExecCtx::Get()->Now();
       if (timeout > 0) {
         gpr_log(GPR_DEBUG,
-                "[grpclb %p] ... retry_timer_active in %" PRIuPTR "ms.",
+                "[grpclb %p] ... retry LB call after %" PRIuPTR "ms.",
                 glb_policy, timeout);
       } else {
-        gpr_log(GPR_DEBUG, "[grpclb %p] ... retry_timer_active immediately.",
+        gpr_log(GPR_DEBUG, "[grpclb %p] ... retry LB call immediately.",
                 glb_policy);
       }
     }
@@ -1321,7 +1320,7 @@ static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
     GRPC_CLOSURE_INIT(&glb_policy->lb_on_call_retry,
                       lb_call_on_retry_timer_locked, glb_policy,
                       grpc_combiner_scheduler(glb_policy->base.combiner));
-    glb_policy->retry_timer_active = true;
+    glb_policy->retry_timer_callback_pending = true;
     grpc_timer_init(&glb_policy->lb_call_retry_timer, next_try,
                     &glb_policy->lb_on_call_retry);
   }
@@ -1348,7 +1347,7 @@ static void client_load_report_done_locked(void* arg, grpc_error* error) {
   grpc_byte_buffer_destroy(glb_policy->client_load_report_payload);
   glb_policy->client_load_report_payload = nullptr;
   if (error != GRPC_ERROR_NONE || glb_policy->lb_call == nullptr) {
-    glb_policy->client_load_report_timer_pending = false;
+    glb_policy->client_load_report_timer_callback_pending = false;
     GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "client_load_report");
     if (glb_policy->lb_call == nullptr) {
       maybe_restart_lb_call(glb_policy);
@@ -1373,7 +1372,7 @@ static bool load_report_counters_are_zero(grpc_grpclb_request* request) {
 static void send_client_load_report_locked(void* arg, grpc_error* error) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)arg;
   if (error == GRPC_ERROR_CANCELLED || glb_policy->lb_call == nullptr) {
-    glb_policy->client_load_report_timer_pending = false;
+    glb_policy->client_load_report_timer_callback_pending = false;
     GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "client_load_report");
     if (glb_policy->lb_call == nullptr) {
       maybe_restart_lb_call(glb_policy);
@@ -1485,7 +1484,7 @@ static void lb_call_destroy_locked(glb_lb_policy* glb_policy) {
   grpc_byte_buffer_destroy(glb_policy->lb_request_payload);
   grpc_slice_unref_internal(glb_policy->lb_call_status_details);
 
-  if (glb_policy->client_load_report_timer_pending) {
+  if (glb_policy->client_load_report_timer_callback_pending) {
     grpc_timer_cancel(&glb_policy->client_load_report_timer);
   }
 }
@@ -1598,7 +1597,7 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) {
         /* take a weak ref (won't prevent calling of \a glb_shutdown() if the
          * strong ref count goes to zero) to be unref'd in
          * send_client_load_report_locked() */
-        glb_policy->client_load_report_timer_pending = true;
+        glb_policy->client_load_report_timer_callback_pending = true;
         GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "client_load_report");
         schedule_next_client_load_report(glb_policy);
       } else if (grpc_lb_glb_trace.enabled()) {
@@ -1647,9 +1646,8 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) {
               /* or dispose of the fallback */
               grpc_lb_addresses_destroy(glb_policy->fallback_backend_addresses);
               glb_policy->fallback_backend_addresses = nullptr;
-              if (glb_policy->fallback_timer_active) {
+              if (glb_policy->fallback_timer_callback_pending) {
                 grpc_timer_cancel(&glb_policy->lb_fallback_timer);
-                glb_policy->fallback_timer_active = false;
               }
             }
             /* and update the copy in the glb_lb_policy instance. This
@@ -1702,7 +1700,7 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) {
 
 static void lb_on_fallback_timer_locked(void* arg, grpc_error* error) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)arg;
-  glb_policy->fallback_timer_active = false;
+  glb_policy->fallback_timer_callback_pending = false;
   /* If we receive a serverlist after the timer fires but before this callback
    * actually runs, don't fall back. */
   if (glb_policy->serverlist == nullptr) {
@@ -1737,7 +1735,7 @@ static void lb_on_server_status_received_locked(void* arg, grpc_error* error) {
   // If the load report timer is still pending, we wait for it to be
   // called before restarting the call.  Otherwise, we restart the call
   // here.
-  if (!glb_policy->client_load_report_timer_pending) {
+  if (!glb_policy->client_load_report_timer_callback_pending) {
     maybe_restart_lb_call(glb_policy);
   }
 }
@@ -1847,9 +1845,8 @@ static void glb_lb_channel_on_connectivity_changed_cb(void* arg,
         // lb_on_server_status_received() will pick up the cancel and reinit
         // lb_call.
       } else if (glb_policy->started_picking) {
-        if (glb_policy->retry_timer_active) {
+        if (glb_policy->retry_timer_callback_pending) {
           grpc_timer_cancel(&glb_policy->lb_call_retry_timer);
-          glb_policy->retry_timer_active = false;
         }
         start_picking_locked(glb_policy);
       }

From 32f9e98581acd8de7c5e986e6a186e5e3d2dadce Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Mon, 8 Jan 2018 18:02:04 -0800
Subject: [PATCH 052/161] Remove garbage file doc/md

Delete accidentally committed file from repo
---
 doc/md | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 doc/md

diff --git a/doc/md b/doc/md
deleted file mode 100644
index e69de29bb2d..00000000000

From 7b0bca2efd6356e4d62391de2bf4698cf1dc0211 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Tue, 9 Jan 2018 12:13:57 +0100
Subject: [PATCH 053/161] do not throw and eat exceptions in C# connectivity
 watcher

---
 src/csharp/Grpc.Core/Channel.cs | 38 ++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/csharp/Grpc.Core/Channel.cs b/src/csharp/Grpc.Core/Channel.cs
index e39da9c1c2f..e7b30cd1e94 100644
--- a/src/csharp/Grpc.Core/Channel.cs
+++ b/src/csharp/Grpc.Core/Channel.cs
@@ -130,15 +130,8 @@ namespace Grpc.Core
         // cached handler for watch connectivity state
         static readonly BatchCompletionDelegate WatchConnectivityStateHandler = (success, ctx, state) =>
         {
-            var tcs = (TaskCompletionSource<object>) state;
-            if (success)
-            {
-                tcs.SetResult(null);
-            }
-            else
-            {
-                tcs.SetCanceled();
-            }
+            var tcs = (TaskCompletionSource<bool>) state;
+            tcs.SetResult(success);
         };
 
         /// <summary>
@@ -146,11 +139,24 @@ namespace Grpc.Core
         /// given lastObservedState. 
         /// If deadline is reached or and error occurs, returned task is cancelled.
         /// </summary>
-        public Task WaitForStateChangedAsync(ChannelState lastObservedState, DateTime? deadline = null)
+        public async Task WaitForStateChangedAsync(ChannelState lastObservedState, DateTime? deadline = null)
+        {
+            var result = await WaitForStateChangedInternalAsync(lastObservedState, deadline).ConfigureAwait(false);
+            if (!result)
+            {
+                throw new TaskCanceledException("Reached deadline.");
+            }
+        }
+
+        /// <summary>
+        /// Returned tasks completes once channel state has become different from
+        /// given lastObservedState (<c>true</c> is returned) or if the wait has timed out (<c>false</c> is returned).
+        /// </summary>
+        internal Task<bool> WaitForStateChangedInternalAsync(ChannelState lastObservedState, DateTime? deadline = null)
         {
             GrpcPreconditions.CheckArgument(lastObservedState != ChannelState.Shutdown,
                 "Shutdown is a terminal state. No further state changes can occur.");
-            var tcs = new TaskCompletionSource<object>();
+            var tcs = new TaskCompletionSource<bool>();
             var deadlineTimespec = deadline.HasValue ? Timespec.FromDateTime(deadline.Value) : Timespec.InfFuture;
             lock (myLock)
             {
@@ -320,14 +326,8 @@ namespace Grpc.Core
                         }
                     }
 
-                    try
-                    {
-                        await WaitForStateChangedAsync(lastState, DateTime.UtcNow.AddSeconds(1)).ConfigureAwait(false);
-                    }
-                    catch (TaskCanceledException)
-                    {
-                        // ignore timeout
-                    }
+                    // ignore the result
+                    await WaitForStateChangedInternalAsync(lastState, DateTime.UtcNow.AddSeconds(1)).ConfigureAwait(false);
                     lastState = State;
                 }
             }

From c0febd3e6494205972f711d7f6f7ce311ef6df0f Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Tue, 9 Jan 2018 10:25:24 -0800
Subject: [PATCH 054/161] Simplify LB policy refcounting.

---
 .../filters/client_channel/client_channel.cc  |  71 +-
 .../ext/filters/client_channel/lb_policy.cc   |  95 +--
 .../ext/filters/client_channel/lb_policy.h    |  90 ++-
 .../client_channel/lb_policy/grpclb/grpclb.cc | 607 +++++++-----------
 .../lb_policy/pick_first/pick_first.cc        |  91 ++-
 .../lb_policy/round_robin/round_robin.cc      | 124 ++--
 .../lb_policy/subchannel_list.cc              |   4 +-
 7 files changed, 415 insertions(+), 667 deletions(-)

diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc
index e99022a91b8..3f3334d44a3 100644
--- a/src/core/ext/filters/client_channel/client_channel.cc
+++ b/src/core/ext/filters/client_channel/client_channel.cc
@@ -553,6 +553,7 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) {
       }
       grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties,
                                        chand->interested_parties);
+      grpc_lb_policy_shutdown_locked(chand->lb_policy, new_lb_policy);
       GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel");
     }
     chand->lb_policy = new_lb_policy;
@@ -658,6 +659,7 @@ static void start_transport_op_locked(void* arg, grpc_error* error_ignored) {
       if (chand->lb_policy != nullptr) {
         grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties,
                                          chand->interested_parties);
+        grpc_lb_policy_shutdown_locked(chand->lb_policy, nullptr);
         GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel");
         chand->lb_policy = nullptr;
       }
@@ -792,6 +794,7 @@ static void cc_destroy_channel_elem(grpc_channel_element* elem) {
   if (chand->lb_policy != nullptr) {
     grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties,
                                      chand->interested_parties);
+    grpc_lb_policy_shutdown_locked(chand->lb_policy, nullptr);
     GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel");
   }
   gpr_free(chand->info_lb_policy_name);
@@ -852,12 +855,10 @@ typedef struct client_channel_call_data {
   grpc_subchannel_call* subchannel_call;
   grpc_error* error;
 
-  grpc_lb_policy* lb_policy;  // Holds ref while LB pick is pending.
+  grpc_lb_policy_pick_state pick;
   grpc_closure lb_pick_closure;
   grpc_closure lb_pick_cancel_closure;
 
-  grpc_connected_subchannel* connected_subchannel;
-  grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
   grpc_polling_entity* pollent;
 
   grpc_transport_stream_op_batch* waiting_for_pick_batches[MAX_WAITING_BATCHES];
@@ -866,8 +867,6 @@ typedef struct client_channel_call_data {
 
   grpc_transport_stream_op_batch* initial_metadata_batch;
 
-  grpc_linked_mdelem lb_token_mdelem;
-
   grpc_closure on_complete;
   grpc_closure* original_on_complete;
 } call_data;
@@ -1005,16 +1004,16 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
   channel_data* chand = (channel_data*)elem->channel_data;
   call_data* calld = (call_data*)elem->call_data;
   const grpc_connected_subchannel_call_args call_args = {
-      calld->pollent,                  // pollent
-      calld->path,                     // path
-      calld->call_start_time,          // start_time
-      calld->deadline,                 // deadline
-      calld->arena,                    // arena
-      calld->subchannel_call_context,  // context
-      calld->call_combiner             // call_combiner
+      calld->pollent,                       // pollent
+      calld->path,                          // path
+      calld->call_start_time,               // start_time
+      calld->deadline,                      // deadline
+      calld->arena,                         // arena
+      calld->pick.subchannel_call_context,  // context
+      calld->call_combiner                  // call_combiner
   };
   grpc_error* new_error = grpc_connected_subchannel_create_call(
-      calld->connected_subchannel, &call_args, &calld->subchannel_call);
+      calld->pick.connected_subchannel, &call_args, &calld->subchannel_call);
   if (grpc_client_channel_trace.enabled()) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: create subchannel_call=%p: error=%s",
             chand, calld, calld->subchannel_call, grpc_error_string(new_error));
@@ -1032,7 +1031,7 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
 static void pick_done_locked(grpc_call_element* elem, grpc_error* error) {
   call_data* calld = (call_data*)elem->call_data;
   channel_data* chand = (channel_data*)elem->channel_data;
-  if (calld->connected_subchannel == nullptr) {
+  if (calld->pick.connected_subchannel == nullptr) {
     // Failed to create subchannel.
     GRPC_ERROR_UNREF(calld->error);
     calld->error = error == GRPC_ERROR_NONE
@@ -1071,13 +1070,16 @@ static void pick_callback_cancel_locked(void* arg, grpc_error* error) {
   grpc_call_element* elem = (grpc_call_element*)arg;
   channel_data* chand = (channel_data*)elem->channel_data;
   call_data* calld = (call_data*)elem->call_data;
-  if (calld->lb_policy != nullptr) {
+  // Note: chand->lb_policy may have changed since we started our pick,
+  // in which case we will be cancelling the pick on a policy other than
+  // the one we started it on.  However, this will just be a no-op.
+  if (error != GRPC_ERROR_NONE && chand->lb_policy != nullptr) {
     if (grpc_client_channel_trace.enabled()) {
       gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p",
-              chand, calld, calld->lb_policy);
+              chand, calld, chand->lb_policy);
     }
-    grpc_lb_policy_cancel_pick_locked(
-        calld->lb_policy, &calld->connected_subchannel, GRPC_ERROR_REF(error));
+    grpc_lb_policy_cancel_pick_locked(chand->lb_policy, &calld->pick,
+                                      GRPC_ERROR_REF(error));
   }
   GRPC_CALL_STACK_UNREF(calld->owning_call, "pick_callback_cancel");
 }
@@ -1092,9 +1094,6 @@ static void pick_callback_done_locked(void* arg, grpc_error* error) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously",
             chand, calld);
   }
-  GPR_ASSERT(calld->lb_policy != nullptr);
-  GRPC_LB_POLICY_UNREF(calld->lb_policy, "pick_subchannel");
-  calld->lb_policy = nullptr;
   async_pick_done_locked(elem, GRPC_ERROR_REF(error));
 }
 
@@ -1128,26 +1127,21 @@ static bool pick_callback_start_locked(grpc_call_element* elem) {
       initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY;
     }
   }
-  const grpc_lb_policy_pick_args inputs = {
+  calld->pick.initial_metadata =
       calld->initial_metadata_batch->payload->send_initial_metadata
-          .send_initial_metadata,
-      initial_metadata_flags, &calld->lb_token_mdelem};
-  // Keep a ref to the LB policy in calld while the pick is pending.
-  GRPC_LB_POLICY_REF(chand->lb_policy, "pick_subchannel");
-  calld->lb_policy = chand->lb_policy;
+          .send_initial_metadata;
+  calld->pick.initial_metadata_flags = initial_metadata_flags;
   GRPC_CLOSURE_INIT(&calld->lb_pick_closure, pick_callback_done_locked, elem,
                     grpc_combiner_scheduler(chand->combiner));
-  const bool pick_done = grpc_lb_policy_pick_locked(
-      chand->lb_policy, &inputs, &calld->connected_subchannel,
-      calld->subchannel_call_context, nullptr, &calld->lb_pick_closure);
+  calld->pick.on_complete = &calld->lb_pick_closure;
+  const bool pick_done =
+      grpc_lb_policy_pick_locked(chand->lb_policy, &calld->pick);
   if (pick_done) {
     /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */
     if (grpc_client_channel_trace.enabled()) {
       gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed synchronously",
               chand, calld);
     }
-    GRPC_LB_POLICY_UNREF(calld->lb_policy, "pick_subchannel");
-    calld->lb_policy = nullptr;
   } else {
     GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel");
     grpc_call_combiner_set_notify_on_cancel(
@@ -1289,7 +1283,7 @@ static void start_pick_locked(void* arg, grpc_error* ignored) {
   grpc_call_element* elem = (grpc_call_element*)arg;
   call_data* calld = (call_data*)elem->call_data;
   channel_data* chand = (channel_data*)elem->channel_data;
-  GPR_ASSERT(calld->connected_subchannel == nullptr);
+  GPR_ASSERT(calld->pick.connected_subchannel == nullptr);
   if (chand->lb_policy != nullptr) {
     // We already have an LB policy, so ask it for a pick.
     if (pick_callback_start_locked(elem)) {
@@ -1467,15 +1461,14 @@ static void cc_destroy_call_elem(grpc_call_element* elem,
     GRPC_SUBCHANNEL_CALL_UNREF(calld->subchannel_call,
                                "client_channel_destroy_call");
   }
-  GPR_ASSERT(calld->lb_policy == nullptr);
   GPR_ASSERT(calld->waiting_for_pick_batches_count == 0);
-  if (calld->connected_subchannel != nullptr) {
-    GRPC_CONNECTED_SUBCHANNEL_UNREF(calld->connected_subchannel, "picked");
+  if (calld->pick.connected_subchannel != nullptr) {
+    GRPC_CONNECTED_SUBCHANNEL_UNREF(calld->pick.connected_subchannel, "picked");
   }
   for (size_t i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
-    if (calld->subchannel_call_context[i].value != nullptr) {
-      calld->subchannel_call_context[i].destroy(
-          calld->subchannel_call_context[i].value);
+    if (calld->pick.subchannel_call_context[i].value != nullptr) {
+      calld->pick.subchannel_call_context[i].destroy(
+          calld->pick.subchannel_call_context[i].value);
     }
   }
   GRPC_CLOSURE_SCHED(then_schedule_closure, GRPC_ERROR_NONE);
diff --git a/src/core/ext/filters/client_channel/lb_policy.cc b/src/core/ext/filters/client_channel/lb_policy.cc
index 7a5a8dec34b..cc4fe7ec627 100644
--- a/src/core/ext/filters/client_channel/lb_policy.cc
+++ b/src/core/ext/filters/client_channel/lb_policy.cc
@@ -19,8 +19,6 @@
 #include "src/core/ext/filters/client_channel/lb_policy.h"
 #include "src/core/lib/iomgr/combiner.h"
 
-#define WEAK_REF_BITS 16
-
 grpc_core::DebugOnlyTraceFlag grpc_trace_lb_policy_refcount(
     false, "lb_policy_refcount");
 
@@ -28,91 +26,60 @@ void grpc_lb_policy_init(grpc_lb_policy* policy,
                          const grpc_lb_policy_vtable* vtable,
                          grpc_combiner* combiner) {
   policy->vtable = vtable;
-  gpr_atm_no_barrier_store(&policy->ref_pair, 1 << WEAK_REF_BITS);
+  gpr_ref_init(&policy->refs, 1);
   policy->interested_parties = grpc_pollset_set_create();
   policy->combiner = GRPC_COMBINER_REF(combiner, "lb_policy");
 }
 
 #ifndef NDEBUG
-#define REF_FUNC_EXTRA_ARGS , const char *file, int line, const char *reason
-#define REF_MUTATE_EXTRA_ARGS REF_FUNC_EXTRA_ARGS, const char* purpose
-#define REF_FUNC_PASS_ARGS(new_reason) , file, line, new_reason
-#define REF_MUTATE_PASS_ARGS(purpose) , file, line, reason, purpose
+void grpc_lb_policy_ref(grpc_lb_policy* lb_policy, const char* file, int line,
+                        const char* reason) {
+  if (grpc_trace_lb_policy_refcount.enabled()) {
+    gpr_atm old_refs = gpr_atm_no_barrier_load(&lb_policy->refs.count);
+    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+            "LB_POLICY:%p   ref %" PRIdPTR " -> %" PRIdPTR " %s", lb_policy,
+            old_refs, old_refs + 1, reason);
+  }
 #else
-#define REF_FUNC_EXTRA_ARGS
-#define REF_MUTATE_EXTRA_ARGS
-#define REF_FUNC_PASS_ARGS(new_reason)
-#define REF_MUTATE_PASS_ARGS(x)
+void grpc_lb_policy_ref(grpc_lb_policy* lb_policy) {
 #endif
+  gpr_ref(&lb_policy->refs);
+}
 
-static gpr_atm ref_mutate(grpc_lb_policy* c, gpr_atm delta,
-                          int barrier REF_MUTATE_EXTRA_ARGS) {
-  gpr_atm old_val = barrier ? gpr_atm_full_fetch_add(&c->ref_pair, delta)
-                            : gpr_atm_no_barrier_fetch_add(&c->ref_pair, delta);
 #ifndef NDEBUG
+void grpc_lb_policy_unref(grpc_lb_policy* lb_policy, const char* file, int line,
+                          const char* reason) {
   if (grpc_trace_lb_policy_refcount.enabled()) {
+    gpr_atm old_refs = gpr_atm_no_barrier_load(&lb_policy->refs.count);
     gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-            "LB_POLICY: %p %12s 0x%" PRIxPTR " -> 0x%" PRIxPTR " [%s]", c,
-            purpose, old_val, old_val + delta, reason);
+            "LB_POLICY:%p unref %" PRIdPTR " -> %" PRIdPTR " %s", lb_policy,
+            old_refs, old_refs - 1, reason);
   }
+#else
+void grpc_lb_policy_unref(grpc_lb_policy* lb_policy) {
 #endif
-  return old_val;
-}
-
-void grpc_lb_policy_ref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
-  ref_mutate(policy, 1 << WEAK_REF_BITS, 0 REF_MUTATE_PASS_ARGS("STRONG_REF"));
-}
-
-static void shutdown_locked(void* arg, grpc_error* error) {
-  grpc_lb_policy* policy = (grpc_lb_policy*)arg;
-  policy->vtable->shutdown_locked(policy);
-  GRPC_LB_POLICY_WEAK_UNREF(policy, "strong-unref");
-}
-
-void grpc_lb_policy_unref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
-  gpr_atm old_val =
-      ref_mutate(policy, (gpr_atm)1 - (gpr_atm)(1 << WEAK_REF_BITS),
-                 1 REF_MUTATE_PASS_ARGS("STRONG_UNREF"));
-  gpr_atm mask = ~(gpr_atm)((1 << WEAK_REF_BITS) - 1);
-  gpr_atm check = 1 << WEAK_REF_BITS;
-  if ((old_val & mask) == check) {
-    GRPC_CLOSURE_SCHED(
-        GRPC_CLOSURE_CREATE(shutdown_locked, policy,
-                            grpc_combiner_scheduler(policy->combiner)),
-        GRPC_ERROR_NONE);
-  } else {
-    grpc_lb_policy_weak_unref(policy REF_FUNC_PASS_ARGS("strong-unref"));
+  if (gpr_unref(&lb_policy->refs)) {
+    grpc_pollset_set_destroy(lb_policy->interested_parties);
+    grpc_combiner* combiner = lb_policy->combiner;
+    lb_policy->vtable->destroy(lb_policy);
+    GRPC_COMBINER_UNREF(combiner, "lb_policy");
   }
 }
 
-void grpc_lb_policy_weak_ref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
-  ref_mutate(policy, 1, 0 REF_MUTATE_PASS_ARGS("WEAK_REF"));
-}
-
-void grpc_lb_policy_weak_unref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
-  gpr_atm old_val =
-      ref_mutate(policy, -(gpr_atm)1, 1 REF_MUTATE_PASS_ARGS("WEAK_UNREF"));
-  if (old_val == 1) {
-    grpc_pollset_set_destroy(policy->interested_parties);
-    grpc_combiner* combiner = policy->combiner;
-    policy->vtable->destroy(policy);
-    GRPC_COMBINER_UNREF(combiner, "lb_policy");
-  }
+void grpc_lb_policy_shutdown_locked(grpc_lb_policy* policy,
+                                    grpc_lb_policy* new_policy) {
+  policy->vtable->shutdown_locked(policy, new_policy);
 }
 
 int grpc_lb_policy_pick_locked(grpc_lb_policy* policy,
-                               const grpc_lb_policy_pick_args* pick_args,
-                               grpc_connected_subchannel** target,
-                               grpc_call_context_element* context,
-                               void** user_data, grpc_closure* on_complete) {
-  return policy->vtable->pick_locked(policy, pick_args, target, context,
-                                     user_data, on_complete);
+                               grpc_lb_policy_pick_state* pick) {
+  return policy->vtable->pick_locked(policy, pick);
 }
 
 void grpc_lb_policy_cancel_pick_locked(grpc_lb_policy* policy,
-                                       grpc_connected_subchannel** target,
+                                       grpc_lb_policy_pick_state* pick,
                                        grpc_error* error) {
-  policy->vtable->cancel_pick_locked(policy, target, error);
+  policy->vtable->cancel_pick_locked(policy, pick, error);
 }
 
 void grpc_lb_policy_cancel_picks_locked(grpc_lb_policy* policy,
diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h
index 3572c97ed14..1176a05b780 100644
--- a/src/core/ext/filters/client_channel/lb_policy.h
+++ b/src/core/ext/filters/client_channel/lb_policy.h
@@ -33,7 +33,7 @@ extern grpc_core::DebugOnlyTraceFlag grpc_trace_lb_policy_refcount;
 
 struct grpc_lb_policy {
   const grpc_lb_policy_vtable* vtable;
-  gpr_atm ref_pair;
+  gpr_refcount refs;
   /* owned pointer to interested parties in load balancing decisions */
   grpc_pollset_set* interested_parties;
   /* combiner under which lb_policy actions take place */
@@ -42,32 +42,42 @@ struct grpc_lb_policy {
   grpc_closure* request_reresolution;
 };
 
-/** Extra arguments for an LB pick */
-typedef struct grpc_lb_policy_pick_args {
-  /** Initial metadata associated with the picking call. */
+/// State used for an LB pick.
+typedef struct grpc_lb_policy_pick_state {
+  /// Initial metadata associated with the picking call.
   grpc_metadata_batch* initial_metadata;
-  /** Bitmask used for selective cancelling. See \a
-   * grpc_lb_policy_cancel_picks() and \a GRPC_INITIAL_METADATA_* in
-   * grpc_types.h */
+  /// Bitmask used for selective cancelling. See \a
+  /// grpc_lb_policy_cancel_picks() and \a GRPC_INITIAL_METADATA_* in
+  /// grpc_types.h.
   uint32_t initial_metadata_flags;
-  /** Storage for LB token in \a initial_metadata, or NULL if not used */
-  grpc_linked_mdelem* lb_token_mdelem_storage;
-} grpc_lb_policy_pick_args;
+  /// Storage for LB token in \a initial_metadata, or NULL if not used.
+  grpc_linked_mdelem lb_token_mdelem_storage;
+  /// Closure to run when pick is complete, if not completed synchronously.
+  grpc_closure* on_complete;
+  /// Will be set to the selected subchannel, or NULL on failure or when
+  /// the LB policy decides to drop the call.
+  grpc_connected_subchannel* connected_subchannel;
+  /// Will be populated with context to pass to the subchannel call, if needed.
+  grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
+  /// Upon success, \a *user_data will be set to whatever opaque information
+  /// may need to be propagated from the LB policy, or NULL if not needed.
+  void** user_data;
+  /// Next pointer.  For internal use by LB policy.
+  struct grpc_lb_policy_pick_state* next;
+} grpc_lb_policy_pick_state;
 
 struct grpc_lb_policy_vtable {
   void (*destroy)(grpc_lb_policy* policy);
-  void (*shutdown_locked)(grpc_lb_policy* policy);
+
+  /// \see grpc_lb_policy_shutdown_locked().
+  void (*shutdown_locked)(grpc_lb_policy* policy, grpc_lb_policy* new_policy);
 
   /** \see grpc_lb_policy_pick */
-  int (*pick_locked)(grpc_lb_policy* policy,
-                     const grpc_lb_policy_pick_args* pick_args,
-                     grpc_connected_subchannel** target,
-                     grpc_call_context_element* context, void** user_data,
-                     grpc_closure* on_complete);
+  int (*pick_locked)(grpc_lb_policy* policy, grpc_lb_policy_pick_state* pick);
 
   /** \see grpc_lb_policy_cancel_pick */
   void (*cancel_pick_locked)(grpc_lb_policy* policy,
-                             grpc_connected_subchannel** target,
+                             grpc_lb_policy_pick_state* pick,
                              grpc_error* error);
 
   /** \see grpc_lb_policy_cancel_picks */
@@ -103,37 +113,19 @@ struct grpc_lb_policy_vtable {
 };
 
 #ifndef NDEBUG
-
-/* Strong references: the policy will shutdown when they reach zero */
 #define GRPC_LB_POLICY_REF(p, r) \
   grpc_lb_policy_ref((p), __FILE__, __LINE__, (r))
 #define GRPC_LB_POLICY_UNREF(p, r) \
   grpc_lb_policy_unref((p), __FILE__, __LINE__, (r))
-
-/* Weak references: they don't prevent the shutdown of the LB policy. When no
- * strong references are left but there are still weak ones, shutdown is called.
- * Once the weak reference also reaches zero, the LB policy is destroyed. */
-#define GRPC_LB_POLICY_WEAK_REF(p, r) \
-  grpc_lb_policy_weak_ref((p), __FILE__, __LINE__, (r))
-#define GRPC_LB_POLICY_WEAK_UNREF(p, r) \
-  grpc_lb_policy_weak_unref((p), __FILE__, __LINE__, (r))
 void grpc_lb_policy_ref(grpc_lb_policy* policy, const char* file, int line,
                         const char* reason);
 void grpc_lb_policy_unref(grpc_lb_policy* policy, const char* file, int line,
                           const char* reason);
-void grpc_lb_policy_weak_ref(grpc_lb_policy* policy, const char* file, int line,
-                             const char* reason);
-void grpc_lb_policy_weak_unref(grpc_lb_policy* policy, const char* file,
-                               int line, const char* reason);
-#else
+#else  // !NDEBUG
 #define GRPC_LB_POLICY_REF(p, r) grpc_lb_policy_ref((p))
 #define GRPC_LB_POLICY_UNREF(p, r) grpc_lb_policy_unref((p))
-#define GRPC_LB_POLICY_WEAK_REF(p, r) grpc_lb_policy_weak_ref((p))
-#define GRPC_LB_POLICY_WEAK_UNREF(p, r) grpc_lb_policy_weak_unref((p))
 void grpc_lb_policy_ref(grpc_lb_policy* policy);
 void grpc_lb_policy_unref(grpc_lb_policy* policy);
-void grpc_lb_policy_weak_ref(grpc_lb_policy* policy);
-void grpc_lb_policy_weak_unref(grpc_lb_policy* policy);
 #endif
 
 /** called by concrete implementations to initialize the base struct */
@@ -141,28 +133,24 @@ void grpc_lb_policy_init(grpc_lb_policy* policy,
                          const grpc_lb_policy_vtable* vtable,
                          grpc_combiner* combiner);
 
-/** Finds an appropriate subchannel for a call, based on \a pick_args.
-
-    \a target will be set to the selected subchannel, or NULL on failure
-    or when the LB policy decides to drop the call.
+/// Shuts down \a policy.
+/// If \a new_policy is non-null, any pending picks will be restarted
+/// on that policy; otherwise, they will be failed.
+void grpc_lb_policy_shutdown_locked(grpc_lb_policy* policy,
+                                    grpc_lb_policy* new_policy);
 
-    Upon success, \a user_data will be set to whatever opaque information
-    may need to be propagated from the LB policy, or NULL if not needed.
-    \a context will be populated with context to pass to the subchannel
-    call, if needed.
+/** Finds an appropriate subchannel for a call, based on data in \a pick.
+    \a pick must remain alive until the pick is complete.
 
     If the pick succeeds and a result is known immediately, a non-zero
-    value will be returned.  Otherwise, \a on_complete will be invoked
+    value will be returned.  Otherwise, \a pick->on_complete will be invoked
     once the pick is complete with its error argument set to indicate
     success or failure.
 
     Any IO should be done under the \a interested_parties \a grpc_pollset_set
     in the \a grpc_lb_policy struct. */
 int grpc_lb_policy_pick_locked(grpc_lb_policy* policy,
-                               const grpc_lb_policy_pick_args* pick_args,
-                               grpc_connected_subchannel** target,
-                               grpc_call_context_element* context,
-                               void** user_data, grpc_closure* on_complete);
+                               grpc_lb_policy_pick_state* pick);
 
 /** Perform a connected subchannel ping (see \a grpc_connected_subchannel_ping)
     against one of the connected subchannels managed by \a policy. */
@@ -170,11 +158,11 @@ void grpc_lb_policy_ping_one_locked(grpc_lb_policy* policy,
                                     grpc_closure* on_initiate,
                                     grpc_closure* on_ack);
 
-/** Cancel picks for \a target.
+/** Cancel picks for \a pick.
     The \a on_complete callback of the pending picks will be invoked with \a
     *target set to NULL. */
 void grpc_lb_policy_cancel_pick_locked(grpc_lb_policy* policy,
-                                       grpc_connected_subchannel** target,
+                                       grpc_lb_policy_pick_state* pick,
                                        grpc_error* error);
 
 /** Cancel all pending picks for which their \a initial_metadata_flags (as given
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index 1317cdcf750..5849ac9d2da 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -54,7 +54,7 @@
  *    operations in progress over the old RR instance. This is done by
  *    decreasing the reference count on the old policy. The moment no more
  *    references are held on the old RR policy, it'll be destroyed and \a
- *    glb_rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN
+ *    on_rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN
  *    state. At this point we can transition to a new RR instance safely, which
  *    is done once again via \a rr_handover_locked().
  *
@@ -128,187 +128,48 @@
 
 grpc_core::TraceFlag grpc_lb_glb_trace(false, "glb");
 
-/* add lb_token of selected subchannel (address) to the call's initial
- * metadata */
-static grpc_error* initial_metadata_add_lb_token(
-    grpc_metadata_batch* initial_metadata,
-    grpc_linked_mdelem* lb_token_mdelem_storage, grpc_mdelem lb_token) {
-  GPR_ASSERT(lb_token_mdelem_storage != nullptr);
-  GPR_ASSERT(!GRPC_MDISNULL(lb_token));
-  return grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage,
-                                      lb_token);
-}
+struct glb_lb_policy;
 
-static void destroy_client_stats(void* arg) {
-  grpc_grpclb_client_stats_unref((grpc_grpclb_client_stats*)arg);
-}
-
-typedef struct wrapped_rr_closure_arg {
-  /* the closure instance using this struct as argument */
-  grpc_closure wrapper_closure;
-
-  /* the original closure. Usually a on_complete/notify cb for pick() and ping()
-   * calls against the internal RR instance, respectively. */
-  grpc_closure* wrapped_closure;
-
-  /* the pick's initial metadata, kept in order to append the LB token for the
-   * pick */
-  grpc_metadata_batch* initial_metadata;
-
-  /* the picked target, used to determine which LB token to add to the pick's
-   * initial metadata */
-  grpc_connected_subchannel** target;
-
-  /* the context to be populated for the subchannel call */
-  grpc_call_context_element* context;
+namespace {
 
-  /* Stats for client-side load reporting. Note that this holds a
-   * reference, which must be either passed on via context or unreffed. */
+/// Linked list of pending pick requests. It stores all information needed to
+/// eventually call (Round Robin's) pick() on them. They mainly stay pending
+/// waiting for the RR policy to be created.
+///
+/// Note that when a pick is sent to the RR policy, we inject our own
+/// on_complete callback, so that we can intercept the result before
+/// invoking the original on_complete callback.  This allows us to set the
+/// LB token metadata and add client_stats to the call context.
+/// See \a pending_pick_complete() for details.
+struct pending_pick {
+  // Our on_complete closure and the original one.
+  grpc_closure on_complete;
+  grpc_closure* original_on_complete;
+  // The original pick.
+  grpc_lb_policy_pick_state* pick;
+  // Stats for client-side load reporting. Note that this holds a
+  // reference, which must be either passed on via context or unreffed.
   grpc_grpclb_client_stats* client_stats;
-
-  /* the LB token associated with the pick */
+  // The LB token associated with the pick.  This is set via user_data in
+  // the pick.
   grpc_mdelem lb_token;
-
-  /* storage for the lb token initial metadata mdelem */
-  grpc_linked_mdelem* lb_token_mdelem_storage;
-
-  /* The RR instance related to the closure */
-  grpc_lb_policy* rr_policy;
-
-  /* The grpclb instance that created the wrapping. This instance is not owned,
-   * reference counts are untouched. It's used only for logging purposes. */
-  grpc_lb_policy* glb_policy;
-
-  /* heap memory to be freed upon closure execution. */
-  void* free_when_done;
-} wrapped_rr_closure_arg;
-
-/* The \a on_complete closure passed as part of the pick requires keeping a
- * reference to its associated round robin instance. We wrap this closure in
- * order to unref the round robin instance upon its invocation */
-static void wrapped_rr_closure(void* arg, grpc_error* error) {
-  wrapped_rr_closure_arg* wc_arg = (wrapped_rr_closure_arg*)arg;
-
-  GPR_ASSERT(wc_arg->wrapped_closure != nullptr);
-  GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_REF(error));
-
-  if (wc_arg->rr_policy != nullptr) {
-    /* if *target is nullptr, no pick has been made by the RR policy (eg, all
-     * addresses failed to connect). There won't be any user_data/token
-     * available */
-    if (*wc_arg->target != nullptr) {
-      if (!GRPC_MDISNULL(wc_arg->lb_token)) {
-        initial_metadata_add_lb_token(wc_arg->initial_metadata,
-                                      wc_arg->lb_token_mdelem_storage,
-                                      GRPC_MDELEM_REF(wc_arg->lb_token));
-      } else {
-        gpr_log(
-            GPR_ERROR,
-            "[grpclb %p] No LB token for connected subchannel pick %p (from RR "
-            "instance %p).",
-            wc_arg->glb_policy, *wc_arg->target, wc_arg->rr_policy);
-        abort();
-      }
-      // Pass on client stats via context. Passes ownership of the reference.
-      GPR_ASSERT(wc_arg->client_stats != nullptr);
-      wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].value = wc_arg->client_stats;
-      wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].destroy = destroy_client_stats;
-    } else {
-      grpc_grpclb_client_stats_unref(wc_arg->client_stats);
-    }
-    if (grpc_lb_glb_trace.enabled()) {
-      gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p", wc_arg->glb_policy,
-              wc_arg->rr_policy);
-    }
-    GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "wrapped_rr_closure");
-  }
-  GPR_ASSERT(wc_arg->free_when_done != nullptr);
-  gpr_free(wc_arg->free_when_done);
-}
-
-namespace {
-/* Linked list of pending pick requests. It stores all information needed to
- * eventually call (Round Robin's) pick() on them. They mainly stay pending
- * waiting for the RR policy to be created/updated.
- *
- * One particularity is the wrapping of the user-provided \a on_complete closure
- * (in \a wrapped_on_complete and \a wrapped_on_complete_arg). This is needed in
- * order to correctly unref the RR policy instance upon completion of the pick.
- * See \a wrapped_rr_closure for details. */
-struct pending_pick {
+  // The grpclb instance that created the wrapping. This instance is not owned,
+  // reference counts are untouched. It's used only for logging purposes.
+  glb_lb_policy* glb_policy;
+  // Next pending pick.
   struct pending_pick* next;
-
-  /* original pick()'s arguments */
-  grpc_lb_policy_pick_args pick_args;
-
-  /* output argument where to store the pick()ed connected subchannel, or
-   * nullptr upon error. */
-  grpc_connected_subchannel** target;
-
-  /* args for wrapped_on_complete */
-  wrapped_rr_closure_arg wrapped_on_complete_arg;
 };
-}  // namespace
-
-static void add_pending_pick(pending_pick** root,
-                             const grpc_lb_policy_pick_args* pick_args,
-                             grpc_connected_subchannel** target,
-                             grpc_call_context_element* context,
-                             grpc_closure* on_complete) {
-  pending_pick* pp = (pending_pick*)gpr_zalloc(sizeof(*pp));
-  pp->next = *root;
-  pp->pick_args = *pick_args;
-  pp->target = target;
-  pp->wrapped_on_complete_arg.wrapped_closure = on_complete;
-  pp->wrapped_on_complete_arg.target = target;
-  pp->wrapped_on_complete_arg.context = context;
-  pp->wrapped_on_complete_arg.initial_metadata = pick_args->initial_metadata;
-  pp->wrapped_on_complete_arg.lb_token_mdelem_storage =
-      pick_args->lb_token_mdelem_storage;
-  pp->wrapped_on_complete_arg.free_when_done = pp;
-  GRPC_CLOSURE_INIT(&pp->wrapped_on_complete_arg.wrapper_closure,
-                    wrapped_rr_closure, &pp->wrapped_on_complete_arg,
-                    grpc_schedule_on_exec_ctx);
-  *root = pp;
-}
 
-/* Same as the \a pending_pick struct but for ping operations */
-typedef struct pending_ping {
+/// A linked list of pending pings waiting for the RR policy to be created.
+struct pending_ping {
+  grpc_closure* on_initiate;
+  grpc_closure* on_ack;
   struct pending_ping* next;
+};
 
-  /* args for sending the ping */
-  wrapped_rr_closure_arg* on_initiate;
-  wrapped_rr_closure_arg* on_ack;
-} pending_ping;
-
-static void add_pending_ping(pending_ping** root, grpc_closure* on_initiate,
-                             grpc_closure* on_ack) {
-  pending_ping* pping = (pending_ping*)gpr_zalloc(sizeof(*pping));
-  if (on_initiate != nullptr) {
-    pping->on_initiate =
-        (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(*pping->on_initiate));
-    pping->on_initiate->wrapped_closure = on_initiate;
-    pping->on_initiate->free_when_done = pping->on_initiate;
-    GRPC_CLOSURE_INIT(&pping->on_initiate->wrapper_closure, wrapped_rr_closure,
-                      &pping->on_initiate, grpc_schedule_on_exec_ctx);
-  }
-  if (on_ack != nullptr) {
-    pping->on_ack = (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(*pping->on_ack));
-    pping->on_ack->wrapped_closure = on_ack;
-    pping->on_ack->free_when_done = pping->on_ack;
-    GRPC_CLOSURE_INIT(&pping->on_ack->wrapper_closure, wrapped_rr_closure,
-                      &pping->on_ack, grpc_schedule_on_exec_ctx);
-  }
-  pping->next = *root;
-  *root = pping;
-}
-
-/*
- * glb_lb_policy
- */
-typedef struct rr_connectivity_data rr_connectivity_data;
+}  // namespace
 
-typedef struct glb_lb_policy {
+struct glb_lb_policy {
   /** base policy: must be first */
   grpc_lb_policy base;
 
@@ -333,6 +194,9 @@ typedef struct glb_lb_policy {
   /** the RR policy to use of the backend servers returned by the LB server */
   grpc_lb_policy* rr_policy;
 
+  grpc_closure on_rr_connectivity_changed;
+  grpc_connectivity_state rr_connectivity_state;
+
   bool started_picking;
 
   /** our connectivity state tracker */
@@ -437,15 +301,85 @@ typedef struct glb_lb_policy {
   grpc_closure client_load_report_closure;
   /* Client load report message payload. */
   grpc_byte_buffer* client_load_report_payload;
-} glb_lb_policy;
-
-/* Keeps track and reacts to changes in connectivity of the RR instance */
-struct rr_connectivity_data {
-  grpc_closure on_change;
-  grpc_connectivity_state state;
-  glb_lb_policy* glb_policy;
 };
 
+/* add lb_token of selected subchannel (address) to the call's initial
+ * metadata */
+static grpc_error* initial_metadata_add_lb_token(
+    grpc_metadata_batch* initial_metadata,
+    grpc_linked_mdelem* lb_token_mdelem_storage, grpc_mdelem lb_token) {
+  GPR_ASSERT(lb_token_mdelem_storage != nullptr);
+  GPR_ASSERT(!GRPC_MDISNULL(lb_token));
+  return grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage,
+                                      lb_token);
+}
+
+static void destroy_client_stats(void* arg) {
+  grpc_grpclb_client_stats_unref((grpc_grpclb_client_stats*)arg);
+}
+
+static void pending_pick_set_metadata_and_context(pending_pick* pp) {
+  /* if connected_subchannel is nullptr, no pick has been made by the RR
+   * policy (e.g., all addresses failed to connect). There won't be any
+   * user_data/token available */
+  if (pp->pick->connected_subchannel != nullptr) {
+    if (!GRPC_MDISNULL(pp->lb_token)) {
+      initial_metadata_add_lb_token(pp->pick->initial_metadata,
+                                    &pp->pick->lb_token_mdelem_storage,
+                                    GRPC_MDELEM_REF(pp->lb_token));
+    } else {
+      gpr_log(GPR_ERROR,
+              "[grpclb %p] No LB token for connected subchannel pick %p",
+              pp->glb_policy, pp->pick);
+      abort();
+    }
+    // Pass on client stats via context. Passes ownership of the reference.
+    GPR_ASSERT(pp->client_stats != nullptr);
+    pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].value =
+        pp->client_stats;
+    pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].destroy =
+        destroy_client_stats;
+  } else {
+    grpc_grpclb_client_stats_unref(pp->client_stats);
+  }
+}
+
+/* The \a on_complete closure passed as part of the pick requires keeping a
+ * reference to its associated round robin instance. We wrap this closure in
+ * order to unref the round robin instance upon its invocation */
+static void pending_pick_complete(void* arg, grpc_error* error) {
+  pending_pick* pp = (pending_pick*)arg;
+  pending_pick_set_metadata_and_context(pp);
+  GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_REF(error));
+  gpr_free(pp);
+}
+
+static pending_pick* pending_pick_create(glb_lb_policy* glb_policy,
+                                         grpc_lb_policy_pick_state* pick) {
+  pending_pick* pp = (pending_pick*)gpr_zalloc(sizeof(*pp));
+  pp->pick = pick;
+  pp->glb_policy = glb_policy;
+  GRPC_CLOSURE_INIT(&pp->on_complete, pending_pick_complete, pp,
+                    grpc_schedule_on_exec_ctx);
+  pp->original_on_complete = pick->on_complete;
+  pp->pick->on_complete = &pp->on_complete;
+  return pp;
+}
+
+static void pending_pick_add(pending_pick** root, pending_pick* new_pp) {
+  new_pp->next = *root;
+  *root = new_pp;
+}
+
+static void pending_ping_add(pending_ping** root, grpc_closure* on_initiate,
+                             grpc_closure* on_ack) {
+  pending_ping* pping = (pending_ping*)gpr_zalloc(sizeof(*pping));
+  pping->on_initiate = on_initiate;
+  pping->on_ack = on_ack;
+  pping->next = *root;
+  *root = pping;
+}
+
 static bool is_server_valid(const grpc_grpclb_server* server, size_t idx,
                             bool log) {
   if (server->drop) return false;
@@ -557,7 +491,6 @@ static grpc_lb_addresses* process_serverlist_locked(
       gpr_free(uri);
       user_data = (void*)GRPC_MDELEM_LB_TOKEN_EMPTY.payload;
     }
-
     grpc_lb_addresses_set_address(lb_addresses, addr_idx, &addr.addr, addr.len,
                                   false /* is_balancer */,
                                   nullptr /* balancer_name */, user_data);
@@ -598,7 +531,6 @@ static void update_lb_connectivity_status_locked(
     grpc_error* rr_state_error) {
   const grpc_connectivity_state curr_glb_state =
       grpc_connectivity_state_check(&glb_policy->state_tracker);
-
   /* The new connectivity status is a function of the previous one and the new
    * input coming from the status of the RR policy.
    *
@@ -628,7 +560,6 @@ static void update_lb_connectivity_status_locked(
    *
    *  (*) This function mustn't be called during shutting down. */
   GPR_ASSERT(curr_glb_state != GRPC_CHANNEL_SHUTDOWN);
-
   switch (rr_state) {
     case GRPC_CHANNEL_TRANSIENT_FAILURE:
     case GRPC_CHANNEL_SHUTDOWN:
@@ -639,7 +570,6 @@ static void update_lb_connectivity_status_locked(
     case GRPC_CHANNEL_READY:
       GPR_ASSERT(rr_state_error == GRPC_ERROR_NONE);
   }
-
   if (grpc_lb_glb_trace.enabled()) {
     gpr_log(
         GPR_INFO,
@@ -657,10 +587,8 @@ static void update_lb_connectivity_status_locked(
  * cleanups this callback would otherwise be responsible for.
  * If \a force_async is true, then we will manually schedule the
  * completion callback even if the pick is available immediately. */
-static bool pick_from_internal_rr_locked(
-    glb_lb_policy* glb_policy, const grpc_lb_policy_pick_args* pick_args,
-    bool force_async, grpc_connected_subchannel** target,
-    wrapped_rr_closure_arg* wc_arg) {
+static bool pick_from_internal_rr_locked(glb_lb_policy* glb_policy,
+                                         bool force_async, pending_pick* pp) {
   // Check for drops if we are not using fallback backend addresses.
   if (glb_policy->serverlist != nullptr) {
     // Look at the index into the serverlist to see if we should drop this call.
@@ -670,57 +598,36 @@ static bool pick_from_internal_rr_locked(
       glb_policy->serverlist_index = 0;  // Wrap-around.
     }
     if (server->drop) {
-      // Not using the RR policy, so unref it.
-      if (grpc_lb_glb_trace.enabled()) {
-        gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p for drop", glb_policy,
-                wc_arg->rr_policy);
-      }
-      GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "glb_pick_sync");
       // Update client load reporting stats to indicate the number of
       // dropped calls.  Note that we have to do this here instead of in
       // the client_load_reporting filter, because we do not create a
       // subchannel call (and therefore no client_load_reporting filter)
       // for dropped calls.
-      GPR_ASSERT(wc_arg->client_stats != nullptr);
+      GPR_ASSERT(glb_policy->client_stats != nullptr);
       grpc_grpclb_client_stats_add_call_dropped_locked(
-          server->load_balance_token, wc_arg->client_stats);
-      grpc_grpclb_client_stats_unref(wc_arg->client_stats);
+          server->load_balance_token, glb_policy->client_stats);
       if (force_async) {
-        GPR_ASSERT(wc_arg->wrapped_closure != nullptr);
-        GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_NONE);
-        gpr_free(wc_arg->free_when_done);
+        GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE);
+        gpr_free(pp);
         return false;
       }
-      gpr_free(wc_arg->free_when_done);
+      gpr_free(pp);
       return true;
     }
   }
+  // Set client_stats and user_data.
+  pp->client_stats = grpc_grpclb_client_stats_ref(glb_policy->client_stats);
+  GPR_ASSERT(pp->pick->user_data == nullptr);
+  pp->pick->user_data = (void**)&pp->lb_token;
   // Pick via the RR policy.
-  const bool pick_done = grpc_lb_policy_pick_locked(
-      wc_arg->rr_policy, pick_args, target, wc_arg->context,
-      (void**)&wc_arg->lb_token, &wc_arg->wrapper_closure);
+  bool pick_done = grpc_lb_policy_pick_locked(glb_policy->rr_policy, pp->pick);
   if (pick_done) {
-    /* synchronous grpc_lb_policy_pick call. Unref the RR policy. */
-    if (grpc_lb_glb_trace.enabled()) {
-      gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p", glb_policy,
-              wc_arg->rr_policy);
-    }
-    GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "glb_pick_sync");
-    /* add the load reporting initial metadata */
-    initial_metadata_add_lb_token(pick_args->initial_metadata,
-                                  pick_args->lb_token_mdelem_storage,
-                                  GRPC_MDELEM_REF(wc_arg->lb_token));
-    // Pass on client stats via context. Passes ownership of the reference.
-    GPR_ASSERT(wc_arg->client_stats != nullptr);
-    wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].value = wc_arg->client_stats;
-    wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].destroy = destroy_client_stats;
+    pending_pick_set_metadata_and_context(pp);
     if (force_async) {
-      GPR_ASSERT(wc_arg->wrapped_closure != nullptr);
-      GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_NONE);
-      gpr_free(wc_arg->free_when_done);
-      return false;
+      GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE);
+      pick_done = false;
     }
-    gpr_free(wc_arg->free_when_done);
+    gpr_free(pp);
   }
   /* else, the pending pick will be registered and taken care of by the
    * pending pick list inside the RR policy (glb_policy->rr_policy).
@@ -762,7 +669,7 @@ static void lb_policy_args_destroy(grpc_lb_policy_args* args) {
   gpr_free(args);
 }
 
-static void glb_rr_connectivity_changed_locked(void* arg, grpc_error* error);
+static void on_rr_connectivity_changed_locked(void* arg, grpc_error* error);
 static void create_rr_locked(glb_lb_policy* glb_policy,
                              grpc_lb_policy_args* args) {
   GPR_ASSERT(glb_policy->rr_policy == nullptr);
@@ -784,72 +691,46 @@ static void create_rr_locked(glb_lb_policy* glb_policy,
   glb_policy->base.request_reresolution = nullptr;
   glb_policy->rr_policy = new_rr_policy;
   grpc_error* rr_state_error = nullptr;
-  const grpc_connectivity_state rr_state =
-      grpc_lb_policy_check_connectivity_locked(glb_policy->rr_policy,
-                                               &rr_state_error);
+  glb_policy->rr_connectivity_state = grpc_lb_policy_check_connectivity_locked(
+      glb_policy->rr_policy, &rr_state_error);
   /* Connectivity state is a function of the RR policy updated/created */
-  update_lb_connectivity_status_locked(glb_policy, rr_state, rr_state_error);
+  update_lb_connectivity_status_locked(
+      glb_policy, glb_policy->rr_connectivity_state, rr_state_error);
   /* Add the gRPC LB's interested_parties pollset_set to that of the newly
    * created RR policy. This will make the RR policy progress upon activity on
    * gRPC LB, which in turn is tied to the application's call */
   grpc_pollset_set_add_pollset_set(glb_policy->rr_policy->interested_parties,
                                    glb_policy->base.interested_parties);
-
-  /* Allocate the data for the tracking of the new RR policy's connectivity.
-   * It'll be deallocated in glb_rr_connectivity_changed() */
-  rr_connectivity_data* rr_connectivity =
-      (rr_connectivity_data*)gpr_zalloc(sizeof(rr_connectivity_data));
-  GRPC_CLOSURE_INIT(&rr_connectivity->on_change,
-                    glb_rr_connectivity_changed_locked, rr_connectivity,
+  GRPC_CLOSURE_INIT(&glb_policy->on_rr_connectivity_changed,
+                    on_rr_connectivity_changed_locked, glb_policy,
                     grpc_combiner_scheduler(glb_policy->base.combiner));
-  rr_connectivity->glb_policy = glb_policy;
-  rr_connectivity->state = rr_state;
-
   /* Subscribe to changes to the connectivity of the new RR */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "glb_rr_connectivity_cb");
-  grpc_lb_policy_notify_on_state_change_locked(glb_policy->rr_policy,
-                                               &rr_connectivity->state,
-                                               &rr_connectivity->on_change);
+  GRPC_LB_POLICY_REF(&glb_policy->base, "glb_rr_connectivity_cb");
+  grpc_lb_policy_notify_on_state_change_locked(
+      glb_policy->rr_policy, &glb_policy->rr_connectivity_state,
+      &glb_policy->on_rr_connectivity_changed);
   grpc_lb_policy_exit_idle_locked(glb_policy->rr_policy);
-
-  /* Update picks and pings in wait */
+  // Send pending picks to RR policy.
   pending_pick* pp;
   while ((pp = glb_policy->pending_picks)) {
     glb_policy->pending_picks = pp->next;
-    GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_pick");
-    pp->wrapped_on_complete_arg.rr_policy = glb_policy->rr_policy;
-    pp->wrapped_on_complete_arg.client_stats =
-        grpc_grpclb_client_stats_ref(glb_policy->client_stats);
     if (grpc_lb_glb_trace.enabled()) {
       gpr_log(GPR_INFO,
               "[grpclb %p] Pending pick about to (async) PICK from RR %p",
               glb_policy, glb_policy->rr_policy);
     }
-    pick_from_internal_rr_locked(glb_policy, &pp->pick_args,
-                                 true /* force_async */, pp->target,
-                                 &pp->wrapped_on_complete_arg);
+    pick_from_internal_rr_locked(glb_policy, true /* force_async */, pp);
   }
-
+  // Send pending pings to RR policy.
   pending_ping* pping;
   while ((pping = glb_policy->pending_pings)) {
     glb_policy->pending_pings = pping->next;
-    grpc_closure* on_initiate = nullptr;
-    grpc_closure* on_ack = nullptr;
-    if (pping->on_initiate != nullptr) {
-      GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_ping");
-      pping->on_initiate->rr_policy = glb_policy->rr_policy;
-      on_initiate = &pping->on_initiate->wrapper_closure;
-    }
-    if (pping->on_ack != nullptr) {
-      GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_ping");
-      pping->on_ack->rr_policy = glb_policy->rr_policy;
-      on_ack = &pping->on_ack->wrapper_closure;
-    }
     if (grpc_lb_glb_trace.enabled()) {
       gpr_log(GPR_INFO, "[grpclb %p] Pending ping about to PING from RR %p",
               glb_policy, glb_policy->rr_policy);
     }
-    grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, on_initiate, on_ack);
+    grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, pping->on_initiate,
+                                   pping->on_ack);
     gpr_free(pping);
   }
 }
@@ -875,31 +756,28 @@ static void rr_handover_locked(glb_lb_policy* glb_policy) {
   lb_policy_args_destroy(args);
 }
 
-static void glb_rr_connectivity_changed_locked(void* arg, grpc_error* error) {
-  rr_connectivity_data* rr_connectivity = (rr_connectivity_data*)arg;
-  glb_lb_policy* glb_policy = rr_connectivity->glb_policy;
+static void on_rr_connectivity_changed_locked(void* arg, grpc_error* error) {
+  glb_lb_policy* glb_policy = (glb_lb_policy*)arg;
   if (glb_policy->shutting_down) {
-    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
-    gpr_free(rr_connectivity);
+    GRPC_LB_POLICY_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
     return;
   }
-  if (rr_connectivity->state == GRPC_CHANNEL_SHUTDOWN) {
+  if (glb_policy->rr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
     /* An RR policy that has transitioned into the SHUTDOWN connectivity state
      * should not be considered for picks or updates: the SHUTDOWN state is a
      * sink, policies can't transition back from it. .*/
     GRPC_LB_POLICY_UNREF(glb_policy->rr_policy, "rr_connectivity_shutdown");
     glb_policy->rr_policy = nullptr;
-    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
-    gpr_free(rr_connectivity);
+    GRPC_LB_POLICY_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
     return;
   }
   /* rr state != SHUTDOWN && !glb_policy->shutting down: biz as usual */
-  update_lb_connectivity_status_locked(glb_policy, rr_connectivity->state,
-                                       GRPC_ERROR_REF(error));
-  /* Resubscribe. Reuse the "glb_rr_connectivity_cb" weak ref. */
-  grpc_lb_policy_notify_on_state_change_locked(glb_policy->rr_policy,
-                                               &rr_connectivity->state,
-                                               &rr_connectivity->on_change);
+  update_lb_connectivity_status_locked(
+      glb_policy, glb_policy->rr_connectivity_state, GRPC_ERROR_REF(error));
+  /* Resubscribe. Reuse the "glb_rr_connectivity_cb" ref. */
+  grpc_lb_policy_notify_on_state_change_locked(
+      glb_policy->rr_policy, &glb_policy->rr_connectivity_state,
+      &glb_policy->on_rr_connectivity_changed);
 }
 
 static void destroy_balancer_name(void* balancer_name) {
@@ -1007,22 +885,17 @@ static void glb_destroy(grpc_lb_policy* pol) {
   gpr_free(glb_policy);
 }
 
-static void glb_shutdown_locked(grpc_lb_policy* pol) {
+static void glb_shutdown_locked(grpc_lb_policy* pol,
+                                grpc_lb_policy* new_policy) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)pol;
   grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   glb_policy->shutting_down = true;
-
-  /* We need a copy of the lb_call pointer because we can't cancell the call
-   * while holding glb_policy->mu: lb_on_server_status_received, invoked due to
-   * the cancel, needs to acquire that same lock */
-  grpc_call* lb_call = glb_policy->lb_call;
-
   /* glb_policy->lb_call and this local lb_call must be consistent at this point
    * because glb_policy->lb_call is only assigned in lb_call_init_locked as part
    * of query_for_backends_locked, which can only be invoked while
    * glb_policy->shutting_down is false. */
-  if (lb_call != nullptr) {
-    grpc_call_cancel(lb_call, nullptr);
+  if (glb_policy->lb_call != nullptr) {
+    grpc_call_cancel(glb_policy->lb_call, nullptr);
     /* lb_on_server_status_received will pick up the cancel and clean up */
   }
   if (glb_policy->retry_timer_callback_pending) {
@@ -1031,12 +904,8 @@ static void glb_shutdown_locked(grpc_lb_policy* pol) {
   if (glb_policy->fallback_timer_callback_pending) {
     grpc_timer_cancel(&glb_policy->lb_fallback_timer);
   }
-
-  pending_pick* pp = glb_policy->pending_picks;
-  glb_policy->pending_picks = nullptr;
-  pending_ping* pping = glb_policy->pending_pings;
-  glb_policy->pending_pings = nullptr;
   if (glb_policy->rr_policy != nullptr) {
+    grpc_lb_policy_shutdown_locked(glb_policy->rr_policy, nullptr);
     GRPC_LB_POLICY_UNREF(glb_policy->rr_policy, "glb_shutdown");
   } else {
     grpc_lb_policy_try_reresolve(pol, &grpc_lb_glb_trace, GRPC_ERROR_CANCELLED);
@@ -1051,28 +920,33 @@ static void glb_shutdown_locked(grpc_lb_policy* pol) {
   }
   grpc_connectivity_state_set(&glb_policy->state_tracker, GRPC_CHANNEL_SHUTDOWN,
                               GRPC_ERROR_REF(error), "glb_shutdown");
-
+  // Clear pending picks.
+  pending_pick* pp = glb_policy->pending_picks;
+  glb_policy->pending_picks = nullptr;
   while (pp != nullptr) {
     pending_pick* next = pp->next;
-    *pp->target = nullptr;
-    GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure,
-                       GRPC_ERROR_REF(error));
-    gpr_free(pp);
+    if (new_policy != nullptr) {
+      // Hand pick over to new policy.
+      grpc_grpclb_client_stats_unref(pp->client_stats);
+      pp->pick->on_complete = pp->original_on_complete;
+      if (grpc_lb_policy_pick_locked(new_policy, pp->pick)) {
+        // Synchronous return; schedule callback.
+        GRPC_CLOSURE_SCHED(pp->pick->on_complete, GRPC_ERROR_NONE);
+      }
+      gpr_free(pp);
+    } else {
+      pp->pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_REF(error));
+    }
     pp = next;
   }
-
+  // Clear pending pings.
+  pending_ping* pping = glb_policy->pending_pings;
+  glb_policy->pending_pings = nullptr;
   while (pping != nullptr) {
     pending_ping* next = pping->next;
-    if (pping->on_initiate != nullptr) {
-      GRPC_CLOSURE_SCHED(&pping->on_initiate->wrapper_closure,
-                         GRPC_ERROR_REF(error));
-      gpr_free(pping->on_initiate);
-    }
-    if (pping->on_ack != nullptr) {
-      GRPC_CLOSURE_SCHED(&pping->on_ack->wrapper_closure,
-                         GRPC_ERROR_REF(error));
-      gpr_free(pping->on_ack);
-    }
+    GRPC_CLOSURE_SCHED(pping->on_initiate, GRPC_ERROR_REF(error));
+    GRPC_CLOSURE_SCHED(pping->on_ack, GRPC_ERROR_REF(error));
     gpr_free(pping);
     pping = next;
   }
@@ -1090,16 +964,16 @@ static void glb_shutdown_locked(grpc_lb_policy* pol) {
 //   level (grpclb), inside the glb_policy->pending_picks list. To cancel these,
 //   we invoke the completion closure and set *target to nullptr right here.
 static void glb_cancel_pick_locked(grpc_lb_policy* pol,
-                                   grpc_connected_subchannel** target,
+                                   grpc_lb_policy_pick_state* pick,
                                    grpc_error* error) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)pol;
   pending_pick* pp = glb_policy->pending_picks;
   glb_policy->pending_picks = nullptr;
   while (pp != nullptr) {
     pending_pick* next = pp->next;
-    if (pp->target == target) {
-      *target = nullptr;
-      GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure,
+    if (pp->pick == pick) {
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(&pp->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
     } else {
@@ -1109,7 +983,7 @@ static void glb_cancel_pick_locked(grpc_lb_policy* pol,
     pp = next;
   }
   if (glb_policy->rr_policy != nullptr) {
-    grpc_lb_policy_cancel_pick_locked(glb_policy->rr_policy, target,
+    grpc_lb_policy_cancel_pick_locked(glb_policy->rr_policy, pick,
                                       GRPC_ERROR_REF(error));
   }
   GRPC_ERROR_UNREF(error);
@@ -1134,9 +1008,9 @@ static void glb_cancel_picks_locked(grpc_lb_policy* pol,
   glb_policy->pending_picks = nullptr;
   while (pp != nullptr) {
     pending_pick* next = pp->next;
-    if ((pp->pick_args.initial_metadata_flags & initial_metadata_flags_mask) ==
+    if ((pp->pick->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure,
+      GRPC_CLOSURE_SCHED(&pp->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
     } else {
@@ -1162,7 +1036,7 @@ static void start_picking_locked(glb_lb_policy* glb_policy) {
       !glb_policy->fallback_timer_callback_pending) {
     grpc_millis deadline =
         grpc_core::ExecCtx::Get()->Now() + glb_policy->lb_fallback_timeout_ms;
-    GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer");
+    GRPC_LB_POLICY_REF(&glb_policy->base, "grpclb_fallback_timer");
     GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked,
                       glb_policy,
                       grpc_combiner_scheduler(glb_policy->base.combiner));
@@ -1184,19 +1058,9 @@ static void glb_exit_idle_locked(grpc_lb_policy* pol) {
 }
 
 static int glb_pick_locked(grpc_lb_policy* pol,
-                           const grpc_lb_policy_pick_args* pick_args,
-                           grpc_connected_subchannel** target,
-                           grpc_call_context_element* context, void** user_data,
-                           grpc_closure* on_complete) {
-  if (pick_args->lb_token_mdelem_storage == nullptr) {
-    *target = nullptr;
-    GRPC_CLOSURE_SCHED(on_complete,
-                       GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                           "No mdelem storage for the LB token. Load reporting "
-                           "won't work without it. Failing"));
-    return 0;
-  }
+                           grpc_lb_policy_pick_state* pick) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)pol;
+  pending_pick* pp = pending_pick_create(glb_policy, pick);
   bool pick_done = false;
   if (glb_policy->rr_policy != nullptr) {
     const grpc_connectivity_state rr_connectivity_state =
@@ -1204,7 +1068,7 @@ static int glb_pick_locked(grpc_lb_policy* pol,
                                                  nullptr);
     // The glb_policy->rr_policy may have transitioned to SHUTDOWN but the
     // callback registered to capture this event
-    // (glb_rr_connectivity_changed_locked) may not have been invoked yet. We
+    // (on_rr_connectivity_changed_locked) may not have been invoked yet. We
     // need to make sure we aren't trying to pick from a RR policy instance
     // that's in shutdown.
     if (rr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
@@ -1214,32 +1078,16 @@ static int glb_pick_locked(grpc_lb_policy* pol,
                 glb_policy, glb_policy->rr_policy,
                 grpc_connectivity_state_name(rr_connectivity_state));
       }
-      add_pending_pick(&glb_policy->pending_picks, pick_args, target, context,
-                       on_complete);
+      pending_pick_add(&glb_policy->pending_picks, pp);
       pick_done = false;
     } else {  // RR not in shutdown
       if (grpc_lb_glb_trace.enabled()) {
         gpr_log(GPR_INFO, "[grpclb %p] about to PICK from RR %p", glb_policy,
                 glb_policy->rr_policy);
       }
-      GRPC_LB_POLICY_REF(glb_policy->rr_policy, "glb_pick");
-      wrapped_rr_closure_arg* wc_arg =
-          (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(wrapped_rr_closure_arg));
-      GRPC_CLOSURE_INIT(&wc_arg->wrapper_closure, wrapped_rr_closure, wc_arg,
-                        grpc_schedule_on_exec_ctx);
-      wc_arg->rr_policy = glb_policy->rr_policy;
-      wc_arg->target = target;
-      wc_arg->context = context;
       GPR_ASSERT(glb_policy->client_stats != nullptr);
-      wc_arg->client_stats =
-          grpc_grpclb_client_stats_ref(glb_policy->client_stats);
-      wc_arg->wrapped_closure = on_complete;
-      wc_arg->lb_token_mdelem_storage = pick_args->lb_token_mdelem_storage;
-      wc_arg->initial_metadata = pick_args->initial_metadata;
-      wc_arg->free_when_done = wc_arg;
-      wc_arg->glb_policy = pol;
-      pick_done = pick_from_internal_rr_locked(
-          glb_policy, pick_args, false /* force_async */, target, wc_arg);
+      pick_done =
+          pick_from_internal_rr_locked(glb_policy, false /* force_async */, pp);
     }
   } else {  // glb_policy->rr_policy == NULL
     if (grpc_lb_glb_trace.enabled()) {
@@ -1247,8 +1095,7 @@ static int glb_pick_locked(grpc_lb_policy* pol,
               "[grpclb %p] No RR policy. Adding to grpclb's pending picks",
               glb_policy);
     }
-    add_pending_pick(&glb_policy->pending_picks, pick_args, target, context,
-                     on_complete);
+    pending_pick_add(&glb_policy->pending_picks, pp);
     if (!glb_policy->started_picking) {
       start_picking_locked(glb_policy);
     }
@@ -1270,7 +1117,7 @@ static void glb_ping_one_locked(grpc_lb_policy* pol, grpc_closure* on_initiate,
   if (glb_policy->rr_policy) {
     grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, on_initiate, on_ack);
   } else {
-    add_pending_ping(&glb_policy->pending_pings, on_initiate, on_ack);
+    pending_ping_add(&glb_policy->pending_pings, on_initiate, on_ack);
     if (!glb_policy->started_picking) {
       start_picking_locked(glb_policy);
     }
@@ -1295,7 +1142,7 @@ static void lb_call_on_retry_timer_locked(void* arg, grpc_error* error) {
     }
     query_for_backends_locked(glb_policy);
   }
-  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "grpclb_retry_timer");
+  GRPC_LB_POLICY_UNREF(&glb_policy->base, "grpclb_retry_timer");
 }
 
 static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
@@ -1321,7 +1168,7 @@ static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
                 glb_policy);
       }
     }
-    GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_retry_timer");
+    GRPC_LB_POLICY_REF(&glb_policy->base, "grpclb_retry_timer");
     GRPC_CLOSURE_INIT(&glb_policy->lb_on_call_retry,
                       lb_call_on_retry_timer_locked, glb_policy,
                       grpc_combiner_scheduler(glb_policy->base.combiner));
@@ -1329,8 +1176,8 @@ static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
     grpc_timer_init(&glb_policy->lb_call_retry_timer, next_try,
                     &glb_policy->lb_on_call_retry);
   }
-  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
-                            "lb_on_server_status_received_locked");
+  GRPC_LB_POLICY_UNREF(&glb_policy->base,
+                       "lb_on_server_status_received_locked");
 }
 
 static void send_client_load_report_locked(void* arg, grpc_error* error);
@@ -1353,7 +1200,7 @@ static void client_load_report_done_locked(void* arg, grpc_error* error) {
   glb_policy->client_load_report_payload = nullptr;
   if (error != GRPC_ERROR_NONE || glb_policy->lb_call == nullptr) {
     glb_policy->client_load_report_timer_callback_pending = false;
-    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "client_load_report");
+    GRPC_LB_POLICY_UNREF(&glb_policy->base, "client_load_report");
     if (glb_policy->lb_call == nullptr) {
       maybe_restart_lb_call(glb_policy);
     }
@@ -1394,7 +1241,7 @@ static void send_client_load_report_locked(void* arg, grpc_error* error) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)arg;
   if (error == GRPC_ERROR_CANCELLED || glb_policy->lb_call == nullptr) {
     glb_policy->client_load_report_timer_callback_pending = false;
-    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "client_load_report");
+    GRPC_LB_POLICY_UNREF(&glb_policy->base, "client_load_report");
     if (glb_policy->lb_call == nullptr) {
       maybe_restart_lb_call(glb_policy);
     }
@@ -1547,10 +1394,8 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) {
   op->flags = 0;
   op->reserved = nullptr;
   op++;
-  /* take a weak ref (won't prevent calling of \a glb_shutdown if the strong ref
-   * count goes to zero) to be unref'd in lb_on_sent_initial_request_locked() */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base,
-                          "lb_on_sent_initial_request_locked");
+  /* take a ref to be released in lb_on_sent_initial_request_locked() */
+  GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_sent_initial_request_locked");
   call_error = grpc_call_start_batch_and_execute(
       glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_sent_initial_request);
@@ -1566,10 +1411,8 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) {
   op->flags = 0;
   op->reserved = nullptr;
   op++;
-  /* take a weak ref (won't prevent calling of \a glb_shutdown if the strong ref
-   * count goes to zero) to be unref'd in lb_on_server_status_received_locked */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base,
-                          "lb_on_server_status_received_locked");
+  /* take a ref to be released in lb_on_server_status_received_locked() */
+  GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_server_status_received_locked");
   call_error = grpc_call_start_batch_and_execute(
       glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_server_status_received);
@@ -1581,9 +1424,8 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) {
   op->flags = 0;
   op->reserved = nullptr;
   op++;
-  /* take another weak ref to be unref'd/reused in
-   * lb_on_response_received_locked */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "lb_on_response_received_locked");
+  /* take a ref to be unref'd/reused in lb_on_response_received_locked() */
+  GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_response_received_locked");
   call_error = grpc_call_start_batch_and_execute(
       glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_response_received);
@@ -1598,8 +1440,7 @@ static void lb_on_sent_initial_request_locked(void* arg, grpc_error* error) {
   if (glb_policy->client_load_report_payload != nullptr) {
     do_send_client_load_report_locked(glb_policy);
   }
-  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
-                            "lb_on_sent_initial_request_locked");
+  GRPC_LB_POLICY_UNREF(&glb_policy->base, "lb_on_sent_initial_request_locked");
 }
 
 static void lb_on_response_received_locked(void* arg, grpc_error* error) {
@@ -1631,11 +1472,9 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) {
                   "client load reporting interval = %" PRIdPTR " milliseconds",
                   glb_policy, glb_policy->client_stats_report_interval);
         }
-        /* take a weak ref (won't prevent calling of \a glb_shutdown() if the
-         * strong ref count goes to zero) to be unref'd in
-         * send_client_load_report_locked() */
+        /* take a ref to be unref'd in send_client_load_report_locked() */
         glb_policy->client_load_report_timer_callback_pending = true;
-        GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "client_load_report");
+        GRPC_LB_POLICY_REF(&glb_policy->base, "client_load_report");
         schedule_next_client_load_report(glb_policy);
       } else if (grpc_lb_glb_trace.enabled()) {
         gpr_log(GPR_INFO,
@@ -1717,21 +1556,21 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) {
       op->flags = 0;
       op->reserved = nullptr;
       op++;
-      /* reuse the "lb_on_response_received_locked" weak ref taken in
+      /* reuse the "lb_on_response_received_locked" ref taken in
        * query_for_backends_locked() */
       const grpc_call_error call_error = grpc_call_start_batch_and_execute(
           glb_policy->lb_call, ops, (size_t)(op - ops),
           &glb_policy->lb_on_response_received); /* loop */
       GPR_ASSERT(GRPC_CALL_OK == call_error);
     } else {
-      GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
-                                "lb_on_response_received_locked_shutdown");
+      GRPC_LB_POLICY_UNREF(&glb_policy->base,
+                           "lb_on_response_received_locked_shutdown");
     }
   } else { /* empty payload: call cancelled. */
-           /* dispose of the "lb_on_response_received_locked" weak ref taken in
+           /* dispose of the "lb_on_response_received_locked" ref taken in
             * query_for_backends_locked() and reused in every reception loop */
-    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
-                              "lb_on_response_received_locked_empty_payload");
+    GRPC_LB_POLICY_UNREF(&glb_policy->base,
+                         "lb_on_response_received_locked_empty_payload");
   }
 }
 
@@ -1751,7 +1590,7 @@ static void lb_on_fallback_timer_locked(void* arg, grpc_error* error) {
       rr_handover_locked(glb_policy);
     }
   }
-  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "grpclb_fallback_timer");
+  GRPC_LB_POLICY_UNREF(&glb_policy->base, "grpclb_fallback_timer");
 }
 
 static void lb_on_server_status_received_locked(void* arg, grpc_error* error) {
@@ -1835,7 +1674,7 @@ static void glb_update_locked(grpc_lb_policy* policy,
         grpc_channel_get_channel_stack(glb_policy->lb_channel));
     GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter);
     glb_policy->watching_lb_channel = true;
-    GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "watch_lb_channel_connectivity");
+    GRPC_LB_POLICY_REF(&glb_policy->base, "watch_lb_channel_connectivity");
     grpc_client_channel_watch_connectivity_state(
         client_channel_elem,
         grpc_polling_entity_create_from_pollset_set(
@@ -1891,8 +1730,8 @@ static void glb_lb_channel_on_connectivity_changed_cb(void* arg,
     case GRPC_CHANNEL_SHUTDOWN:
     done:
       glb_policy->watching_lb_channel = false;
-      GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
-                                "watch_lb_channel_connectivity_cb_shutdown");
+      GRPC_LB_POLICY_UNREF(&glb_policy->base,
+                           "watch_lb_channel_connectivity_cb_shutdown");
       break;
   }
 }
diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
index 9ff40aa53c0..60385272cf5 100644
--- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
@@ -31,15 +31,6 @@
 
 grpc_core::TraceFlag grpc_lb_pick_first_trace(false, "pick_first");
 
-namespace {
-struct pending_pick {
-  struct pending_pick* next;
-  uint32_t initial_metadata_flags;
-  grpc_connected_subchannel** target;
-  grpc_closure* on_complete;
-};
-}  // namespace
-
 typedef struct {
   /** base policy: must be first */
   grpc_lb_policy base;
@@ -54,7 +45,7 @@ typedef struct {
   /** are we shut down? */
   bool shutdown;
   /** list of picks that are waiting on connectivity */
-  pending_pick* pending_picks;
+  grpc_lb_policy_pick_state* pending_picks;
   /** our connectivity state tracker */
   grpc_connectivity_state_tracker state_tracker;
 } pick_first_lb_policy;
@@ -72,19 +63,27 @@ static void pf_destroy(grpc_lb_policy* pol) {
   }
 }
 
-static void pf_shutdown_locked(grpc_lb_policy* pol) {
+static void pf_shutdown_locked(grpc_lb_policy* pol,
+                               grpc_lb_policy* new_policy) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
   grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   if (grpc_lb_pick_first_trace.enabled()) {
     gpr_log(GPR_DEBUG, "Pick First %p Shutting down", p);
   }
   p->shutdown = true;
-  pending_pick* pp;
-  while ((pp = p->pending_picks) != nullptr) {
-    p->pending_picks = pp->next;
-    *pp->target = nullptr;
-    GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_REF(error));
-    gpr_free(pp);
+  grpc_lb_policy_pick_state* pick;
+  while ((pick = p->pending_picks) != nullptr) {
+    p->pending_picks = pick->next;
+    if (new_policy != nullptr) {
+      // Hand off to new LB policy.
+      if (grpc_lb_policy_pick_locked(new_policy, pick)) {
+        // Synchronous return, schedule closure.
+        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
+      }
+    } else {
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error));
+    }
   }
   grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
                               GRPC_ERROR_REF(error), "shutdown");
@@ -104,19 +103,18 @@ static void pf_shutdown_locked(grpc_lb_policy* pol) {
 }
 
 static void pf_cancel_pick_locked(grpc_lb_policy* pol,
-                                  grpc_connected_subchannel** target,
+                                  grpc_lb_policy_pick_state* pick,
                                   grpc_error* error) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
-  pending_pick* pp = p->pending_picks;
+  grpc_lb_policy_pick_state* pp = p->pending_picks;
   p->pending_picks = nullptr;
   while (pp != nullptr) {
-    pending_pick* next = pp->next;
-    if (pp->target == target) {
-      *target = nullptr;
-      GRPC_CLOSURE_SCHED(pp->on_complete,
+    grpc_lb_policy_pick_state* next = pp->next;
+    if (pp == pick) {
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(pick->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
-      gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
       p->pending_picks = pp;
@@ -131,21 +129,20 @@ static void pf_cancel_picks_locked(grpc_lb_policy* pol,
                                    uint32_t initial_metadata_flags_eq,
                                    grpc_error* error) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
-  pending_pick* pp = p->pending_picks;
+  grpc_lb_policy_pick_state* pick = p->pending_picks;
   p->pending_picks = nullptr;
-  while (pp != nullptr) {
-    pending_pick* next = pp->next;
-    if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
+  while (pick != nullptr) {
+    grpc_lb_policy_pick_state* next = pick->next;
+    if ((pick->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      GRPC_CLOSURE_SCHED(pp->on_complete,
+      GRPC_CLOSURE_SCHED(pick->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
-      gpr_free(pp);
     } else {
-      pp->next = p->pending_picks;
-      p->pending_picks = pp;
+      pick->next = p->pending_picks;
+      p->pending_picks = pick;
     }
-    pp = next;
+    pick = next;
   }
   GRPC_ERROR_UNREF(error);
 }
@@ -175,27 +172,20 @@ static void pf_exit_idle_locked(grpc_lb_policy* pol) {
 }
 
 static int pf_pick_locked(grpc_lb_policy* pol,
-                          const grpc_lb_policy_pick_args* pick_args,
-                          grpc_connected_subchannel** target,
-                          grpc_call_context_element* context, void** user_data,
-                          grpc_closure* on_complete) {
+                          grpc_lb_policy_pick_state* pick) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
   // If we have a selected subchannel already, return synchronously.
   if (p->selected != nullptr) {
-    *target = GRPC_CONNECTED_SUBCHANNEL_REF(p->selected->connected_subchannel,
-                                            "picked");
+    pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
+        p->selected->connected_subchannel, "picked");
     return 1;
   }
   // No subchannel selected yet, so handle asynchronously.
   if (!p->started_picking) {
     start_picking_locked(p);
   }
-  pending_pick* pp = (pending_pick*)gpr_malloc(sizeof(*pp));
-  pp->next = p->pending_picks;
-  pp->target = target;
-  pp->initial_metadata_flags = pick_args->initial_metadata_flags;
-  pp->on_complete = on_complete;
-  p->pending_picks = pp;
+  pick->next = p->pending_picks;
+  p->pending_picks = pick;
   return 0;
 }
 
@@ -481,18 +471,17 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
       // Drop all other subchannels, since we are now connected.
       destroy_unselected_subchannels_locked(p);
       // Update any calls that were waiting for a pick.
-      pending_pick* pp;
-      while ((pp = p->pending_picks)) {
-        p->pending_picks = pp->next;
-        *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF(
+      grpc_lb_policy_pick_state* pick;
+      while ((pick = p->pending_picks)) {
+        p->pending_picks = pick->next;
+        pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
             p->selected->connected_subchannel, "picked");
         if (grpc_lb_pick_first_trace.enabled()) {
           gpr_log(GPR_INFO,
                   "Servicing pending pick with selected subchannel %p",
                   (void*)p->selected);
         }
-        GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_NONE);
-        gpr_free(pp);
+        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
       }
       // Renew notification.
       grpc_lb_subchannel_data_start_connectivity_watch(sd);
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
index a964af06270..92c7d5bd5d0 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
@@ -41,31 +41,6 @@
 
 grpc_core::TraceFlag grpc_lb_round_robin_trace(false, "round_robin");
 
-namespace {
-/** List of entities waiting for a pick.
- *
- * Once a pick is available, \a target is updated and \a on_complete called. */
-struct pending_pick {
-  pending_pick* next;
-
-  /* output argument where to store the pick()ed user_data. It'll be NULL if no
-   * such data is present or there's an error (the definite test for errors is
-   * \a target being NULL). */
-  void** user_data;
-
-  /* bitmask passed to pick() and used for selective cancelling. See
-   * grpc_lb_policy_cancel_picks() */
-  uint32_t initial_metadata_flags;
-
-  /* output argument where to store the pick()ed connected subchannel, or NULL
-   * upon error. */
-  grpc_connected_subchannel** target;
-
-  /* to be invoked once the pick() has completed (regardless of success) */
-  grpc_closure* on_complete;
-};
-}  // namespace
-
 typedef struct round_robin_lb_policy {
   /** base policy: must be first */
   grpc_lb_policy base;
@@ -77,7 +52,7 @@ typedef struct round_robin_lb_policy {
   /** are we shutting down? */
   bool shutdown;
   /** List of picks that are waiting on connectivity */
-  pending_pick* pending_picks;
+  grpc_lb_policy_pick_state* pending_picks;
 
   /** our connectivity state tracker */
   grpc_connectivity_state_tracker state_tracker;
@@ -169,19 +144,27 @@ static void rr_destroy(grpc_lb_policy* pol) {
   gpr_free(p);
 }
 
-static void rr_shutdown_locked(grpc_lb_policy* pol) {
+static void rr_shutdown_locked(grpc_lb_policy* pol,
+                               grpc_lb_policy* new_policy) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
   grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   if (grpc_lb_round_robin_trace.enabled()) {
     gpr_log(GPR_DEBUG, "[RR %p] Shutting down", p);
   }
   p->shutdown = true;
-  pending_pick* pp;
-  while ((pp = p->pending_picks) != nullptr) {
-    p->pending_picks = pp->next;
-    *pp->target = nullptr;
-    GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_REF(error));
-    gpr_free(pp);
+  grpc_lb_policy_pick_state* pick;
+  while ((pick = p->pending_picks) != nullptr) {
+    p->pending_picks = pick->next;
+    if (new_policy != nullptr) {
+      // Hand off to new LB policy.
+      if (grpc_lb_policy_pick_locked(new_policy, pick)) {
+        // Synchronous return; schedule callback.
+        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
+      }
+    } else {
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error));
+    }
   }
   grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
                               GRPC_ERROR_REF(error), "rr_shutdown");
@@ -201,19 +184,18 @@ static void rr_shutdown_locked(grpc_lb_policy* pol) {
 }
 
 static void rr_cancel_pick_locked(grpc_lb_policy* pol,
-                                  grpc_connected_subchannel** target,
+                                  grpc_lb_policy_pick_state* pick,
                                   grpc_error* error) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
-  pending_pick* pp = p->pending_picks;
+  grpc_lb_policy_pick_state* pp = p->pending_picks;
   p->pending_picks = nullptr;
   while (pp != nullptr) {
-    pending_pick* next = pp->next;
-    if (pp->target == target) {
-      *target = nullptr;
-      GRPC_CLOSURE_SCHED(pp->on_complete,
+    grpc_lb_policy_pick_state* next = pp->next;
+    if (pp == pick) {
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(pick->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick cancelled", &error, 1));
-      gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
       p->pending_picks = pp;
@@ -228,22 +210,21 @@ static void rr_cancel_picks_locked(grpc_lb_policy* pol,
                                    uint32_t initial_metadata_flags_eq,
                                    grpc_error* error) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
-  pending_pick* pp = p->pending_picks;
+  grpc_lb_policy_pick_state* pick = p->pending_picks;
   p->pending_picks = nullptr;
-  while (pp != nullptr) {
-    pending_pick* next = pp->next;
-    if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
+  while (pick != nullptr) {
+    grpc_lb_policy_pick_state* next = pick->next;
+    if ((pick->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      *pp->target = nullptr;
-      GRPC_CLOSURE_SCHED(pp->on_complete,
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(pick->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick cancelled", &error, 1));
-      gpr_free(pp);
     } else {
-      pp->next = p->pending_picks;
-      p->pending_picks = pp;
+      pick->next = p->pending_picks;
+      p->pending_picks = pick;
     }
-    pp = next;
+    pick = next;
   }
   GRPC_ERROR_UNREF(error);
 }
@@ -268,13 +249,10 @@ static void rr_exit_idle_locked(grpc_lb_policy* pol) {
 }
 
 static int rr_pick_locked(grpc_lb_policy* pol,
-                          const grpc_lb_policy_pick_args* pick_args,
-                          grpc_connected_subchannel** target,
-                          grpc_call_context_element* context, void** user_data,
-                          grpc_closure* on_complete) {
+                          grpc_lb_policy_pick_state* pick) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
   if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", (void*)pol,
+    gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", pol,
             p->shutdown);
   }
   GPR_ASSERT(!p->shutdown);
@@ -284,18 +262,18 @@ static int rr_pick_locked(grpc_lb_policy* pol,
       /* readily available, report right away */
       grpc_lb_subchannel_data* sd =
           &p->subchannel_list->subchannels[next_ready_index];
-      *target =
+      pick->connected_subchannel =
           GRPC_CONNECTED_SUBCHANNEL_REF(sd->connected_subchannel, "rr_picked");
-      if (user_data != nullptr) {
-        *user_data = sd->user_data;
+      if (pick->user_data != nullptr) {
+        *pick->user_data = sd->user_data;
       }
       if (grpc_lb_round_robin_trace.enabled()) {
         gpr_log(
             GPR_DEBUG,
             "[RR %p] Picked target <-- Subchannel %p (connected %p) (sl %p, "
-            "index %lu)",
-            (void*)p, (void*)sd->subchannel, (void*)*target,
-            (void*)sd->subchannel_list, (unsigned long)next_ready_index);
+            "index %" PRIuPTR ")",
+            p, sd->subchannel, pick->connected_subchannel, sd->subchannel_list,
+            next_ready_index);
       }
       /* only advance the last picked pointer if the selection was used */
       update_last_ready_subchannel_index_locked(p, next_ready_index);
@@ -306,13 +284,8 @@ static int rr_pick_locked(grpc_lb_policy* pol,
   if (!p->started_picking) {
     start_picking_locked(p);
   }
-  pending_pick* pp = (pending_pick*)gpr_malloc(sizeof(*pp));
-  pp->next = p->pending_picks;
-  pp->target = target;
-  pp->on_complete = on_complete;
-  pp->initial_metadata_flags = pick_args->initial_metadata_flags;
-  pp->user_data = user_data;
-  p->pending_picks = pp;
+  pick->next = p->pending_picks;
+  p->pending_picks = pick;
   return 0;
 }
 
@@ -495,13 +468,13 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
         // picks, update the last picked pointer
         update_last_ready_subchannel_index_locked(p, next_ready_index);
       }
-      pending_pick* pp;
-      while ((pp = p->pending_picks)) {
-        p->pending_picks = pp->next;
-        *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF(
+      grpc_lb_policy_pick_state* pick;
+      while ((pick = p->pending_picks)) {
+        p->pending_picks = pick->next;
+        pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
             selected->connected_subchannel, "rr_picked");
-        if (pp->user_data != nullptr) {
-          *pp->user_data = selected->user_data;
+        if (pick->user_data != nullptr) {
+          *pick->user_data = selected->user_data;
         }
         if (grpc_lb_round_robin_trace.enabled()) {
           gpr_log(GPR_DEBUG,
@@ -510,8 +483,7 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
                   (void*)p, (void*)selected->subchannel,
                   (void*)p->subchannel_list, (unsigned long)next_ready_index);
         }
-        GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_NONE);
-        gpr_free(pp);
+        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
       }
     }
     // Renew notification.
diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
index a3b4c8e524d..5ce1298afc4 100644
--- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
@@ -213,13 +213,13 @@ void grpc_lb_subchannel_list_unref(grpc_lb_subchannel_list* subchannel_list,
 
 void grpc_lb_subchannel_list_ref_for_connectivity_watch(
     grpc_lb_subchannel_list* subchannel_list, const char* reason) {
-  GRPC_LB_POLICY_WEAK_REF(subchannel_list->policy, reason);
+  GRPC_LB_POLICY_REF(subchannel_list->policy, reason);
   grpc_lb_subchannel_list_ref(subchannel_list, reason);
 }
 
 void grpc_lb_subchannel_list_unref_for_connectivity_watch(
     grpc_lb_subchannel_list* subchannel_list, const char* reason) {
-  GRPC_LB_POLICY_WEAK_UNREF(subchannel_list->policy, reason);
+  GRPC_LB_POLICY_UNREF(subchannel_list->policy, reason);
   grpc_lb_subchannel_list_unref(subchannel_list, reason);
 }
 

From 75005775938c8844d42946f92b052fd1be79a0a9 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Tue, 9 Jan 2018 10:55:37 -0800
Subject: [PATCH 055/161] Address review feedback; stop using result of 'what'

---
 include/grpc++/impl/codegen/method_handler_impl.h | 4 +---
 test/cpp/end2end/exception_test.cc                | 1 -
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h
index ed6c146e6c2..daf090f86ca 100644
--- a/include/grpc++/impl/codegen/method_handler_impl.h
+++ b/include/grpc++/impl/codegen/method_handler_impl.h
@@ -40,10 +40,8 @@ Status CatchingFunctionHandler(Callable&& handler) {
 #if GRPC_ALLOW_EXCEPTIONS
   try {
     return handler();
-  } catch (const std::exception& e) {
-    return Status(StatusCode::UNKNOWN, e.what());
   } catch (...) {
-    return Status(StatusCode::UNKNOWN, "Exception in method handler");
+    return Status(StatusCode::UNKNOWN, "Unexpected error in RPC handling");
   }
 #else   // GRPC_ALLOW_EXCEPTIONS
   return handler();
diff --git a/test/cpp/end2end/exception_test.cc b/test/cpp/end2end/exception_test.cc
index 722276d149f..76272ad08a6 100644
--- a/test/cpp/end2end/exception_test.cc
+++ b/test/cpp/end2end/exception_test.cc
@@ -105,7 +105,6 @@ TEST_F(ExceptionTest, RequestStream) {
 
   EXPECT_FALSE(s.ok());
   EXPECT_EQ(s.error_code(), StatusCode::UNKNOWN);
-  EXPECT_EQ(s.error_message(), kErrorMessage);
 }
 
 #endif  // GRPC_ALLOW_EXCEPTIONS

From baf1ac7af91eab2da6024b05ddb83720d9644b94 Mon Sep 17 00:00:00 2001
From: David Garcia Quintas <dgq@google.com>
Date: Tue, 9 Jan 2018 14:24:32 -0800
Subject: [PATCH 056/161] PR comments

---
 .../filters/client_channel/client_channel.cc  |   6 +-
 .../ext/filters/client_channel/lb_policy.cc   |   4 +-
 .../ext/filters/client_channel/lb_policy.h    |  11 +-
 .../client_channel/lb_policy/grpclb/grpclb.cc |  12 +-
 .../lb_policy/pick_first/pick_first.cc        |  84 +++++--------
 .../lb_policy/round_robin/round_robin.cc      |  50 ++++----
 .../lb_policy/subchannel_list.h               |   2 +-
 .../ext/filters/client_channel/subchannel.cc  | 115 +++++++++---------
 .../ext/filters/client_channel/subchannel.h   |  35 +++---
 src/core/lib/support/ref_counted.h            |   3 +
 10 files changed, 149 insertions(+), 173 deletions(-)

diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc
index 4f3b774212f..fce5f3582b6 100644
--- a/src/core/ext/filters/client_channel/client_channel.cc
+++ b/src/core/ext/filters/client_channel/client_channel.cc
@@ -856,7 +856,7 @@ typedef struct client_channel_call_data {
   grpc_closure lb_pick_closure;
   grpc_closure lb_pick_cancel_closure;
 
-  grpc_connected_subchannel* connected_subchannel;
+  grpc_core::ConnectedSubchannel* connected_subchannel;
   grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
   grpc_polling_entity* pollent;
 
@@ -1004,7 +1004,7 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
                                           grpc_error* error) {
   channel_data* chand = (channel_data*)elem->channel_data;
   call_data* calld = (call_data*)elem->call_data;
-  const grpc_connected_subchannel::CallArgs call_args = {
+  const grpc_core::ConnectedSubchannel::CallArgs call_args = {
       calld->pollent,                  // pollent
       calld->path,                     // path
       calld->call_start_time,          // start_time
@@ -1014,7 +1014,7 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
       calld->call_combiner             // call_combiner
   };
   grpc_error* new_error = calld->connected_subchannel->CreateCall(
-      &call_args, &calld->subchannel_call);
+      call_args, &calld->subchannel_call);
   if (grpc_client_channel_trace.enabled()) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: create subchannel_call=%p: error=%s",
             chand, calld, calld->subchannel_call, grpc_error_string(new_error));
diff --git a/src/core/ext/filters/client_channel/lb_policy.cc b/src/core/ext/filters/client_channel/lb_policy.cc
index 7a5a8dec34b..ebaeaadfc58 100644
--- a/src/core/ext/filters/client_channel/lb_policy.cc
+++ b/src/core/ext/filters/client_channel/lb_policy.cc
@@ -102,7 +102,7 @@ void grpc_lb_policy_weak_unref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
 
 int grpc_lb_policy_pick_locked(grpc_lb_policy* policy,
                                const grpc_lb_policy_pick_args* pick_args,
-                               grpc_connected_subchannel** target,
+                               grpc_core::ConnectedSubchannel** target,
                                grpc_call_context_element* context,
                                void** user_data, grpc_closure* on_complete) {
   return policy->vtable->pick_locked(policy, pick_args, target, context,
@@ -110,7 +110,7 @@ int grpc_lb_policy_pick_locked(grpc_lb_policy* policy,
 }
 
 void grpc_lb_policy_cancel_pick_locked(grpc_lb_policy* policy,
-                                       grpc_connected_subchannel** target,
+                                       grpc_core::ConnectedSubchannel** target,
                                        grpc_error* error) {
   policy->vtable->cancel_pick_locked(policy, target, error);
 }
diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h
index 628127106b1..967253418e8 100644
--- a/src/core/ext/filters/client_channel/lb_policy.h
+++ b/src/core/ext/filters/client_channel/lb_policy.h
@@ -61,13 +61,13 @@ struct grpc_lb_policy_vtable {
   /** \see grpc_lb_policy_pick */
   int (*pick_locked)(grpc_lb_policy* policy,
                      const grpc_lb_policy_pick_args* pick_args,
-                     grpc_connected_subchannel** target,
+                     grpc_core::ConnectedSubchannel** target,
                      grpc_call_context_element* context, void** user_data,
                      grpc_closure* on_complete);
 
   /** \see grpc_lb_policy_cancel_pick */
   void (*cancel_pick_locked)(grpc_lb_policy* policy,
-                             grpc_connected_subchannel** target,
+                             grpc_core::ConnectedSubchannel** target,
                              grpc_error* error);
 
   /** \see grpc_lb_policy_cancel_picks */
@@ -160,11 +160,12 @@ void grpc_lb_policy_init(grpc_lb_policy* policy,
     in the \a grpc_lb_policy struct. */
 int grpc_lb_policy_pick_locked(grpc_lb_policy* policy,
                                const grpc_lb_policy_pick_args* pick_args,
-                               grpc_connected_subchannel** target,
+                               grpc_core::ConnectedSubchannel** target,
                                grpc_call_context_element* context,
                                void** user_data, grpc_closure* on_complete);
 
-/** Perform a connected subchannel ping (see \a grpc_connected_subchannel::Ping)
+/** Perform a connected subchannel ping (see \a
+   grpc_core::ConnectedSubchannel::Ping)
     against one of the connected subchannels managed by \a policy. */
 void grpc_lb_policy_ping_one_locked(grpc_lb_policy* policy,
                                     grpc_closure* on_initiate,
@@ -174,7 +175,7 @@ void grpc_lb_policy_ping_one_locked(grpc_lb_policy* policy,
     The \a on_complete callback of the pending picks will be invoked with \a
     *target set to NULL. */
 void grpc_lb_policy_cancel_pick_locked(grpc_lb_policy* policy,
-                                       grpc_connected_subchannel** target,
+                                       grpc_core::ConnectedSubchannel** target,
                                        grpc_error* error);
 
 /** Cancel all pending picks for which their \a initial_metadata_flags (as given
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index ba4e90d4c21..ebc7fdac4c8 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -157,7 +157,7 @@ typedef struct wrapped_rr_closure_arg {
 
   /* the picked target, used to determine which LB token to add to the pick's
    * initial metadata */
-  grpc_connected_subchannel** target;
+  grpc_core::ConnectedSubchannel** target;
 
   /* the context to be populated for the subchannel call */
   grpc_call_context_element* context;
@@ -242,7 +242,7 @@ typedef struct pending_pick {
 
   /* output argument where to store the pick()ed connected subchannel, or
    * nullptr upon error. */
-  grpc_connected_subchannel** target;
+  grpc_core::ConnectedSubchannel** target;
 
   /* args for wrapped_on_complete */
   wrapped_rr_closure_arg wrapped_on_complete_arg;
@@ -250,7 +250,7 @@ typedef struct pending_pick {
 
 static void add_pending_pick(pending_pick** root,
                              const grpc_lb_policy_pick_args* pick_args,
-                             grpc_connected_subchannel** target,
+                             grpc_core::ConnectedSubchannel** target,
                              grpc_call_context_element* context,
                              grpc_closure* on_complete) {
   pending_pick* pp = (pending_pick*)gpr_zalloc(sizeof(*pp));
@@ -657,7 +657,7 @@ static void update_lb_connectivity_status_locked(
  * completion callback even if the pick is available immediately. */
 static bool pick_from_internal_rr_locked(
     glb_lb_policy* glb_policy, const grpc_lb_policy_pick_args* pick_args,
-    bool force_async, grpc_connected_subchannel** target,
+    bool force_async, grpc_core::ConnectedSubchannel** target,
     wrapped_rr_closure_arg* wc_arg) {
   // Check for drops if we are not using fallback backend addresses.
   if (glb_policy->serverlist != nullptr) {
@@ -1090,7 +1090,7 @@ static void glb_shutdown_locked(grpc_lb_policy* pol) {
 //   level (grpclb), inside the glb_policy->pending_picks list. To cancel these,
 //   we invoke the completion closure and set *target to nullptr right here.
 static void glb_cancel_pick_locked(grpc_lb_policy* pol,
-                                   grpc_connected_subchannel** target,
+                                   grpc_core::ConnectedSubchannel** target,
                                    grpc_error* error) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)pol;
   pending_pick* pp = glb_policy->pending_picks;
@@ -1184,7 +1184,7 @@ static void glb_exit_idle_locked(grpc_lb_policy* pol) {
 
 static int glb_pick_locked(grpc_lb_policy* pol,
                            const grpc_lb_policy_pick_args* pick_args,
-                           grpc_connected_subchannel** target,
+                           grpc_core::ConnectedSubchannel** target,
                            grpc_call_context_element* context, void** user_data,
                            grpc_closure* on_complete) {
   if (pick_args->lb_token_mdelem_storage == nullptr) {
diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
index a3b05aacaf6..e70f2a8c52b 100644
--- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
@@ -34,7 +34,7 @@ grpc_core::TraceFlag grpc_lb_pick_first_trace(false, "pick_first");
 typedef struct pending_pick {
   struct pending_pick* next;
   uint32_t initial_metadata_flags;
-  grpc_connected_subchannel** target;
+  grpc_core::ConnectedSubchannel** target;
   grpc_closure* on_complete;
 } pending_pick;
 
@@ -102,7 +102,7 @@ static void pf_shutdown_locked(grpc_lb_policy* pol) {
 }
 
 static void pf_cancel_pick_locked(grpc_lb_policy* pol,
-                                  grpc_connected_subchannel** target,
+                                  grpc_core::ConnectedSubchannel** target,
                                   grpc_error* error) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
   pending_pick* pp = p->pending_picks;
@@ -174,7 +174,7 @@ static void pf_exit_idle_locked(grpc_lb_policy* pol) {
 
 static int pf_pick_locked(grpc_lb_policy* pol,
                           const grpc_lb_policy_pick_args* pick_args,
-                          grpc_connected_subchannel** target,
+                          grpc_core::ConnectedSubchannel** target,
                           grpc_call_context_element* context, void** user_data,
                           grpc_closure* on_complete) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
@@ -396,6 +396,8 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
   sd->curr_connectivity_state = sd->pending_connectivity_state_unsafe;
   // Handle updates for the currently selected subchannel.
   if (p->selected == sd) {
+    gpr_log(GPR_INFO, "BAR selected. subchannel %p, conn subchannel %p",
+            sd->subchannel, p->selected->connected_subchannel);
     // If the new state is anything other than READY and there is a
     // pending update, switch to the pending update.
     if (sd->curr_connectivity_state != GRPC_CHANNEL_READY &&
@@ -412,25 +414,13 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
           &p->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
           GRPC_ERROR_REF(error), "selected_not_ready+switch_to_update");
     } else {
-      if (sd->curr_connectivity_state < GRPC_CHANNEL_TRANSIENT_FAILURE) {
-        // Renew notification.
-        grpc_lb_subchannel_data_start_connectivity_watch(sd);
-      } else {  // in transient failure or shutdown. Rely on re-resolution to
-                // recover.
-        p->selected = nullptr;
-        grpc_lb_subchannel_data_stop_connectivity_watch(sd);
-        grpc_lb_subchannel_list_unref_for_connectivity_watch(
-            sd->subchannel_list, "pf_selected_shutdown");
-        grpc_lb_subchannel_data_unref_subchannel(
-            sd, "pf_selected_shutdown");  // Unrefs connected subchannel
-      }
       // TODO(juanlishen): we re-resolve when the selected subchannel goes to
       // TRANSIENT_FAILURE because we used to shut down in this case before
       // re-resolution is introduced. But we need to investigate whether we
       // really want to take any action instead of waiting for the selected
       // subchannel reconnecting.
-      if (sd->curr_connectivity_state == GRPC_CHANNEL_SHUTDOWN ||
-          sd->curr_connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+      GPR_ASSERT(sd->curr_connectivity_state != GRPC_CHANNEL_SHUTDOWN);
+      if (sd->curr_connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
         // If the selected channel goes bad, request a re-resolution.
         grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_IDLE,
                                     GRPC_ERROR_NONE,
@@ -438,10 +428,20 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
         p->started_picking = false;
         grpc_lb_policy_try_reresolve(&p->base, &grpc_lb_pick_first_trace,
                                      GRPC_ERROR_NONE);
+        // in transient failure. Rely on re-resolution to recover.
+        p->selected = nullptr;
+        grpc_lb_subchannel_data_stop_connectivity_watch(sd);
+        grpc_lb_subchannel_list_unref_for_connectivity_watch(
+            sd->subchannel_list, "pf_selected_shutdown");
+        grpc_lb_subchannel_data_unref_subchannel(
+            sd, "pf_selected_shutdown");  // Unrefs connected subchannel
+
       } else {
         grpc_connectivity_state_set(&p->state_tracker,
                                     sd->curr_connectivity_state,
                                     GRPC_ERROR_REF(error), "selected_changed");
+        // Renew notification.
+        grpc_lb_subchannel_data_start_connectivity_watch(sd);
       }
     }
     return;
@@ -459,6 +459,16 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
     case GRPC_CHANNEL_READY: {
       // Case 2.  Promote p->latest_pending_subchannel_list to
       // p->subchannel_list.
+      grpc_core::ConnectedSubchannel* con =
+          grpc_subchannel_get_connected_subchannel(sd->subchannel);
+      if (con == nullptr) {
+        // The subchannel may have become disconnected by the time this callback
+        // is invoked. Simply ignore and resubscribe: ulterior connectivity
+        // states
+        // must be in the pipeline and will eventually be invoked.
+        grpc_lb_subchannel_data_start_connectivity_watch(sd);
+        break;
+      }
       if (sd->subchannel_list == p->latest_pending_subchannel_list) {
         GPR_ASSERT(p->subchannel_list != nullptr);
         grpc_lb_subchannel_list_shutdown_and_unref(p->subchannel_list,
@@ -469,9 +479,8 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
       // Cases 1 and 2.
       grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_READY,
                                   GRPC_ERROR_NONE, "connecting_ready");
-      sd->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
-          grpc_subchannel_get_connected_subchannel(sd->subchannel),
-          "connected");
+      sd->connected_subchannel =
+          GRPC_CONNECTED_SUBCHANNEL_REF(con, "connected");
       p->selected = sd;
       if (grpc_lb_pick_first_trace.enabled()) {
         gpr_log(GPR_INFO, "Pick First %p selected subchannel %p", (void*)p,
@@ -530,39 +539,8 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
       grpc_lb_subchannel_data_start_connectivity_watch(sd);
       break;
     }
-    case GRPC_CHANNEL_SHUTDOWN: {
-      grpc_lb_subchannel_data_stop_connectivity_watch(sd);
-      grpc_lb_subchannel_data_unref_subchannel(sd, "pf_candidate_shutdown");
-      // Advance to next subchannel and check its state.
-      grpc_lb_subchannel_data* original_sd = sd;
-      do {
-        sd->subchannel_list->checking_subchannel =
-            (sd->subchannel_list->checking_subchannel + 1) %
-            sd->subchannel_list->num_subchannels;
-        sd = &sd->subchannel_list
-                  ->subchannels[sd->subchannel_list->checking_subchannel];
-      } while (sd->subchannel == nullptr && sd != original_sd);
-      if (sd == original_sd) {
-        grpc_lb_subchannel_list_unref_for_connectivity_watch(
-            sd->subchannel_list, "pf_exhausted_subchannels");
-        if (sd->subchannel_list == p->subchannel_list) {
-          grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_IDLE,
-                                      GRPC_ERROR_NONE,
-                                      "exhausted_subchannels+reresolve");
-          p->started_picking = false;
-          grpc_lb_policy_try_reresolve(&p->base, &grpc_lb_pick_first_trace,
-                                       GRPC_ERROR_NONE);
-        }
-      } else {
-        if (sd->subchannel_list == p->subchannel_list) {
-          grpc_connectivity_state_set(
-              &p->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
-              GRPC_ERROR_REF(error), "subchannel_failed");
-        }
-        // Reuses the connectivity refs from the previous watch.
-        grpc_lb_subchannel_data_start_connectivity_watch(sd);
-      }
-    }
+    case GRPC_CHANNEL_SHUTDOWN:
+      GPR_UNREACHABLE_CODE(break);
   }
 }
 
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
index 0836dad2f6b..a6a8fbb3cfe 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
@@ -58,7 +58,7 @@ typedef struct pending_pick {
 
   /* output argument where to store the pick()ed connected subchannel, or NULL
    * upon error. */
-  grpc_connected_subchannel** target;
+  grpc_core::ConnectedSubchannel** target;
 
   /* to be invoked once the pick() has completed (regardless of success) */
   grpc_closure* on_complete;
@@ -199,7 +199,7 @@ static void rr_shutdown_locked(grpc_lb_policy* pol) {
 }
 
 static void rr_cancel_pick_locked(grpc_lb_policy* pol,
-                                  grpc_connected_subchannel** target,
+                                  grpc_core::ConnectedSubchannel** target,
                                   grpc_error* error) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
   pending_pick* pp = p->pending_picks;
@@ -267,7 +267,7 @@ static void rr_exit_idle_locked(grpc_lb_policy* pol) {
 
 static int rr_pick_locked(grpc_lb_policy* pol,
                           const grpc_lb_policy_pick_args* pick_args,
-                          grpc_connected_subchannel** target,
+                          grpc_core::ConnectedSubchannel** target,
                           grpc_call_context_element* context, void** user_data,
                           grpc_closure* on_complete) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
@@ -316,15 +316,14 @@ static int rr_pick_locked(grpc_lb_policy* pol,
 
 static void update_state_counters_locked(grpc_lb_subchannel_data* sd) {
   grpc_lb_subchannel_list* subchannel_list = sd->subchannel_list;
+  GPR_ASSERT(sd->prev_connectivity_state != GRPC_CHANNEL_SHUTDOWN);
+  GPR_ASSERT(sd->curr_connectivity_state != GRPC_CHANNEL_SHUTDOWN);
   if (sd->prev_connectivity_state == GRPC_CHANNEL_READY) {
     GPR_ASSERT(subchannel_list->num_ready > 0);
     --subchannel_list->num_ready;
   } else if (sd->prev_connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
     GPR_ASSERT(subchannel_list->num_transient_failures > 0);
     --subchannel_list->num_transient_failures;
-  } else if (sd->prev_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
-    GPR_ASSERT(subchannel_list->num_shutdown > 0);
-    --subchannel_list->num_shutdown;
   } else if (sd->prev_connectivity_state == GRPC_CHANNEL_IDLE) {
     GPR_ASSERT(subchannel_list->num_idle > 0);
     --subchannel_list->num_idle;
@@ -334,8 +333,6 @@ static void update_state_counters_locked(grpc_lb_subchannel_data* sd) {
     ++subchannel_list->num_ready;
   } else if (sd->curr_connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
     ++subchannel_list->num_transient_failures;
-  } else if (sd->curr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
-    ++subchannel_list->num_shutdown;
   } else if (sd->curr_connectivity_state == GRPC_CHANNEL_IDLE) {
     ++subchannel_list->num_idle;
   }
@@ -401,6 +398,7 @@ static void update_lb_connectivity_status_locked(grpc_lb_subchannel_data* sd,
 
 static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
   grpc_lb_subchannel_data* sd = (grpc_lb_subchannel_data*)arg;
+  GPR_ASSERT(sd->curr_connectivity_state != GRPC_CHANNEL_SHUTDOWN);
   round_robin_lb_policy* p =
       (round_robin_lb_policy*)sd->subchannel_list->policy;
   if (grpc_lb_round_robin_trace.enabled()) {
@@ -444,23 +442,16 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
   update_lb_connectivity_status_locked(sd, GRPC_ERROR_REF(error));
   // If the sd's new state is TRANSIENT_FAILURE, unref the *connected*
   // subchannel, if any.
-  if (sd->curr_connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-    if (sd->connected_subchannel != nullptr) {
-      GRPC_CONNECTED_SUBCHANNEL_UNREF(sd->connected_subchannel,
-                                      "connected_subchannel_transient_failure");
-      sd->connected_subchannel = nullptr;
+  switch (sd->curr_connectivity_state) {
+    case GRPC_CHANNEL_TRANSIENT_FAILURE: {
+      if (sd->connected_subchannel != nullptr) {
+        GRPC_CONNECTED_SUBCHANNEL_UNREF(
+            sd->connected_subchannel, "connected_subchannel_transient_failure");
+        sd->connected_subchannel = nullptr;
+      }
+      break;
     }
-    // Renew notification.
-    grpc_lb_subchannel_data_start_connectivity_watch(sd);
-  }
-  // If the sd's new state is SHUTDOWN, unref the subchannel.
-  else if (sd->curr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
-    grpc_lb_subchannel_data_stop_connectivity_watch(sd);
-    grpc_lb_subchannel_data_unref_subchannel(sd, "rr_connectivity_shutdown");
-    grpc_lb_subchannel_list_unref_for_connectivity_watch(
-        sd->subchannel_list, "rr_connectivity_shutdown");
-  } else {  // sd not in SHUTDOWN
-    if (sd->curr_connectivity_state == GRPC_CHANNEL_READY) {
+    case GRPC_CHANNEL_READY: {
       if (sd->connected_subchannel == nullptr) {
         sd->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
             grpc_subchannel_get_connected_subchannel(sd->subchannel),
@@ -522,10 +513,15 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
         GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_NONE);
         gpr_free(pp);
       }
+      break;
     }
-    // Renew notification.
-    grpc_lb_subchannel_data_start_connectivity_watch(sd);
+    case GRPC_CHANNEL_SHUTDOWN:
+      GPR_UNREACHABLE_CODE();
+    case GRPC_CHANNEL_CONNECTING:
+    case GRPC_CHANNEL_IDLE:;  // fallthrough
   }
+  // Renew notification.
+  grpc_lb_subchannel_data_start_connectivity_watch(sd);
 }
 
 static grpc_connectivity_state rr_check_connectivity_locked(
@@ -549,7 +545,7 @@ static void rr_ping_one_locked(grpc_lb_policy* pol, grpc_closure* on_initiate,
   if (next_ready_index < p->subchannel_list->num_subchannels) {
     grpc_lb_subchannel_data* selected =
         &p->subchannel_list->subchannels[next_ready_index];
-    grpc_connected_subchannel* target = GRPC_CONNECTED_SUBCHANNEL_REF(
+    grpc_core::ConnectedSubchannel* target = GRPC_CONNECTED_SUBCHANNEL_REF(
         selected->connected_subchannel, "rr_ping");
     target->Ping(on_initiate, on_ack);
     GRPC_CONNECTED_SUBCHANNEL_UNREF(target, "rr_ping");
diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
index 0f8cea9347e..e4db3ef464b 100644
--- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
+++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
@@ -43,7 +43,7 @@ typedef struct {
   grpc_lb_subchannel_list* subchannel_list;
   /** subchannel itself */
   grpc_subchannel* subchannel;
-  grpc_connected_subchannel* connected_subchannel;
+  grpc_core::ConnectedSubchannel* connected_subchannel;
   /** Is a connectivity notification pending? */
   bool connectivity_notification_pending;
   /** notification that connectivity has changed on subchannel */
diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc
index 25615b6326f..2f1662e63bd 100644
--- a/src/core/ext/filters/client_channel/subchannel.cc
+++ b/src/core/ext/filters/client_channel/subchannel.cc
@@ -56,8 +56,8 @@
 #define GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS 120
 #define GRPC_SUBCHANNEL_RECONNECT_JITTER 0.2
 
-#define GET_CONNECTED_SUBCHANNEL(subchannel, barrier)     \
-  ((grpc_connected_subchannel*)(gpr_atm_##barrier##_load( \
+#define GET_CONNECTED_SUBCHANNEL(subchannel, barrier)          \
+  ((grpc_core::ConnectedSubchannel*)(gpr_atm_##barrier##_load( \
       &(subchannel)->connected_subchannel)))
 
 typedef struct {
@@ -106,7 +106,8 @@ struct grpc_subchannel {
       being setup */
   grpc_pollset_set* pollset_set;
 
-  /** active connection, or null; of type grpc_connected_subchannel */
+  /** active connection, or null; of type grpc_core::ConnectedSubchannel
+   */
   gpr_atm connected_subchannel;
 
   /** mutex protecting remaining elements */
@@ -135,7 +136,7 @@ struct grpc_subchannel {
 };
 
 struct grpc_subchannel_call {
-  grpc_connected_subchannel* connection;
+  grpc_core::ConnectedSubchannel* connection;
   grpc_closure* schedule_closure_after_destroy;
 };
 
@@ -166,14 +167,14 @@ static void connection_destroy(void* arg, grpc_error* error) {
   gpr_free(stk);
 }
 
-grpc_connected_subchannel* grpc_connected_subchannel_ref(
-    grpc_connected_subchannel* c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+grpc_core::ConnectedSubchannel* ConnectedSubchannel_ref(
+    grpc_core::ConnectedSubchannel* c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
   c->Ref(DEBUG_LOCATION, REF_REASON);
   return c;
 }
 
-void grpc_connected_subchannel_unref(
-    grpc_connected_subchannel* c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
+void ConnectedSubchannel_unref(
+    grpc_core::ConnectedSubchannel* c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
   c->Unref(DEBUG_LOCATION, REF_REASON);
 }
 
@@ -247,7 +248,7 @@ static void disconnect(grpc_subchannel* c) {
   c->disconnected = true;
   grpc_connector_shutdown(c->connector, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                                             "Subchannel disconnected"));
-  grpc_connected_subchannel* con = GET_CONNECTED_SUBCHANNEL(c, no_barrier);
+  grpc_core::ConnectedSubchannel* con = GET_CONNECTED_SUBCHANNEL(c, no_barrier);
   if (con != nullptr) {
     GRPC_CONNECTED_SUBCHANNEL_UNREF(con, "disconnect");
     gpr_atm_no_barrier_store(&c->connected_subchannel, (gpr_atm)0xdeadbeef);
@@ -535,11 +536,15 @@ static void on_connected_subchannel_connectivity_changed(void* p,
 
   auto* con = GET_CONNECTED_SUBCHANNEL(c, no_barrier);
   /* if we failed just leave this closure */
-  if (connected_subchannel_watcher->connectivity_state ==
+  if (connected_subchannel_watcher->connectivity_state >=
       GRPC_CHANNEL_TRANSIENT_FAILURE) {
     if (!c->disconnected && con != nullptr) {
       GRPC_CONNECTED_SUBCHANNEL_UNREF(con, "transient_failure");
       gpr_atm_no_barrier_store(&c->connected_subchannel, (gpr_atm) nullptr);
+      gpr_log(
+          GPR_INFO,
+          "LOL FORMER Connected subchannel %p of subchannel %p is now NULL.",
+          con, c);
       grpc_connectivity_state_set(&c->state_tracker,
                                   GRPC_CHANNEL_TRANSIENT_FAILURE,
                                   GRPC_ERROR_REF(error), "reflect_child");
@@ -547,28 +552,28 @@ static void on_connected_subchannel_connectivity_changed(void* p,
       c->backoff->Reset();
       if (grpc_trace_stream_refcount.enabled()) {
         gpr_log(GPR_INFO,
-                "Connected subchannel %p of subchannel %p has gone into "
-                "TRANSIENT_FAILURE. Attempting to reconnect.",
-                con, c);
+                "Connected subchannel %p of subchannel %p has gone into %s. "
+                "Attempting to reconnect.",
+                con, c, grpc_connectivity_state_name(
+                            connected_subchannel_watcher->connectivity_state));
       }
       maybe_start_connecting_locked(c);
-      goto done;
     } else {
       connected_subchannel_watcher->connectivity_state = GRPC_CHANNEL_SHUTDOWN;
     }
+  } else {
+    grpc_connectivity_state_set(
+        &c->state_tracker, connected_subchannel_watcher->connectivity_state,
+        GRPC_ERROR_REF(error), "reflect_child");
+    if (connected_subchannel_watcher->connectivity_state <
+        GRPC_CHANNEL_TRANSIENT_FAILURE) {
+      GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
+      con->NotifyOnStateChange(
+          nullptr, &connected_subchannel_watcher->connectivity_state,
+          &connected_subchannel_watcher->closure);
+      connected_subchannel_watcher = nullptr;
+    }
   }
-  grpc_connectivity_state_set(&c->state_tracker,
-                              connected_subchannel_watcher->connectivity_state,
-                              GRPC_ERROR_REF(error), "reflect_child");
-  if (connected_subchannel_watcher->connectivity_state <
-      GRPC_CHANNEL_TRANSIENT_FAILURE) {
-    GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
-    con->NotifyOnStateChange(nullptr,
-                             &connected_subchannel_watcher->connectivity_state,
-                             &connected_subchannel_watcher->closure);
-    connected_subchannel_watcher = nullptr;
-  }
-done:
   gpr_mu_unlock(mu);
   GRPC_SUBCHANNEL_WEAK_UNREF(c, "state_watcher");
   gpr_free(connected_subchannel_watcher);
@@ -619,8 +624,8 @@ static bool publish_transport_locked(grpc_subchannel* c) {
                     I'd have expected the rel_cas below to be enough, but
                     seemingly it's not.
                     Re-evaluate if we really need this. */
-  grpc_connected_subchannel* con =
-      grpc_core::New<grpc_connected_subchannel>(stk);
+  grpc_core::ConnectedSubchannel* con =
+      grpc_core::New<grpc_core::ConnectedSubchannel>(stk);
   gpr_atm_full_barrier();
   GPR_ASSERT(gpr_atm_rel_cas(&c->connected_subchannel, 0, (gpr_atm)con));
 
@@ -677,7 +682,7 @@ static void subchannel_call_destroy(void* call, grpc_error* error) {
   grpc_subchannel_call* c = (grpc_subchannel_call*)call;
   GPR_ASSERT(c->schedule_closure_after_destroy != nullptr);
   GPR_TIMER_BEGIN("grpc_subchannel_call_unref.destroy", 0);
-  grpc_connected_subchannel* connection = c->connection;
+  grpc_core::ConnectedSubchannel* connection = c->connection;
   grpc_call_stack_destroy(SUBCHANNEL_CALL_TO_CALL_STACK(c), nullptr,
                           c->schedule_closure_after_destroy);
   GRPC_CONNECTED_SUBCHANNEL_UNREF(connection, "subchannel_call");
@@ -711,7 +716,7 @@ void grpc_subchannel_call_process_op(grpc_subchannel_call* call,
   GPR_TIMER_END("grpc_subchannel_call_process_op", 0);
 }
 
-grpc_connected_subchannel* grpc_subchannel_get_connected_subchannel(
+grpc_core::ConnectedSubchannel* grpc_subchannel_get_connected_subchannel(
     grpc_subchannel* c) {
   return GET_CONNECTED_SUBCHANNEL(c, acq);
 }
@@ -757,24 +762,16 @@ grpc_arg grpc_create_subchannel_address_arg(const grpc_resolved_address* addr) {
       addr->len > 0 ? grpc_sockaddr_to_uri(addr) : gpr_strdup(""));
 }
 
-grpc_connected_subchannel::grpc_connected_subchannel(
-    grpc_channel_stack* channel_stack)
+namespace grpc_core {
+ConnectedSubchannel::ConnectedSubchannel(grpc_channel_stack* channel_stack)
     : grpc_core::RefCountedWithTracing(&grpc_trace_stream_refcount),
       channel_stack_(channel_stack) {}
 
-grpc_connected_subchannel* grpc_connected_subchannel::Ref(
-    const grpc_core::DebugLocation& location, const char* reason) {
-  GRPC_CHANNEL_STACK_REF(channel_stack_, REF_REASON);
-  grpc_core::RefCountedWithTracing::Ref(location, reason);
-  return this;
-}
-void grpc_connected_subchannel::Unref(const grpc_core::DebugLocation& location,
-                                      const char* reason) {
-  GRPC_CHANNEL_STACK_UNREF(channel_stack_, REF_REASON);
-  grpc_core::RefCountedWithTracing::Unref(location, reason);
+ConnectedSubchannel::~ConnectedSubchannel() {
+  GRPC_CHANNEL_STACK_UNREF(channel_stack_, "connected_subchannel_dtor");
 }
 
-void grpc_connected_subchannel::NotifyOnStateChange(
+void ConnectedSubchannel::NotifyOnStateChange(
     grpc_pollset_set* interested_parties, grpc_connectivity_state* state,
     grpc_closure* closure) {
   grpc_transport_op* op = grpc_make_transport_op(nullptr);
@@ -786,8 +783,8 @@ void grpc_connected_subchannel::NotifyOnStateChange(
   elem->filter->start_transport_op(elem, op);
 }
 
-void grpc_connected_subchannel::Ping(grpc_closure* on_initiate,
-                                     grpc_closure* on_ack) {
+void ConnectedSubchannel::Ping(grpc_closure* on_initiate,
+                               grpc_closure* on_ack) {
   grpc_transport_op* op = grpc_make_transport_op(nullptr);
   grpc_channel_element* elem;
   op->send_ping.on_initiate = on_initiate;
@@ -796,22 +793,23 @@ void grpc_connected_subchannel::Ping(grpc_closure* on_initiate,
   elem->filter->start_transport_op(elem, op);
 }
 
-grpc_error* grpc_connected_subchannel::CreateCall(const CallArgs* args,
-                                                  grpc_subchannel_call** call) {
+grpc_error* ConnectedSubchannel::CreateCall(const CallArgs& args,
+                                            grpc_subchannel_call** call) {
   *call = (grpc_subchannel_call*)gpr_arena_alloc(
-      args->arena,
+      args.arena,
       sizeof(grpc_subchannel_call) + channel_stack_->call_stack_size);
   grpc_call_stack* callstk = SUBCHANNEL_CALL_TO_CALL_STACK(*call);
-  (*call)->connection = Ref(DEBUG_LOCATION, "subchannel_call");
+  Ref(DEBUG_LOCATION, "subchannel_call");
+  (*call)->connection = this;
   const grpc_call_element_args call_args = {
-      callstk,            /* call_stack */
-      nullptr,            /* server_transport_data */
-      args->context,      /* context */
-      args->path,         /* path */
-      args->start_time,   /* start_time */
-      args->deadline,     /* deadline */
-      args->arena,        /* arena */
-      args->call_combiner /* call_combiner */
+      callstk,           /* call_stack */
+      nullptr,           /* server_transport_data */
+      args.context,      /* context */
+      args.path,         /* path */
+      args.start_time,   /* start_time */
+      args.deadline,     /* deadline */
+      args.arena,        /* arena */
+      args.call_combiner /* call_combiner */
   };
   grpc_error* error = grpc_call_stack_init(
       channel_stack_, 1, subchannel_call_destroy, *call, &call_args);
@@ -820,6 +818,7 @@ grpc_error* grpc_connected_subchannel::CreateCall(const CallArgs* args,
     gpr_log(GPR_ERROR, "error: %s", error_string);
     return error;
   }
-  grpc_call_stack_set_pollset_or_pollset_set(callstk, args->pollent);
+  grpc_call_stack_set_pollset_or_pollset_set(callstk, args.pollent);
   return GRPC_ERROR_NONE;
 }
+}  // namespace grpc_core
diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h
index d9a850daaea..dbc57427873 100644
--- a/src/core/ext/filters/client_channel/subchannel.h
+++ b/src/core/ext/filters/client_channel/subchannel.h
@@ -49,9 +49,9 @@ typedef struct grpc_subchannel_key grpc_subchannel_key;
 #define GRPC_SUBCHANNEL_WEAK_UNREF(p, r) \
   grpc_subchannel_weak_unref((p), __FILE__, __LINE__, (r))
 #define GRPC_CONNECTED_SUBCHANNEL_REF(p, r) \
-  grpc_connected_subchannel_ref((p), __FILE__, __LINE__, (r))
+  ConnectedSubchannel_ref((p), __FILE__, __LINE__, (r))
 #define GRPC_CONNECTED_SUBCHANNEL_UNREF(p, r) \
-  grpc_connected_subchannel_unref((p), __FILE__, __LINE__, (r))
+  ConnectedSubchannel_unref((p), __FILE__, __LINE__, (r))
 #define GRPC_SUBCHANNEL_CALL_REF(p, r) \
   grpc_subchannel_call_ref((p), __FILE__, __LINE__, (r))
 #define GRPC_SUBCHANNEL_CALL_UNREF(p, r) \
@@ -65,15 +65,15 @@ typedef struct grpc_subchannel_key grpc_subchannel_key;
 #define GRPC_SUBCHANNEL_UNREF(p, r) grpc_subchannel_unref((p))
 #define GRPC_SUBCHANNEL_WEAK_REF(p, r) grpc_subchannel_weak_ref((p))
 #define GRPC_SUBCHANNEL_WEAK_UNREF(p, r) grpc_subchannel_weak_unref((p))
-#define GRPC_CONNECTED_SUBCHANNEL_REF(p, r) grpc_connected_subchannel_ref((p))
-#define GRPC_CONNECTED_SUBCHANNEL_UNREF(p, r) \
-  grpc_connected_subchannel_unref((p))
+#define GRPC_CONNECTED_SUBCHANNEL_REF(p, r) ConnectedSubchannel_ref((p))
+#define GRPC_CONNECTED_SUBCHANNEL_UNREF(p, r) ConnectedSubchannel_unref((p))
 #define GRPC_SUBCHANNEL_CALL_REF(p, r) grpc_subchannel_call_ref((p))
 #define GRPC_SUBCHANNEL_CALL_UNREF(p, r) grpc_subchannel_call_unref((p))
 #define GRPC_SUBCHANNEL_REF_EXTRA_ARGS
 #endif
 
-class grpc_connected_subchannel : public grpc_core::RefCountedWithTracing {
+namespace grpc_core {
+class ConnectedSubchannel : public grpc_core::RefCountedWithTracing {
  public:
   struct CallArgs {
     grpc_polling_entity* pollent;
@@ -85,21 +85,20 @@ class grpc_connected_subchannel : public grpc_core::RefCountedWithTracing {
     grpc_call_combiner* call_combiner;
   };
 
-  grpc_connected_subchannel(grpc_channel_stack* channel_stack);
-  grpc_connected_subchannel* Ref(const grpc_core::DebugLocation& location,
-                                 const char* reason);
-  void Unref(const grpc_core::DebugLocation& location, const char* reason);
+  explicit ConnectedSubchannel(grpc_channel_stack* channel_stack);
+  ~ConnectedSubchannel();
+
   grpc_channel_stack* channel_stack() { return channel_stack_; }
   void NotifyOnStateChange(grpc_pollset_set* interested_parties,
                            grpc_connectivity_state* state,
                            grpc_closure* closure);
   void Ping(grpc_closure* on_initiate, grpc_closure* on_ack);
-
-  grpc_error* CreateCall(const CallArgs* args, grpc_subchannel_call** call);
+  grpc_error* CreateCall(const CallArgs& args, grpc_subchannel_call** call);
 
  private:
   grpc_channel_stack* channel_stack_;
 };
+}  // namespace grpc_core
 
 grpc_subchannel* grpc_subchannel_ref(
     grpc_subchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
@@ -111,10 +110,10 @@ grpc_subchannel* grpc_subchannel_weak_ref(
     grpc_subchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_weak_unref(
     grpc_subchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-grpc_connected_subchannel* grpc_connected_subchannel_ref(
-    grpc_connected_subchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-void grpc_connected_subchannel_unref(
-    grpc_connected_subchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+grpc_core::ConnectedSubchannel* ConnectedSubchannel_ref(
+    grpc_core::ConnectedSubchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+void ConnectedSubchannel_unref(
+    grpc_core::ConnectedSubchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_call_ref(
     grpc_subchannel_call* call GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_call_unref(
@@ -130,9 +129,9 @@ void grpc_subchannel_notify_on_state_change(
     grpc_subchannel* channel, grpc_pollset_set* interested_parties,
     grpc_connectivity_state* state, grpc_closure* notify);
 
-/** retrieve the grpc_connected_subchannel - or NULL if called before
+/** retrieve the grpc_core::ConnectedSubchannel - or NULL if called before
     the subchannel becomes connected */
-grpc_connected_subchannel* grpc_subchannel_get_connected_subchannel(
+grpc_core::ConnectedSubchannel* grpc_subchannel_get_connected_subchannel(
     grpc_subchannel* subchannel);
 
 /** return the subchannel index key for \a subchannel */
diff --git a/src/core/lib/support/ref_counted.h b/src/core/lib/support/ref_counted.h
index f2182baea19..8fdc3458d16 100644
--- a/src/core/lib/support/ref_counted.h
+++ b/src/core/lib/support/ref_counted.h
@@ -45,6 +45,7 @@ class RefCounted {
   // Not copyable nor movable.
   RefCounted(const RefCounted&) = delete;
   RefCounted& operator=(const RefCounted&) = delete;
+  GRPC_ABSTRACT_BASE_CLASS
 
  protected:
   // Allow Delete() to access destructor.
@@ -112,6 +113,8 @@ class RefCountedWithTracing {
     gpr_ref_init(&refs_, 1);
   }
 
+  virtual ~RefCountedWithTracing() {}
+
  private:
   TraceFlag* trace_flag_ = nullptr;
   gpr_refcount refs_;

From 0b62b2f6ed7c068d205f82a31adf3e29dbf02c2d Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Wed, 10 Jan 2018 11:38:37 +0100
Subject: [PATCH 057/161] reintroduce check_sources_and_headers optimization

---
 tools/run_tests/sanity/check_sources_and_headers.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py
index 6a704eb2e0b..57ae5c5acc4 100755
--- a/tools/run_tests/sanity/check_sources_and_headers.py
+++ b/tools/run_tests/sanity/check_sources_and_headers.py
@@ -69,13 +69,10 @@ target_headers_transitive = get_headers_transitive()
 
 
 def target_has_header(target, name):
-    if name.startswith('absl/'): return True
-    # print target['name'], name
-    if name in target['headers']:
+    if name in target_headers_transitive[target['name']]:
+        return True
+    if name.startswith('absl/'):
         return True
-    for dep in target['deps']:
-        if target_has_header(get_target(dep), name):
-            return True
     if name in [
             'src/core/lib/profiling/stap_probes.h',
             'src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h'

From 92e267ec502d018eae66f04faac0f5d57eadf572 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Wed, 10 Jan 2018 00:39:06 +0000
Subject: [PATCH 058/161] Add needed header

---
 src/cpp/thread_manager/thread_manager.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cpp/thread_manager/thread_manager.h b/src/cpp/thread_manager/thread_manager.h
index 4fa8a6c563b..c1783baa607 100644
--- a/src/cpp/thread_manager/thread_manager.h
+++ b/src/cpp/thread_manager/thread_manager.h
@@ -20,6 +20,7 @@
 #define GRPC_INTERNAL_CPP_THREAD_MANAGER_H
 
 #include <condition_variable>
+#include <functional>
 #include <list>
 #include <memory>
 #include <mutex>

From fd4884a768a03b8c459b02e7751c072f6efd147c Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Wed, 10 Jan 2018 18:47:09 +0000
Subject: [PATCH 059/161] Address review comments

---
 include/grpc++/server.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/include/grpc++/server.h b/include/grpc++/server.h
index 456603e4e7b..cf590185d1a 100644
--- a/include/grpc++/server.h
+++ b/include/grpc++/server.h
@@ -142,6 +142,9 @@ class Server final : public ServerInterface, private GrpcLibraryCodegen {
   ///
   /// \param thread_creator The thread creation function for the sync
   /// server. Typically gpr_thd_new
+  ///
+  /// \param thread_joiner The thread joining function for the sync
+  /// server. Typically gpr_thd_join
   Server(int max_message_size, ChannelArguments* args,
          std::shared_ptr<std::vector<std::unique_ptr<ServerCompletionQueue>>>
              sync_server_cqs,

From babd579449ab72fb0c21f15290ffc4832829db7d Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Wed, 10 Jan 2018 11:12:43 -0800
Subject: [PATCH 060/161] Mention requirements for regenerating projects

---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index af462468225..fc02f16f3b4 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -44,7 +44,7 @@ How to get your contributions merged smoothly and quickly.
  
 - Keep your PR up to date with upstream/master (if there are merge conflicts, we can't really merge your change).
  
-- if you are regenerating the projects using `tools/buildgen/generate_projects.sh`, make changes to generated files a separate commit with commit message `regenerate projects`. Mixing changes to generated and hand-written files make your PR difficult to review.
+- if you are regenerating the projects using `tools/buildgen/generate_projects.sh`, make changes to generated files a separate commit with commit message `regenerate projects`. Mixing changes to generated and hand-written files make your PR difficult to review. Note that running this script requires the installation of Python packages `pyyaml` and `mako` (typically installed using `pip`) as well as a recent version of [`go`](https://golang.org/doc/install#install).
  
 - **All tests need to be passing** before your change can be merged. We recommend you **run tests locally** before creating your PR to catch breakages early on (see [tools/run_tests](tools/run_tests). Ultimately, the green signal will be provided by our testing infrastructure. The reviewer will help you if there are test failures that seem not related to the change you are making.
  

From b3fa256dd5582c838b4928bb64ef2daaca5ad85a Mon Sep 17 00:00:00 2001
From: Yash Tibrewal <yashkt@google.com>
Date: Wed, 10 Jan 2018 12:13:06 -0800
Subject: [PATCH 061/161] Revert "Simplify LB policy refcounting."

---
 .../filters/client_channel/client_channel.cc  |  71 +-
 .../ext/filters/client_channel/lb_policy.cc   |  95 ++-
 .../ext/filters/client_channel/lb_policy.h    |  90 +--
 .../client_channel/lb_policy/grpclb/grpclb.cc | 607 +++++++++++-------
 .../lb_policy/pick_first/pick_first.cc        |  91 +--
 .../lb_policy/round_robin/round_robin.cc      | 124 ++--
 .../lb_policy/subchannel_list.cc              |   4 +-
 7 files changed, 667 insertions(+), 415 deletions(-)

diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc
index 3f3334d44a3..e99022a91b8 100644
--- a/src/core/ext/filters/client_channel/client_channel.cc
+++ b/src/core/ext/filters/client_channel/client_channel.cc
@@ -553,7 +553,6 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) {
       }
       grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties,
                                        chand->interested_parties);
-      grpc_lb_policy_shutdown_locked(chand->lb_policy, new_lb_policy);
       GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel");
     }
     chand->lb_policy = new_lb_policy;
@@ -659,7 +658,6 @@ static void start_transport_op_locked(void* arg, grpc_error* error_ignored) {
       if (chand->lb_policy != nullptr) {
         grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties,
                                          chand->interested_parties);
-        grpc_lb_policy_shutdown_locked(chand->lb_policy, nullptr);
         GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel");
         chand->lb_policy = nullptr;
       }
@@ -794,7 +792,6 @@ static void cc_destroy_channel_elem(grpc_channel_element* elem) {
   if (chand->lb_policy != nullptr) {
     grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties,
                                      chand->interested_parties);
-    grpc_lb_policy_shutdown_locked(chand->lb_policy, nullptr);
     GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel");
   }
   gpr_free(chand->info_lb_policy_name);
@@ -855,10 +852,12 @@ typedef struct client_channel_call_data {
   grpc_subchannel_call* subchannel_call;
   grpc_error* error;
 
-  grpc_lb_policy_pick_state pick;
+  grpc_lb_policy* lb_policy;  // Holds ref while LB pick is pending.
   grpc_closure lb_pick_closure;
   grpc_closure lb_pick_cancel_closure;
 
+  grpc_connected_subchannel* connected_subchannel;
+  grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
   grpc_polling_entity* pollent;
 
   grpc_transport_stream_op_batch* waiting_for_pick_batches[MAX_WAITING_BATCHES];
@@ -867,6 +866,8 @@ typedef struct client_channel_call_data {
 
   grpc_transport_stream_op_batch* initial_metadata_batch;
 
+  grpc_linked_mdelem lb_token_mdelem;
+
   grpc_closure on_complete;
   grpc_closure* original_on_complete;
 } call_data;
@@ -1004,16 +1005,16 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
   channel_data* chand = (channel_data*)elem->channel_data;
   call_data* calld = (call_data*)elem->call_data;
   const grpc_connected_subchannel_call_args call_args = {
-      calld->pollent,                       // pollent
-      calld->path,                          // path
-      calld->call_start_time,               // start_time
-      calld->deadline,                      // deadline
-      calld->arena,                         // arena
-      calld->pick.subchannel_call_context,  // context
-      calld->call_combiner                  // call_combiner
+      calld->pollent,                  // pollent
+      calld->path,                     // path
+      calld->call_start_time,          // start_time
+      calld->deadline,                 // deadline
+      calld->arena,                    // arena
+      calld->subchannel_call_context,  // context
+      calld->call_combiner             // call_combiner
   };
   grpc_error* new_error = grpc_connected_subchannel_create_call(
-      calld->pick.connected_subchannel, &call_args, &calld->subchannel_call);
+      calld->connected_subchannel, &call_args, &calld->subchannel_call);
   if (grpc_client_channel_trace.enabled()) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: create subchannel_call=%p: error=%s",
             chand, calld, calld->subchannel_call, grpc_error_string(new_error));
@@ -1031,7 +1032,7 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
 static void pick_done_locked(grpc_call_element* elem, grpc_error* error) {
   call_data* calld = (call_data*)elem->call_data;
   channel_data* chand = (channel_data*)elem->channel_data;
-  if (calld->pick.connected_subchannel == nullptr) {
+  if (calld->connected_subchannel == nullptr) {
     // Failed to create subchannel.
     GRPC_ERROR_UNREF(calld->error);
     calld->error = error == GRPC_ERROR_NONE
@@ -1070,16 +1071,13 @@ static void pick_callback_cancel_locked(void* arg, grpc_error* error) {
   grpc_call_element* elem = (grpc_call_element*)arg;
   channel_data* chand = (channel_data*)elem->channel_data;
   call_data* calld = (call_data*)elem->call_data;
-  // Note: chand->lb_policy may have changed since we started our pick,
-  // in which case we will be cancelling the pick on a policy other than
-  // the one we started it on.  However, this will just be a no-op.
-  if (error != GRPC_ERROR_NONE && chand->lb_policy != nullptr) {
+  if (calld->lb_policy != nullptr) {
     if (grpc_client_channel_trace.enabled()) {
       gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p",
-              chand, calld, chand->lb_policy);
+              chand, calld, calld->lb_policy);
     }
-    grpc_lb_policy_cancel_pick_locked(chand->lb_policy, &calld->pick,
-                                      GRPC_ERROR_REF(error));
+    grpc_lb_policy_cancel_pick_locked(
+        calld->lb_policy, &calld->connected_subchannel, GRPC_ERROR_REF(error));
   }
   GRPC_CALL_STACK_UNREF(calld->owning_call, "pick_callback_cancel");
 }
@@ -1094,6 +1092,9 @@ static void pick_callback_done_locked(void* arg, grpc_error* error) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously",
             chand, calld);
   }
+  GPR_ASSERT(calld->lb_policy != nullptr);
+  GRPC_LB_POLICY_UNREF(calld->lb_policy, "pick_subchannel");
+  calld->lb_policy = nullptr;
   async_pick_done_locked(elem, GRPC_ERROR_REF(error));
 }
 
@@ -1127,21 +1128,26 @@ static bool pick_callback_start_locked(grpc_call_element* elem) {
       initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY;
     }
   }
-  calld->pick.initial_metadata =
+  const grpc_lb_policy_pick_args inputs = {
       calld->initial_metadata_batch->payload->send_initial_metadata
-          .send_initial_metadata;
-  calld->pick.initial_metadata_flags = initial_metadata_flags;
+          .send_initial_metadata,
+      initial_metadata_flags, &calld->lb_token_mdelem};
+  // Keep a ref to the LB policy in calld while the pick is pending.
+  GRPC_LB_POLICY_REF(chand->lb_policy, "pick_subchannel");
+  calld->lb_policy = chand->lb_policy;
   GRPC_CLOSURE_INIT(&calld->lb_pick_closure, pick_callback_done_locked, elem,
                     grpc_combiner_scheduler(chand->combiner));
-  calld->pick.on_complete = &calld->lb_pick_closure;
-  const bool pick_done =
-      grpc_lb_policy_pick_locked(chand->lb_policy, &calld->pick);
+  const bool pick_done = grpc_lb_policy_pick_locked(
+      chand->lb_policy, &inputs, &calld->connected_subchannel,
+      calld->subchannel_call_context, nullptr, &calld->lb_pick_closure);
   if (pick_done) {
     /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */
     if (grpc_client_channel_trace.enabled()) {
       gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed synchronously",
               chand, calld);
     }
+    GRPC_LB_POLICY_UNREF(calld->lb_policy, "pick_subchannel");
+    calld->lb_policy = nullptr;
   } else {
     GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel");
     grpc_call_combiner_set_notify_on_cancel(
@@ -1283,7 +1289,7 @@ static void start_pick_locked(void* arg, grpc_error* ignored) {
   grpc_call_element* elem = (grpc_call_element*)arg;
   call_data* calld = (call_data*)elem->call_data;
   channel_data* chand = (channel_data*)elem->channel_data;
-  GPR_ASSERT(calld->pick.connected_subchannel == nullptr);
+  GPR_ASSERT(calld->connected_subchannel == nullptr);
   if (chand->lb_policy != nullptr) {
     // We already have an LB policy, so ask it for a pick.
     if (pick_callback_start_locked(elem)) {
@@ -1461,14 +1467,15 @@ static void cc_destroy_call_elem(grpc_call_element* elem,
     GRPC_SUBCHANNEL_CALL_UNREF(calld->subchannel_call,
                                "client_channel_destroy_call");
   }
+  GPR_ASSERT(calld->lb_policy == nullptr);
   GPR_ASSERT(calld->waiting_for_pick_batches_count == 0);
-  if (calld->pick.connected_subchannel != nullptr) {
-    GRPC_CONNECTED_SUBCHANNEL_UNREF(calld->pick.connected_subchannel, "picked");
+  if (calld->connected_subchannel != nullptr) {
+    GRPC_CONNECTED_SUBCHANNEL_UNREF(calld->connected_subchannel, "picked");
   }
   for (size_t i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
-    if (calld->pick.subchannel_call_context[i].value != nullptr) {
-      calld->pick.subchannel_call_context[i].destroy(
-          calld->pick.subchannel_call_context[i].value);
+    if (calld->subchannel_call_context[i].value != nullptr) {
+      calld->subchannel_call_context[i].destroy(
+          calld->subchannel_call_context[i].value);
     }
   }
   GRPC_CLOSURE_SCHED(then_schedule_closure, GRPC_ERROR_NONE);
diff --git a/src/core/ext/filters/client_channel/lb_policy.cc b/src/core/ext/filters/client_channel/lb_policy.cc
index cc4fe7ec627..7a5a8dec34b 100644
--- a/src/core/ext/filters/client_channel/lb_policy.cc
+++ b/src/core/ext/filters/client_channel/lb_policy.cc
@@ -19,6 +19,8 @@
 #include "src/core/ext/filters/client_channel/lb_policy.h"
 #include "src/core/lib/iomgr/combiner.h"
 
+#define WEAK_REF_BITS 16
+
 grpc_core::DebugOnlyTraceFlag grpc_trace_lb_policy_refcount(
     false, "lb_policy_refcount");
 
@@ -26,60 +28,91 @@ void grpc_lb_policy_init(grpc_lb_policy* policy,
                          const grpc_lb_policy_vtable* vtable,
                          grpc_combiner* combiner) {
   policy->vtable = vtable;
-  gpr_ref_init(&policy->refs, 1);
+  gpr_atm_no_barrier_store(&policy->ref_pair, 1 << WEAK_REF_BITS);
   policy->interested_parties = grpc_pollset_set_create();
   policy->combiner = GRPC_COMBINER_REF(combiner, "lb_policy");
 }
 
 #ifndef NDEBUG
-void grpc_lb_policy_ref(grpc_lb_policy* lb_policy, const char* file, int line,
-                        const char* reason) {
-  if (grpc_trace_lb_policy_refcount.enabled()) {
-    gpr_atm old_refs = gpr_atm_no_barrier_load(&lb_policy->refs.count);
-    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-            "LB_POLICY:%p   ref %" PRIdPTR " -> %" PRIdPTR " %s", lb_policy,
-            old_refs, old_refs + 1, reason);
-  }
+#define REF_FUNC_EXTRA_ARGS , const char *file, int line, const char *reason
+#define REF_MUTATE_EXTRA_ARGS REF_FUNC_EXTRA_ARGS, const char* purpose
+#define REF_FUNC_PASS_ARGS(new_reason) , file, line, new_reason
+#define REF_MUTATE_PASS_ARGS(purpose) , file, line, reason, purpose
 #else
-void grpc_lb_policy_ref(grpc_lb_policy* lb_policy) {
+#define REF_FUNC_EXTRA_ARGS
+#define REF_MUTATE_EXTRA_ARGS
+#define REF_FUNC_PASS_ARGS(new_reason)
+#define REF_MUTATE_PASS_ARGS(x)
 #endif
-  gpr_ref(&lb_policy->refs);
-}
 
+static gpr_atm ref_mutate(grpc_lb_policy* c, gpr_atm delta,
+                          int barrier REF_MUTATE_EXTRA_ARGS) {
+  gpr_atm old_val = barrier ? gpr_atm_full_fetch_add(&c->ref_pair, delta)
+                            : gpr_atm_no_barrier_fetch_add(&c->ref_pair, delta);
 #ifndef NDEBUG
-void grpc_lb_policy_unref(grpc_lb_policy* lb_policy, const char* file, int line,
-                          const char* reason) {
   if (grpc_trace_lb_policy_refcount.enabled()) {
-    gpr_atm old_refs = gpr_atm_no_barrier_load(&lb_policy->refs.count);
     gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-            "LB_POLICY:%p unref %" PRIdPTR " -> %" PRIdPTR " %s", lb_policy,
-            old_refs, old_refs - 1, reason);
+            "LB_POLICY: %p %12s 0x%" PRIxPTR " -> 0x%" PRIxPTR " [%s]", c,
+            purpose, old_val, old_val + delta, reason);
   }
-#else
-void grpc_lb_policy_unref(grpc_lb_policy* lb_policy) {
 #endif
-  if (gpr_unref(&lb_policy->refs)) {
-    grpc_pollset_set_destroy(lb_policy->interested_parties);
-    grpc_combiner* combiner = lb_policy->combiner;
-    lb_policy->vtable->destroy(lb_policy);
-    GRPC_COMBINER_UNREF(combiner, "lb_policy");
+  return old_val;
+}
+
+void grpc_lb_policy_ref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
+  ref_mutate(policy, 1 << WEAK_REF_BITS, 0 REF_MUTATE_PASS_ARGS("STRONG_REF"));
+}
+
+static void shutdown_locked(void* arg, grpc_error* error) {
+  grpc_lb_policy* policy = (grpc_lb_policy*)arg;
+  policy->vtable->shutdown_locked(policy);
+  GRPC_LB_POLICY_WEAK_UNREF(policy, "strong-unref");
+}
+
+void grpc_lb_policy_unref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
+  gpr_atm old_val =
+      ref_mutate(policy, (gpr_atm)1 - (gpr_atm)(1 << WEAK_REF_BITS),
+                 1 REF_MUTATE_PASS_ARGS("STRONG_UNREF"));
+  gpr_atm mask = ~(gpr_atm)((1 << WEAK_REF_BITS) - 1);
+  gpr_atm check = 1 << WEAK_REF_BITS;
+  if ((old_val & mask) == check) {
+    GRPC_CLOSURE_SCHED(
+        GRPC_CLOSURE_CREATE(shutdown_locked, policy,
+                            grpc_combiner_scheduler(policy->combiner)),
+        GRPC_ERROR_NONE);
+  } else {
+    grpc_lb_policy_weak_unref(policy REF_FUNC_PASS_ARGS("strong-unref"));
   }
 }
 
-void grpc_lb_policy_shutdown_locked(grpc_lb_policy* policy,
-                                    grpc_lb_policy* new_policy) {
-  policy->vtable->shutdown_locked(policy, new_policy);
+void grpc_lb_policy_weak_ref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
+  ref_mutate(policy, 1, 0 REF_MUTATE_PASS_ARGS("WEAK_REF"));
+}
+
+void grpc_lb_policy_weak_unref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
+  gpr_atm old_val =
+      ref_mutate(policy, -(gpr_atm)1, 1 REF_MUTATE_PASS_ARGS("WEAK_UNREF"));
+  if (old_val == 1) {
+    grpc_pollset_set_destroy(policy->interested_parties);
+    grpc_combiner* combiner = policy->combiner;
+    policy->vtable->destroy(policy);
+    GRPC_COMBINER_UNREF(combiner, "lb_policy");
+  }
 }
 
 int grpc_lb_policy_pick_locked(grpc_lb_policy* policy,
-                               grpc_lb_policy_pick_state* pick) {
-  return policy->vtable->pick_locked(policy, pick);
+                               const grpc_lb_policy_pick_args* pick_args,
+                               grpc_connected_subchannel** target,
+                               grpc_call_context_element* context,
+                               void** user_data, grpc_closure* on_complete) {
+  return policy->vtable->pick_locked(policy, pick_args, target, context,
+                                     user_data, on_complete);
 }
 
 void grpc_lb_policy_cancel_pick_locked(grpc_lb_policy* policy,
-                                       grpc_lb_policy_pick_state* pick,
+                                       grpc_connected_subchannel** target,
                                        grpc_error* error) {
-  policy->vtable->cancel_pick_locked(policy, pick, error);
+  policy->vtable->cancel_pick_locked(policy, target, error);
 }
 
 void grpc_lb_policy_cancel_picks_locked(grpc_lb_policy* policy,
diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h
index 1176a05b780..3572c97ed14 100644
--- a/src/core/ext/filters/client_channel/lb_policy.h
+++ b/src/core/ext/filters/client_channel/lb_policy.h
@@ -33,7 +33,7 @@ extern grpc_core::DebugOnlyTraceFlag grpc_trace_lb_policy_refcount;
 
 struct grpc_lb_policy {
   const grpc_lb_policy_vtable* vtable;
-  gpr_refcount refs;
+  gpr_atm ref_pair;
   /* owned pointer to interested parties in load balancing decisions */
   grpc_pollset_set* interested_parties;
   /* combiner under which lb_policy actions take place */
@@ -42,42 +42,32 @@ struct grpc_lb_policy {
   grpc_closure* request_reresolution;
 };
 
-/// State used for an LB pick.
-typedef struct grpc_lb_policy_pick_state {
-  /// Initial metadata associated with the picking call.
+/** Extra arguments for an LB pick */
+typedef struct grpc_lb_policy_pick_args {
+  /** Initial metadata associated with the picking call. */
   grpc_metadata_batch* initial_metadata;
-  /// Bitmask used for selective cancelling. See \a
-  /// grpc_lb_policy_cancel_picks() and \a GRPC_INITIAL_METADATA_* in
-  /// grpc_types.h.
+  /** Bitmask used for selective cancelling. See \a
+   * grpc_lb_policy_cancel_picks() and \a GRPC_INITIAL_METADATA_* in
+   * grpc_types.h */
   uint32_t initial_metadata_flags;
-  /// Storage for LB token in \a initial_metadata, or NULL if not used.
-  grpc_linked_mdelem lb_token_mdelem_storage;
-  /// Closure to run when pick is complete, if not completed synchronously.
-  grpc_closure* on_complete;
-  /// Will be set to the selected subchannel, or NULL on failure or when
-  /// the LB policy decides to drop the call.
-  grpc_connected_subchannel* connected_subchannel;
-  /// Will be populated with context to pass to the subchannel call, if needed.
-  grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
-  /// Upon success, \a *user_data will be set to whatever opaque information
-  /// may need to be propagated from the LB policy, or NULL if not needed.
-  void** user_data;
-  /// Next pointer.  For internal use by LB policy.
-  struct grpc_lb_policy_pick_state* next;
-} grpc_lb_policy_pick_state;
+  /** Storage for LB token in \a initial_metadata, or NULL if not used */
+  grpc_linked_mdelem* lb_token_mdelem_storage;
+} grpc_lb_policy_pick_args;
 
 struct grpc_lb_policy_vtable {
   void (*destroy)(grpc_lb_policy* policy);
-
-  /// \see grpc_lb_policy_shutdown_locked().
-  void (*shutdown_locked)(grpc_lb_policy* policy, grpc_lb_policy* new_policy);
+  void (*shutdown_locked)(grpc_lb_policy* policy);
 
   /** \see grpc_lb_policy_pick */
-  int (*pick_locked)(grpc_lb_policy* policy, grpc_lb_policy_pick_state* pick);
+  int (*pick_locked)(grpc_lb_policy* policy,
+                     const grpc_lb_policy_pick_args* pick_args,
+                     grpc_connected_subchannel** target,
+                     grpc_call_context_element* context, void** user_data,
+                     grpc_closure* on_complete);
 
   /** \see grpc_lb_policy_cancel_pick */
   void (*cancel_pick_locked)(grpc_lb_policy* policy,
-                             grpc_lb_policy_pick_state* pick,
+                             grpc_connected_subchannel** target,
                              grpc_error* error);
 
   /** \see grpc_lb_policy_cancel_picks */
@@ -113,19 +103,37 @@ struct grpc_lb_policy_vtable {
 };
 
 #ifndef NDEBUG
+
+/* Strong references: the policy will shutdown when they reach zero */
 #define GRPC_LB_POLICY_REF(p, r) \
   grpc_lb_policy_ref((p), __FILE__, __LINE__, (r))
 #define GRPC_LB_POLICY_UNREF(p, r) \
   grpc_lb_policy_unref((p), __FILE__, __LINE__, (r))
+
+/* Weak references: they don't prevent the shutdown of the LB policy. When no
+ * strong references are left but there are still weak ones, shutdown is called.
+ * Once the weak reference also reaches zero, the LB policy is destroyed. */
+#define GRPC_LB_POLICY_WEAK_REF(p, r) \
+  grpc_lb_policy_weak_ref((p), __FILE__, __LINE__, (r))
+#define GRPC_LB_POLICY_WEAK_UNREF(p, r) \
+  grpc_lb_policy_weak_unref((p), __FILE__, __LINE__, (r))
 void grpc_lb_policy_ref(grpc_lb_policy* policy, const char* file, int line,
                         const char* reason);
 void grpc_lb_policy_unref(grpc_lb_policy* policy, const char* file, int line,
                           const char* reason);
-#else  // !NDEBUG
+void grpc_lb_policy_weak_ref(grpc_lb_policy* policy, const char* file, int line,
+                             const char* reason);
+void grpc_lb_policy_weak_unref(grpc_lb_policy* policy, const char* file,
+                               int line, const char* reason);
+#else
 #define GRPC_LB_POLICY_REF(p, r) grpc_lb_policy_ref((p))
 #define GRPC_LB_POLICY_UNREF(p, r) grpc_lb_policy_unref((p))
+#define GRPC_LB_POLICY_WEAK_REF(p, r) grpc_lb_policy_weak_ref((p))
+#define GRPC_LB_POLICY_WEAK_UNREF(p, r) grpc_lb_policy_weak_unref((p))
 void grpc_lb_policy_ref(grpc_lb_policy* policy);
 void grpc_lb_policy_unref(grpc_lb_policy* policy);
+void grpc_lb_policy_weak_ref(grpc_lb_policy* policy);
+void grpc_lb_policy_weak_unref(grpc_lb_policy* policy);
 #endif
 
 /** called by concrete implementations to initialize the base struct */
@@ -133,24 +141,28 @@ void grpc_lb_policy_init(grpc_lb_policy* policy,
                          const grpc_lb_policy_vtable* vtable,
                          grpc_combiner* combiner);
 
-/// Shuts down \a policy.
-/// If \a new_policy is non-null, any pending picks will be restarted
-/// on that policy; otherwise, they will be failed.
-void grpc_lb_policy_shutdown_locked(grpc_lb_policy* policy,
-                                    grpc_lb_policy* new_policy);
+/** Finds an appropriate subchannel for a call, based on \a pick_args.
+
+    \a target will be set to the selected subchannel, or NULL on failure
+    or when the LB policy decides to drop the call.
 
-/** Finds an appropriate subchannel for a call, based on data in \a pick.
-    \a pick must remain alive until the pick is complete.
+    Upon success, \a user_data will be set to whatever opaque information
+    may need to be propagated from the LB policy, or NULL if not needed.
+    \a context will be populated with context to pass to the subchannel
+    call, if needed.
 
     If the pick succeeds and a result is known immediately, a non-zero
-    value will be returned.  Otherwise, \a pick->on_complete will be invoked
+    value will be returned.  Otherwise, \a on_complete will be invoked
     once the pick is complete with its error argument set to indicate
     success or failure.
 
     Any IO should be done under the \a interested_parties \a grpc_pollset_set
     in the \a grpc_lb_policy struct. */
 int grpc_lb_policy_pick_locked(grpc_lb_policy* policy,
-                               grpc_lb_policy_pick_state* pick);
+                               const grpc_lb_policy_pick_args* pick_args,
+                               grpc_connected_subchannel** target,
+                               grpc_call_context_element* context,
+                               void** user_data, grpc_closure* on_complete);
 
 /** Perform a connected subchannel ping (see \a grpc_connected_subchannel_ping)
     against one of the connected subchannels managed by \a policy. */
@@ -158,11 +170,11 @@ void grpc_lb_policy_ping_one_locked(grpc_lb_policy* policy,
                                     grpc_closure* on_initiate,
                                     grpc_closure* on_ack);
 
-/** Cancel picks for \a pick.
+/** Cancel picks for \a target.
     The \a on_complete callback of the pending picks will be invoked with \a
     *target set to NULL. */
 void grpc_lb_policy_cancel_pick_locked(grpc_lb_policy* policy,
-                                       grpc_lb_policy_pick_state* pick,
+                                       grpc_connected_subchannel** target,
                                        grpc_error* error);
 
 /** Cancel all pending picks for which their \a initial_metadata_flags (as given
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index 5849ac9d2da..1317cdcf750 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -54,7 +54,7 @@
  *    operations in progress over the old RR instance. This is done by
  *    decreasing the reference count on the old policy. The moment no more
  *    references are held on the old RR policy, it'll be destroyed and \a
- *    on_rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN
+ *    glb_rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN
  *    state. At this point we can transition to a new RR instance safely, which
  *    is done once again via \a rr_handover_locked().
  *
@@ -128,48 +128,187 @@
 
 grpc_core::TraceFlag grpc_lb_glb_trace(false, "glb");
 
-struct glb_lb_policy;
+/* add lb_token of selected subchannel (address) to the call's initial
+ * metadata */
+static grpc_error* initial_metadata_add_lb_token(
+    grpc_metadata_batch* initial_metadata,
+    grpc_linked_mdelem* lb_token_mdelem_storage, grpc_mdelem lb_token) {
+  GPR_ASSERT(lb_token_mdelem_storage != nullptr);
+  GPR_ASSERT(!GRPC_MDISNULL(lb_token));
+  return grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage,
+                                      lb_token);
+}
 
-namespace {
+static void destroy_client_stats(void* arg) {
+  grpc_grpclb_client_stats_unref((grpc_grpclb_client_stats*)arg);
+}
 
-/// Linked list of pending pick requests. It stores all information needed to
-/// eventually call (Round Robin's) pick() on them. They mainly stay pending
-/// waiting for the RR policy to be created.
-///
-/// Note that when a pick is sent to the RR policy, we inject our own
-/// on_complete callback, so that we can intercept the result before
-/// invoking the original on_complete callback.  This allows us to set the
-/// LB token metadata and add client_stats to the call context.
-/// See \a pending_pick_complete() for details.
-struct pending_pick {
-  // Our on_complete closure and the original one.
-  grpc_closure on_complete;
-  grpc_closure* original_on_complete;
-  // The original pick.
-  grpc_lb_policy_pick_state* pick;
-  // Stats for client-side load reporting. Note that this holds a
-  // reference, which must be either passed on via context or unreffed.
+typedef struct wrapped_rr_closure_arg {
+  /* the closure instance using this struct as argument */
+  grpc_closure wrapper_closure;
+
+  /* the original closure. Usually a on_complete/notify cb for pick() and ping()
+   * calls against the internal RR instance, respectively. */
+  grpc_closure* wrapped_closure;
+
+  /* the pick's initial metadata, kept in order to append the LB token for the
+   * pick */
+  grpc_metadata_batch* initial_metadata;
+
+  /* the picked target, used to determine which LB token to add to the pick's
+   * initial metadata */
+  grpc_connected_subchannel** target;
+
+  /* the context to be populated for the subchannel call */
+  grpc_call_context_element* context;
+
+  /* Stats for client-side load reporting. Note that this holds a
+   * reference, which must be either passed on via context or unreffed. */
   grpc_grpclb_client_stats* client_stats;
-  // The LB token associated with the pick.  This is set via user_data in
-  // the pick.
+
+  /* the LB token associated with the pick */
   grpc_mdelem lb_token;
-  // The grpclb instance that created the wrapping. This instance is not owned,
-  // reference counts are untouched. It's used only for logging purposes.
-  glb_lb_policy* glb_policy;
-  // Next pending pick.
+
+  /* storage for the lb token initial metadata mdelem */
+  grpc_linked_mdelem* lb_token_mdelem_storage;
+
+  /* The RR instance related to the closure */
+  grpc_lb_policy* rr_policy;
+
+  /* The grpclb instance that created the wrapping. This instance is not owned,
+   * reference counts are untouched. It's used only for logging purposes. */
+  grpc_lb_policy* glb_policy;
+
+  /* heap memory to be freed upon closure execution. */
+  void* free_when_done;
+} wrapped_rr_closure_arg;
+
+/* The \a on_complete closure passed as part of the pick requires keeping a
+ * reference to its associated round robin instance. We wrap this closure in
+ * order to unref the round robin instance upon its invocation */
+static void wrapped_rr_closure(void* arg, grpc_error* error) {
+  wrapped_rr_closure_arg* wc_arg = (wrapped_rr_closure_arg*)arg;
+
+  GPR_ASSERT(wc_arg->wrapped_closure != nullptr);
+  GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_REF(error));
+
+  if (wc_arg->rr_policy != nullptr) {
+    /* if *target is nullptr, no pick has been made by the RR policy (eg, all
+     * addresses failed to connect). There won't be any user_data/token
+     * available */
+    if (*wc_arg->target != nullptr) {
+      if (!GRPC_MDISNULL(wc_arg->lb_token)) {
+        initial_metadata_add_lb_token(wc_arg->initial_metadata,
+                                      wc_arg->lb_token_mdelem_storage,
+                                      GRPC_MDELEM_REF(wc_arg->lb_token));
+      } else {
+        gpr_log(
+            GPR_ERROR,
+            "[grpclb %p] No LB token for connected subchannel pick %p (from RR "
+            "instance %p).",
+            wc_arg->glb_policy, *wc_arg->target, wc_arg->rr_policy);
+        abort();
+      }
+      // Pass on client stats via context. Passes ownership of the reference.
+      GPR_ASSERT(wc_arg->client_stats != nullptr);
+      wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].value = wc_arg->client_stats;
+      wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].destroy = destroy_client_stats;
+    } else {
+      grpc_grpclb_client_stats_unref(wc_arg->client_stats);
+    }
+    if (grpc_lb_glb_trace.enabled()) {
+      gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p", wc_arg->glb_policy,
+              wc_arg->rr_policy);
+    }
+    GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "wrapped_rr_closure");
+  }
+  GPR_ASSERT(wc_arg->free_when_done != nullptr);
+  gpr_free(wc_arg->free_when_done);
+}
+
+namespace {
+/* Linked list of pending pick requests. It stores all information needed to
+ * eventually call (Round Robin's) pick() on them. They mainly stay pending
+ * waiting for the RR policy to be created/updated.
+ *
+ * One particularity is the wrapping of the user-provided \a on_complete closure
+ * (in \a wrapped_on_complete and \a wrapped_on_complete_arg). This is needed in
+ * order to correctly unref the RR policy instance upon completion of the pick.
+ * See \a wrapped_rr_closure for details. */
+struct pending_pick {
   struct pending_pick* next;
+
+  /* original pick()'s arguments */
+  grpc_lb_policy_pick_args pick_args;
+
+  /* output argument where to store the pick()ed connected subchannel, or
+   * nullptr upon error. */
+  grpc_connected_subchannel** target;
+
+  /* args for wrapped_on_complete */
+  wrapped_rr_closure_arg wrapped_on_complete_arg;
 };
+}  // namespace
+
+static void add_pending_pick(pending_pick** root,
+                             const grpc_lb_policy_pick_args* pick_args,
+                             grpc_connected_subchannel** target,
+                             grpc_call_context_element* context,
+                             grpc_closure* on_complete) {
+  pending_pick* pp = (pending_pick*)gpr_zalloc(sizeof(*pp));
+  pp->next = *root;
+  pp->pick_args = *pick_args;
+  pp->target = target;
+  pp->wrapped_on_complete_arg.wrapped_closure = on_complete;
+  pp->wrapped_on_complete_arg.target = target;
+  pp->wrapped_on_complete_arg.context = context;
+  pp->wrapped_on_complete_arg.initial_metadata = pick_args->initial_metadata;
+  pp->wrapped_on_complete_arg.lb_token_mdelem_storage =
+      pick_args->lb_token_mdelem_storage;
+  pp->wrapped_on_complete_arg.free_when_done = pp;
+  GRPC_CLOSURE_INIT(&pp->wrapped_on_complete_arg.wrapper_closure,
+                    wrapped_rr_closure, &pp->wrapped_on_complete_arg,
+                    grpc_schedule_on_exec_ctx);
+  *root = pp;
+}
 
-/// A linked list of pending pings waiting for the RR policy to be created.
-struct pending_ping {
-  grpc_closure* on_initiate;
-  grpc_closure* on_ack;
+/* Same as the \a pending_pick struct but for ping operations */
+typedef struct pending_ping {
   struct pending_ping* next;
-};
 
-}  // namespace
+  /* args for sending the ping */
+  wrapped_rr_closure_arg* on_initiate;
+  wrapped_rr_closure_arg* on_ack;
+} pending_ping;
+
+static void add_pending_ping(pending_ping** root, grpc_closure* on_initiate,
+                             grpc_closure* on_ack) {
+  pending_ping* pping = (pending_ping*)gpr_zalloc(sizeof(*pping));
+  if (on_initiate != nullptr) {
+    pping->on_initiate =
+        (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(*pping->on_initiate));
+    pping->on_initiate->wrapped_closure = on_initiate;
+    pping->on_initiate->free_when_done = pping->on_initiate;
+    GRPC_CLOSURE_INIT(&pping->on_initiate->wrapper_closure, wrapped_rr_closure,
+                      &pping->on_initiate, grpc_schedule_on_exec_ctx);
+  }
+  if (on_ack != nullptr) {
+    pping->on_ack = (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(*pping->on_ack));
+    pping->on_ack->wrapped_closure = on_ack;
+    pping->on_ack->free_when_done = pping->on_ack;
+    GRPC_CLOSURE_INIT(&pping->on_ack->wrapper_closure, wrapped_rr_closure,
+                      &pping->on_ack, grpc_schedule_on_exec_ctx);
+  }
+  pping->next = *root;
+  *root = pping;
+}
 
-struct glb_lb_policy {
+/*
+ * glb_lb_policy
+ */
+typedef struct rr_connectivity_data rr_connectivity_data;
+
+typedef struct glb_lb_policy {
   /** base policy: must be first */
   grpc_lb_policy base;
 
@@ -194,9 +333,6 @@ struct glb_lb_policy {
   /** the RR policy to use of the backend servers returned by the LB server */
   grpc_lb_policy* rr_policy;
 
-  grpc_closure on_rr_connectivity_changed;
-  grpc_connectivity_state rr_connectivity_state;
-
   bool started_picking;
 
   /** our connectivity state tracker */
@@ -301,84 +437,14 @@ struct glb_lb_policy {
   grpc_closure client_load_report_closure;
   /* Client load report message payload. */
   grpc_byte_buffer* client_load_report_payload;
-};
-
-/* add lb_token of selected subchannel (address) to the call's initial
- * metadata */
-static grpc_error* initial_metadata_add_lb_token(
-    grpc_metadata_batch* initial_metadata,
-    grpc_linked_mdelem* lb_token_mdelem_storage, grpc_mdelem lb_token) {
-  GPR_ASSERT(lb_token_mdelem_storage != nullptr);
-  GPR_ASSERT(!GRPC_MDISNULL(lb_token));
-  return grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage,
-                                      lb_token);
-}
-
-static void destroy_client_stats(void* arg) {
-  grpc_grpclb_client_stats_unref((grpc_grpclb_client_stats*)arg);
-}
+} glb_lb_policy;
 
-static void pending_pick_set_metadata_and_context(pending_pick* pp) {
-  /* if connected_subchannel is nullptr, no pick has been made by the RR
-   * policy (e.g., all addresses failed to connect). There won't be any
-   * user_data/token available */
-  if (pp->pick->connected_subchannel != nullptr) {
-    if (!GRPC_MDISNULL(pp->lb_token)) {
-      initial_metadata_add_lb_token(pp->pick->initial_metadata,
-                                    &pp->pick->lb_token_mdelem_storage,
-                                    GRPC_MDELEM_REF(pp->lb_token));
-    } else {
-      gpr_log(GPR_ERROR,
-              "[grpclb %p] No LB token for connected subchannel pick %p",
-              pp->glb_policy, pp->pick);
-      abort();
-    }
-    // Pass on client stats via context. Passes ownership of the reference.
-    GPR_ASSERT(pp->client_stats != nullptr);
-    pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].value =
-        pp->client_stats;
-    pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].destroy =
-        destroy_client_stats;
-  } else {
-    grpc_grpclb_client_stats_unref(pp->client_stats);
-  }
-}
-
-/* The \a on_complete closure passed as part of the pick requires keeping a
- * reference to its associated round robin instance. We wrap this closure in
- * order to unref the round robin instance upon its invocation */
-static void pending_pick_complete(void* arg, grpc_error* error) {
-  pending_pick* pp = (pending_pick*)arg;
-  pending_pick_set_metadata_and_context(pp);
-  GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_REF(error));
-  gpr_free(pp);
-}
-
-static pending_pick* pending_pick_create(glb_lb_policy* glb_policy,
-                                         grpc_lb_policy_pick_state* pick) {
-  pending_pick* pp = (pending_pick*)gpr_zalloc(sizeof(*pp));
-  pp->pick = pick;
-  pp->glb_policy = glb_policy;
-  GRPC_CLOSURE_INIT(&pp->on_complete, pending_pick_complete, pp,
-                    grpc_schedule_on_exec_ctx);
-  pp->original_on_complete = pick->on_complete;
-  pp->pick->on_complete = &pp->on_complete;
-  return pp;
-}
-
-static void pending_pick_add(pending_pick** root, pending_pick* new_pp) {
-  new_pp->next = *root;
-  *root = new_pp;
-}
-
-static void pending_ping_add(pending_ping** root, grpc_closure* on_initiate,
-                             grpc_closure* on_ack) {
-  pending_ping* pping = (pending_ping*)gpr_zalloc(sizeof(*pping));
-  pping->on_initiate = on_initiate;
-  pping->on_ack = on_ack;
-  pping->next = *root;
-  *root = pping;
-}
+/* Keeps track and reacts to changes in connectivity of the RR instance */
+struct rr_connectivity_data {
+  grpc_closure on_change;
+  grpc_connectivity_state state;
+  glb_lb_policy* glb_policy;
+};
 
 static bool is_server_valid(const grpc_grpclb_server* server, size_t idx,
                             bool log) {
@@ -491,6 +557,7 @@ static grpc_lb_addresses* process_serverlist_locked(
       gpr_free(uri);
       user_data = (void*)GRPC_MDELEM_LB_TOKEN_EMPTY.payload;
     }
+
     grpc_lb_addresses_set_address(lb_addresses, addr_idx, &addr.addr, addr.len,
                                   false /* is_balancer */,
                                   nullptr /* balancer_name */, user_data);
@@ -531,6 +598,7 @@ static void update_lb_connectivity_status_locked(
     grpc_error* rr_state_error) {
   const grpc_connectivity_state curr_glb_state =
       grpc_connectivity_state_check(&glb_policy->state_tracker);
+
   /* The new connectivity status is a function of the previous one and the new
    * input coming from the status of the RR policy.
    *
@@ -560,6 +628,7 @@ static void update_lb_connectivity_status_locked(
    *
    *  (*) This function mustn't be called during shutting down. */
   GPR_ASSERT(curr_glb_state != GRPC_CHANNEL_SHUTDOWN);
+
   switch (rr_state) {
     case GRPC_CHANNEL_TRANSIENT_FAILURE:
     case GRPC_CHANNEL_SHUTDOWN:
@@ -570,6 +639,7 @@ static void update_lb_connectivity_status_locked(
     case GRPC_CHANNEL_READY:
       GPR_ASSERT(rr_state_error == GRPC_ERROR_NONE);
   }
+
   if (grpc_lb_glb_trace.enabled()) {
     gpr_log(
         GPR_INFO,
@@ -587,8 +657,10 @@ static void update_lb_connectivity_status_locked(
  * cleanups this callback would otherwise be responsible for.
  * If \a force_async is true, then we will manually schedule the
  * completion callback even if the pick is available immediately. */
-static bool pick_from_internal_rr_locked(glb_lb_policy* glb_policy,
-                                         bool force_async, pending_pick* pp) {
+static bool pick_from_internal_rr_locked(
+    glb_lb_policy* glb_policy, const grpc_lb_policy_pick_args* pick_args,
+    bool force_async, grpc_connected_subchannel** target,
+    wrapped_rr_closure_arg* wc_arg) {
   // Check for drops if we are not using fallback backend addresses.
   if (glb_policy->serverlist != nullptr) {
     // Look at the index into the serverlist to see if we should drop this call.
@@ -598,36 +670,57 @@ static bool pick_from_internal_rr_locked(glb_lb_policy* glb_policy,
       glb_policy->serverlist_index = 0;  // Wrap-around.
     }
     if (server->drop) {
+      // Not using the RR policy, so unref it.
+      if (grpc_lb_glb_trace.enabled()) {
+        gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p for drop", glb_policy,
+                wc_arg->rr_policy);
+      }
+      GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "glb_pick_sync");
       // Update client load reporting stats to indicate the number of
       // dropped calls.  Note that we have to do this here instead of in
       // the client_load_reporting filter, because we do not create a
       // subchannel call (and therefore no client_load_reporting filter)
       // for dropped calls.
-      GPR_ASSERT(glb_policy->client_stats != nullptr);
+      GPR_ASSERT(wc_arg->client_stats != nullptr);
       grpc_grpclb_client_stats_add_call_dropped_locked(
-          server->load_balance_token, glb_policy->client_stats);
+          server->load_balance_token, wc_arg->client_stats);
+      grpc_grpclb_client_stats_unref(wc_arg->client_stats);
       if (force_async) {
-        GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE);
-        gpr_free(pp);
+        GPR_ASSERT(wc_arg->wrapped_closure != nullptr);
+        GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_NONE);
+        gpr_free(wc_arg->free_when_done);
         return false;
       }
-      gpr_free(pp);
+      gpr_free(wc_arg->free_when_done);
       return true;
     }
   }
-  // Set client_stats and user_data.
-  pp->client_stats = grpc_grpclb_client_stats_ref(glb_policy->client_stats);
-  GPR_ASSERT(pp->pick->user_data == nullptr);
-  pp->pick->user_data = (void**)&pp->lb_token;
   // Pick via the RR policy.
-  bool pick_done = grpc_lb_policy_pick_locked(glb_policy->rr_policy, pp->pick);
+  const bool pick_done = grpc_lb_policy_pick_locked(
+      wc_arg->rr_policy, pick_args, target, wc_arg->context,
+      (void**)&wc_arg->lb_token, &wc_arg->wrapper_closure);
   if (pick_done) {
-    pending_pick_set_metadata_and_context(pp);
+    /* synchronous grpc_lb_policy_pick call. Unref the RR policy. */
+    if (grpc_lb_glb_trace.enabled()) {
+      gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p", glb_policy,
+              wc_arg->rr_policy);
+    }
+    GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "glb_pick_sync");
+    /* add the load reporting initial metadata */
+    initial_metadata_add_lb_token(pick_args->initial_metadata,
+                                  pick_args->lb_token_mdelem_storage,
+                                  GRPC_MDELEM_REF(wc_arg->lb_token));
+    // Pass on client stats via context. Passes ownership of the reference.
+    GPR_ASSERT(wc_arg->client_stats != nullptr);
+    wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].value = wc_arg->client_stats;
+    wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].destroy = destroy_client_stats;
     if (force_async) {
-      GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE);
-      pick_done = false;
+      GPR_ASSERT(wc_arg->wrapped_closure != nullptr);
+      GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_NONE);
+      gpr_free(wc_arg->free_when_done);
+      return false;
     }
-    gpr_free(pp);
+    gpr_free(wc_arg->free_when_done);
   }
   /* else, the pending pick will be registered and taken care of by the
    * pending pick list inside the RR policy (glb_policy->rr_policy).
@@ -669,7 +762,7 @@ static void lb_policy_args_destroy(grpc_lb_policy_args* args) {
   gpr_free(args);
 }
 
-static void on_rr_connectivity_changed_locked(void* arg, grpc_error* error);
+static void glb_rr_connectivity_changed_locked(void* arg, grpc_error* error);
 static void create_rr_locked(glb_lb_policy* glb_policy,
                              grpc_lb_policy_args* args) {
   GPR_ASSERT(glb_policy->rr_policy == nullptr);
@@ -691,46 +784,72 @@ static void create_rr_locked(glb_lb_policy* glb_policy,
   glb_policy->base.request_reresolution = nullptr;
   glb_policy->rr_policy = new_rr_policy;
   grpc_error* rr_state_error = nullptr;
-  glb_policy->rr_connectivity_state = grpc_lb_policy_check_connectivity_locked(
-      glb_policy->rr_policy, &rr_state_error);
+  const grpc_connectivity_state rr_state =
+      grpc_lb_policy_check_connectivity_locked(glb_policy->rr_policy,
+                                               &rr_state_error);
   /* Connectivity state is a function of the RR policy updated/created */
-  update_lb_connectivity_status_locked(
-      glb_policy, glb_policy->rr_connectivity_state, rr_state_error);
+  update_lb_connectivity_status_locked(glb_policy, rr_state, rr_state_error);
   /* Add the gRPC LB's interested_parties pollset_set to that of the newly
    * created RR policy. This will make the RR policy progress upon activity on
    * gRPC LB, which in turn is tied to the application's call */
   grpc_pollset_set_add_pollset_set(glb_policy->rr_policy->interested_parties,
                                    glb_policy->base.interested_parties);
-  GRPC_CLOSURE_INIT(&glb_policy->on_rr_connectivity_changed,
-                    on_rr_connectivity_changed_locked, glb_policy,
+
+  /* Allocate the data for the tracking of the new RR policy's connectivity.
+   * It'll be deallocated in glb_rr_connectivity_changed() */
+  rr_connectivity_data* rr_connectivity =
+      (rr_connectivity_data*)gpr_zalloc(sizeof(rr_connectivity_data));
+  GRPC_CLOSURE_INIT(&rr_connectivity->on_change,
+                    glb_rr_connectivity_changed_locked, rr_connectivity,
                     grpc_combiner_scheduler(glb_policy->base.combiner));
+  rr_connectivity->glb_policy = glb_policy;
+  rr_connectivity->state = rr_state;
+
   /* Subscribe to changes to the connectivity of the new RR */
-  GRPC_LB_POLICY_REF(&glb_policy->base, "glb_rr_connectivity_cb");
-  grpc_lb_policy_notify_on_state_change_locked(
-      glb_policy->rr_policy, &glb_policy->rr_connectivity_state,
-      &glb_policy->on_rr_connectivity_changed);
+  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "glb_rr_connectivity_cb");
+  grpc_lb_policy_notify_on_state_change_locked(glb_policy->rr_policy,
+                                               &rr_connectivity->state,
+                                               &rr_connectivity->on_change);
   grpc_lb_policy_exit_idle_locked(glb_policy->rr_policy);
-  // Send pending picks to RR policy.
+
+  /* Update picks and pings in wait */
   pending_pick* pp;
   while ((pp = glb_policy->pending_picks)) {
     glb_policy->pending_picks = pp->next;
+    GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_pick");
+    pp->wrapped_on_complete_arg.rr_policy = glb_policy->rr_policy;
+    pp->wrapped_on_complete_arg.client_stats =
+        grpc_grpclb_client_stats_ref(glb_policy->client_stats);
     if (grpc_lb_glb_trace.enabled()) {
       gpr_log(GPR_INFO,
               "[grpclb %p] Pending pick about to (async) PICK from RR %p",
               glb_policy, glb_policy->rr_policy);
     }
-    pick_from_internal_rr_locked(glb_policy, true /* force_async */, pp);
+    pick_from_internal_rr_locked(glb_policy, &pp->pick_args,
+                                 true /* force_async */, pp->target,
+                                 &pp->wrapped_on_complete_arg);
   }
-  // Send pending pings to RR policy.
+
   pending_ping* pping;
   while ((pping = glb_policy->pending_pings)) {
     glb_policy->pending_pings = pping->next;
+    grpc_closure* on_initiate = nullptr;
+    grpc_closure* on_ack = nullptr;
+    if (pping->on_initiate != nullptr) {
+      GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_ping");
+      pping->on_initiate->rr_policy = glb_policy->rr_policy;
+      on_initiate = &pping->on_initiate->wrapper_closure;
+    }
+    if (pping->on_ack != nullptr) {
+      GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_ping");
+      pping->on_ack->rr_policy = glb_policy->rr_policy;
+      on_ack = &pping->on_ack->wrapper_closure;
+    }
     if (grpc_lb_glb_trace.enabled()) {
       gpr_log(GPR_INFO, "[grpclb %p] Pending ping about to PING from RR %p",
               glb_policy, glb_policy->rr_policy);
     }
-    grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, pping->on_initiate,
-                                   pping->on_ack);
+    grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, on_initiate, on_ack);
     gpr_free(pping);
   }
 }
@@ -756,28 +875,31 @@ static void rr_handover_locked(glb_lb_policy* glb_policy) {
   lb_policy_args_destroy(args);
 }
 
-static void on_rr_connectivity_changed_locked(void* arg, grpc_error* error) {
-  glb_lb_policy* glb_policy = (glb_lb_policy*)arg;
+static void glb_rr_connectivity_changed_locked(void* arg, grpc_error* error) {
+  rr_connectivity_data* rr_connectivity = (rr_connectivity_data*)arg;
+  glb_lb_policy* glb_policy = rr_connectivity->glb_policy;
   if (glb_policy->shutting_down) {
-    GRPC_LB_POLICY_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
+    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
+    gpr_free(rr_connectivity);
     return;
   }
-  if (glb_policy->rr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
+  if (rr_connectivity->state == GRPC_CHANNEL_SHUTDOWN) {
     /* An RR policy that has transitioned into the SHUTDOWN connectivity state
      * should not be considered for picks or updates: the SHUTDOWN state is a
      * sink, policies can't transition back from it. .*/
     GRPC_LB_POLICY_UNREF(glb_policy->rr_policy, "rr_connectivity_shutdown");
     glb_policy->rr_policy = nullptr;
-    GRPC_LB_POLICY_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
+    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
+    gpr_free(rr_connectivity);
     return;
   }
   /* rr state != SHUTDOWN && !glb_policy->shutting down: biz as usual */
-  update_lb_connectivity_status_locked(
-      glb_policy, glb_policy->rr_connectivity_state, GRPC_ERROR_REF(error));
-  /* Resubscribe. Reuse the "glb_rr_connectivity_cb" ref. */
-  grpc_lb_policy_notify_on_state_change_locked(
-      glb_policy->rr_policy, &glb_policy->rr_connectivity_state,
-      &glb_policy->on_rr_connectivity_changed);
+  update_lb_connectivity_status_locked(glb_policy, rr_connectivity->state,
+                                       GRPC_ERROR_REF(error));
+  /* Resubscribe. Reuse the "glb_rr_connectivity_cb" weak ref. */
+  grpc_lb_policy_notify_on_state_change_locked(glb_policy->rr_policy,
+                                               &rr_connectivity->state,
+                                               &rr_connectivity->on_change);
 }
 
 static void destroy_balancer_name(void* balancer_name) {
@@ -885,17 +1007,22 @@ static void glb_destroy(grpc_lb_policy* pol) {
   gpr_free(glb_policy);
 }
 
-static void glb_shutdown_locked(grpc_lb_policy* pol,
-                                grpc_lb_policy* new_policy) {
+static void glb_shutdown_locked(grpc_lb_policy* pol) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)pol;
   grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   glb_policy->shutting_down = true;
+
+  /* We need a copy of the lb_call pointer because we can't cancell the call
+   * while holding glb_policy->mu: lb_on_server_status_received, invoked due to
+   * the cancel, needs to acquire that same lock */
+  grpc_call* lb_call = glb_policy->lb_call;
+
   /* glb_policy->lb_call and this local lb_call must be consistent at this point
    * because glb_policy->lb_call is only assigned in lb_call_init_locked as part
    * of query_for_backends_locked, which can only be invoked while
    * glb_policy->shutting_down is false. */
-  if (glb_policy->lb_call != nullptr) {
-    grpc_call_cancel(glb_policy->lb_call, nullptr);
+  if (lb_call != nullptr) {
+    grpc_call_cancel(lb_call, nullptr);
     /* lb_on_server_status_received will pick up the cancel and clean up */
   }
   if (glb_policy->retry_timer_callback_pending) {
@@ -904,8 +1031,12 @@ static void glb_shutdown_locked(grpc_lb_policy* pol,
   if (glb_policy->fallback_timer_callback_pending) {
     grpc_timer_cancel(&glb_policy->lb_fallback_timer);
   }
+
+  pending_pick* pp = glb_policy->pending_picks;
+  glb_policy->pending_picks = nullptr;
+  pending_ping* pping = glb_policy->pending_pings;
+  glb_policy->pending_pings = nullptr;
   if (glb_policy->rr_policy != nullptr) {
-    grpc_lb_policy_shutdown_locked(glb_policy->rr_policy, nullptr);
     GRPC_LB_POLICY_UNREF(glb_policy->rr_policy, "glb_shutdown");
   } else {
     grpc_lb_policy_try_reresolve(pol, &grpc_lb_glb_trace, GRPC_ERROR_CANCELLED);
@@ -920,33 +1051,28 @@ static void glb_shutdown_locked(grpc_lb_policy* pol,
   }
   grpc_connectivity_state_set(&glb_policy->state_tracker, GRPC_CHANNEL_SHUTDOWN,
                               GRPC_ERROR_REF(error), "glb_shutdown");
-  // Clear pending picks.
-  pending_pick* pp = glb_policy->pending_picks;
-  glb_policy->pending_picks = nullptr;
+
   while (pp != nullptr) {
     pending_pick* next = pp->next;
-    if (new_policy != nullptr) {
-      // Hand pick over to new policy.
-      grpc_grpclb_client_stats_unref(pp->client_stats);
-      pp->pick->on_complete = pp->original_on_complete;
-      if (grpc_lb_policy_pick_locked(new_policy, pp->pick)) {
-        // Synchronous return; schedule callback.
-        GRPC_CLOSURE_SCHED(pp->pick->on_complete, GRPC_ERROR_NONE);
-      }
-      gpr_free(pp);
-    } else {
-      pp->pick->connected_subchannel = nullptr;
-      GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_REF(error));
-    }
+    *pp->target = nullptr;
+    GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure,
+                       GRPC_ERROR_REF(error));
+    gpr_free(pp);
     pp = next;
   }
-  // Clear pending pings.
-  pending_ping* pping = glb_policy->pending_pings;
-  glb_policy->pending_pings = nullptr;
+
   while (pping != nullptr) {
     pending_ping* next = pping->next;
-    GRPC_CLOSURE_SCHED(pping->on_initiate, GRPC_ERROR_REF(error));
-    GRPC_CLOSURE_SCHED(pping->on_ack, GRPC_ERROR_REF(error));
+    if (pping->on_initiate != nullptr) {
+      GRPC_CLOSURE_SCHED(&pping->on_initiate->wrapper_closure,
+                         GRPC_ERROR_REF(error));
+      gpr_free(pping->on_initiate);
+    }
+    if (pping->on_ack != nullptr) {
+      GRPC_CLOSURE_SCHED(&pping->on_ack->wrapper_closure,
+                         GRPC_ERROR_REF(error));
+      gpr_free(pping->on_ack);
+    }
     gpr_free(pping);
     pping = next;
   }
@@ -964,16 +1090,16 @@ static void glb_shutdown_locked(grpc_lb_policy* pol,
 //   level (grpclb), inside the glb_policy->pending_picks list. To cancel these,
 //   we invoke the completion closure and set *target to nullptr right here.
 static void glb_cancel_pick_locked(grpc_lb_policy* pol,
-                                   grpc_lb_policy_pick_state* pick,
+                                   grpc_connected_subchannel** target,
                                    grpc_error* error) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)pol;
   pending_pick* pp = glb_policy->pending_picks;
   glb_policy->pending_picks = nullptr;
   while (pp != nullptr) {
     pending_pick* next = pp->next;
-    if (pp->pick == pick) {
-      pick->connected_subchannel = nullptr;
-      GRPC_CLOSURE_SCHED(&pp->on_complete,
+    if (pp->target == target) {
+      *target = nullptr;
+      GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
     } else {
@@ -983,7 +1109,7 @@ static void glb_cancel_pick_locked(grpc_lb_policy* pol,
     pp = next;
   }
   if (glb_policy->rr_policy != nullptr) {
-    grpc_lb_policy_cancel_pick_locked(glb_policy->rr_policy, pick,
+    grpc_lb_policy_cancel_pick_locked(glb_policy->rr_policy, target,
                                       GRPC_ERROR_REF(error));
   }
   GRPC_ERROR_UNREF(error);
@@ -1008,9 +1134,9 @@ static void glb_cancel_picks_locked(grpc_lb_policy* pol,
   glb_policy->pending_picks = nullptr;
   while (pp != nullptr) {
     pending_pick* next = pp->next;
-    if ((pp->pick->initial_metadata_flags & initial_metadata_flags_mask) ==
+    if ((pp->pick_args.initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      GRPC_CLOSURE_SCHED(&pp->on_complete,
+      GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
     } else {
@@ -1036,7 +1162,7 @@ static void start_picking_locked(glb_lb_policy* glb_policy) {
       !glb_policy->fallback_timer_callback_pending) {
     grpc_millis deadline =
         grpc_core::ExecCtx::Get()->Now() + glb_policy->lb_fallback_timeout_ms;
-    GRPC_LB_POLICY_REF(&glb_policy->base, "grpclb_fallback_timer");
+    GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer");
     GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked,
                       glb_policy,
                       grpc_combiner_scheduler(glb_policy->base.combiner));
@@ -1058,9 +1184,19 @@ static void glb_exit_idle_locked(grpc_lb_policy* pol) {
 }
 
 static int glb_pick_locked(grpc_lb_policy* pol,
-                           grpc_lb_policy_pick_state* pick) {
+                           const grpc_lb_policy_pick_args* pick_args,
+                           grpc_connected_subchannel** target,
+                           grpc_call_context_element* context, void** user_data,
+                           grpc_closure* on_complete) {
+  if (pick_args->lb_token_mdelem_storage == nullptr) {
+    *target = nullptr;
+    GRPC_CLOSURE_SCHED(on_complete,
+                       GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                           "No mdelem storage for the LB token. Load reporting "
+                           "won't work without it. Failing"));
+    return 0;
+  }
   glb_lb_policy* glb_policy = (glb_lb_policy*)pol;
-  pending_pick* pp = pending_pick_create(glb_policy, pick);
   bool pick_done = false;
   if (glb_policy->rr_policy != nullptr) {
     const grpc_connectivity_state rr_connectivity_state =
@@ -1068,7 +1204,7 @@ static int glb_pick_locked(grpc_lb_policy* pol,
                                                  nullptr);
     // The glb_policy->rr_policy may have transitioned to SHUTDOWN but the
     // callback registered to capture this event
-    // (on_rr_connectivity_changed_locked) may not have been invoked yet. We
+    // (glb_rr_connectivity_changed_locked) may not have been invoked yet. We
     // need to make sure we aren't trying to pick from a RR policy instance
     // that's in shutdown.
     if (rr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
@@ -1078,16 +1214,32 @@ static int glb_pick_locked(grpc_lb_policy* pol,
                 glb_policy, glb_policy->rr_policy,
                 grpc_connectivity_state_name(rr_connectivity_state));
       }
-      pending_pick_add(&glb_policy->pending_picks, pp);
+      add_pending_pick(&glb_policy->pending_picks, pick_args, target, context,
+                       on_complete);
       pick_done = false;
     } else {  // RR not in shutdown
       if (grpc_lb_glb_trace.enabled()) {
         gpr_log(GPR_INFO, "[grpclb %p] about to PICK from RR %p", glb_policy,
                 glb_policy->rr_policy);
       }
+      GRPC_LB_POLICY_REF(glb_policy->rr_policy, "glb_pick");
+      wrapped_rr_closure_arg* wc_arg =
+          (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(wrapped_rr_closure_arg));
+      GRPC_CLOSURE_INIT(&wc_arg->wrapper_closure, wrapped_rr_closure, wc_arg,
+                        grpc_schedule_on_exec_ctx);
+      wc_arg->rr_policy = glb_policy->rr_policy;
+      wc_arg->target = target;
+      wc_arg->context = context;
       GPR_ASSERT(glb_policy->client_stats != nullptr);
-      pick_done =
-          pick_from_internal_rr_locked(glb_policy, false /* force_async */, pp);
+      wc_arg->client_stats =
+          grpc_grpclb_client_stats_ref(glb_policy->client_stats);
+      wc_arg->wrapped_closure = on_complete;
+      wc_arg->lb_token_mdelem_storage = pick_args->lb_token_mdelem_storage;
+      wc_arg->initial_metadata = pick_args->initial_metadata;
+      wc_arg->free_when_done = wc_arg;
+      wc_arg->glb_policy = pol;
+      pick_done = pick_from_internal_rr_locked(
+          glb_policy, pick_args, false /* force_async */, target, wc_arg);
     }
   } else {  // glb_policy->rr_policy == NULL
     if (grpc_lb_glb_trace.enabled()) {
@@ -1095,7 +1247,8 @@ static int glb_pick_locked(grpc_lb_policy* pol,
               "[grpclb %p] No RR policy. Adding to grpclb's pending picks",
               glb_policy);
     }
-    pending_pick_add(&glb_policy->pending_picks, pp);
+    add_pending_pick(&glb_policy->pending_picks, pick_args, target, context,
+                     on_complete);
     if (!glb_policy->started_picking) {
       start_picking_locked(glb_policy);
     }
@@ -1117,7 +1270,7 @@ static void glb_ping_one_locked(grpc_lb_policy* pol, grpc_closure* on_initiate,
   if (glb_policy->rr_policy) {
     grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, on_initiate, on_ack);
   } else {
-    pending_ping_add(&glb_policy->pending_pings, on_initiate, on_ack);
+    add_pending_ping(&glb_policy->pending_pings, on_initiate, on_ack);
     if (!glb_policy->started_picking) {
       start_picking_locked(glb_policy);
     }
@@ -1142,7 +1295,7 @@ static void lb_call_on_retry_timer_locked(void* arg, grpc_error* error) {
     }
     query_for_backends_locked(glb_policy);
   }
-  GRPC_LB_POLICY_UNREF(&glb_policy->base, "grpclb_retry_timer");
+  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "grpclb_retry_timer");
 }
 
 static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
@@ -1168,7 +1321,7 @@ static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
                 glb_policy);
       }
     }
-    GRPC_LB_POLICY_REF(&glb_policy->base, "grpclb_retry_timer");
+    GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_retry_timer");
     GRPC_CLOSURE_INIT(&glb_policy->lb_on_call_retry,
                       lb_call_on_retry_timer_locked, glb_policy,
                       grpc_combiner_scheduler(glb_policy->base.combiner));
@@ -1176,8 +1329,8 @@ static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
     grpc_timer_init(&glb_policy->lb_call_retry_timer, next_try,
                     &glb_policy->lb_on_call_retry);
   }
-  GRPC_LB_POLICY_UNREF(&glb_policy->base,
-                       "lb_on_server_status_received_locked");
+  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
+                            "lb_on_server_status_received_locked");
 }
 
 static void send_client_load_report_locked(void* arg, grpc_error* error);
@@ -1200,7 +1353,7 @@ static void client_load_report_done_locked(void* arg, grpc_error* error) {
   glb_policy->client_load_report_payload = nullptr;
   if (error != GRPC_ERROR_NONE || glb_policy->lb_call == nullptr) {
     glb_policy->client_load_report_timer_callback_pending = false;
-    GRPC_LB_POLICY_UNREF(&glb_policy->base, "client_load_report");
+    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "client_load_report");
     if (glb_policy->lb_call == nullptr) {
       maybe_restart_lb_call(glb_policy);
     }
@@ -1241,7 +1394,7 @@ static void send_client_load_report_locked(void* arg, grpc_error* error) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)arg;
   if (error == GRPC_ERROR_CANCELLED || glb_policy->lb_call == nullptr) {
     glb_policy->client_load_report_timer_callback_pending = false;
-    GRPC_LB_POLICY_UNREF(&glb_policy->base, "client_load_report");
+    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "client_load_report");
     if (glb_policy->lb_call == nullptr) {
       maybe_restart_lb_call(glb_policy);
     }
@@ -1394,8 +1547,10 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) {
   op->flags = 0;
   op->reserved = nullptr;
   op++;
-  /* take a ref to be released in lb_on_sent_initial_request_locked() */
-  GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_sent_initial_request_locked");
+  /* take a weak ref (won't prevent calling of \a glb_shutdown if the strong ref
+   * count goes to zero) to be unref'd in lb_on_sent_initial_request_locked() */
+  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base,
+                          "lb_on_sent_initial_request_locked");
   call_error = grpc_call_start_batch_and_execute(
       glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_sent_initial_request);
@@ -1411,8 +1566,10 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) {
   op->flags = 0;
   op->reserved = nullptr;
   op++;
-  /* take a ref to be released in lb_on_server_status_received_locked() */
-  GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_server_status_received_locked");
+  /* take a weak ref (won't prevent calling of \a glb_shutdown if the strong ref
+   * count goes to zero) to be unref'd in lb_on_server_status_received_locked */
+  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base,
+                          "lb_on_server_status_received_locked");
   call_error = grpc_call_start_batch_and_execute(
       glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_server_status_received);
@@ -1424,8 +1581,9 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) {
   op->flags = 0;
   op->reserved = nullptr;
   op++;
-  /* take a ref to be unref'd/reused in lb_on_response_received_locked() */
-  GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_response_received_locked");
+  /* take another weak ref to be unref'd/reused in
+   * lb_on_response_received_locked */
+  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "lb_on_response_received_locked");
   call_error = grpc_call_start_batch_and_execute(
       glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_response_received);
@@ -1440,7 +1598,8 @@ static void lb_on_sent_initial_request_locked(void* arg, grpc_error* error) {
   if (glb_policy->client_load_report_payload != nullptr) {
     do_send_client_load_report_locked(glb_policy);
   }
-  GRPC_LB_POLICY_UNREF(&glb_policy->base, "lb_on_sent_initial_request_locked");
+  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
+                            "lb_on_sent_initial_request_locked");
 }
 
 static void lb_on_response_received_locked(void* arg, grpc_error* error) {
@@ -1472,9 +1631,11 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) {
                   "client load reporting interval = %" PRIdPTR " milliseconds",
                   glb_policy, glb_policy->client_stats_report_interval);
         }
-        /* take a ref to be unref'd in send_client_load_report_locked() */
+        /* take a weak ref (won't prevent calling of \a glb_shutdown() if the
+         * strong ref count goes to zero) to be unref'd in
+         * send_client_load_report_locked() */
         glb_policy->client_load_report_timer_callback_pending = true;
-        GRPC_LB_POLICY_REF(&glb_policy->base, "client_load_report");
+        GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "client_load_report");
         schedule_next_client_load_report(glb_policy);
       } else if (grpc_lb_glb_trace.enabled()) {
         gpr_log(GPR_INFO,
@@ -1556,21 +1717,21 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) {
       op->flags = 0;
       op->reserved = nullptr;
       op++;
-      /* reuse the "lb_on_response_received_locked" ref taken in
+      /* reuse the "lb_on_response_received_locked" weak ref taken in
        * query_for_backends_locked() */
       const grpc_call_error call_error = grpc_call_start_batch_and_execute(
           glb_policy->lb_call, ops, (size_t)(op - ops),
           &glb_policy->lb_on_response_received); /* loop */
       GPR_ASSERT(GRPC_CALL_OK == call_error);
     } else {
-      GRPC_LB_POLICY_UNREF(&glb_policy->base,
-                           "lb_on_response_received_locked_shutdown");
+      GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
+                                "lb_on_response_received_locked_shutdown");
     }
   } else { /* empty payload: call cancelled. */
-           /* dispose of the "lb_on_response_received_locked" ref taken in
+           /* dispose of the "lb_on_response_received_locked" weak ref taken in
             * query_for_backends_locked() and reused in every reception loop */
-    GRPC_LB_POLICY_UNREF(&glb_policy->base,
-                         "lb_on_response_received_locked_empty_payload");
+    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
+                              "lb_on_response_received_locked_empty_payload");
   }
 }
 
@@ -1590,7 +1751,7 @@ static void lb_on_fallback_timer_locked(void* arg, grpc_error* error) {
       rr_handover_locked(glb_policy);
     }
   }
-  GRPC_LB_POLICY_UNREF(&glb_policy->base, "grpclb_fallback_timer");
+  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "grpclb_fallback_timer");
 }
 
 static void lb_on_server_status_received_locked(void* arg, grpc_error* error) {
@@ -1674,7 +1835,7 @@ static void glb_update_locked(grpc_lb_policy* policy,
         grpc_channel_get_channel_stack(glb_policy->lb_channel));
     GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter);
     glb_policy->watching_lb_channel = true;
-    GRPC_LB_POLICY_REF(&glb_policy->base, "watch_lb_channel_connectivity");
+    GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "watch_lb_channel_connectivity");
     grpc_client_channel_watch_connectivity_state(
         client_channel_elem,
         grpc_polling_entity_create_from_pollset_set(
@@ -1730,8 +1891,8 @@ static void glb_lb_channel_on_connectivity_changed_cb(void* arg,
     case GRPC_CHANNEL_SHUTDOWN:
     done:
       glb_policy->watching_lb_channel = false;
-      GRPC_LB_POLICY_UNREF(&glb_policy->base,
-                           "watch_lb_channel_connectivity_cb_shutdown");
+      GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
+                                "watch_lb_channel_connectivity_cb_shutdown");
       break;
   }
 }
diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
index 60385272cf5..9ff40aa53c0 100644
--- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
@@ -31,6 +31,15 @@
 
 grpc_core::TraceFlag grpc_lb_pick_first_trace(false, "pick_first");
 
+namespace {
+struct pending_pick {
+  struct pending_pick* next;
+  uint32_t initial_metadata_flags;
+  grpc_connected_subchannel** target;
+  grpc_closure* on_complete;
+};
+}  // namespace
+
 typedef struct {
   /** base policy: must be first */
   grpc_lb_policy base;
@@ -45,7 +54,7 @@ typedef struct {
   /** are we shut down? */
   bool shutdown;
   /** list of picks that are waiting on connectivity */
-  grpc_lb_policy_pick_state* pending_picks;
+  pending_pick* pending_picks;
   /** our connectivity state tracker */
   grpc_connectivity_state_tracker state_tracker;
 } pick_first_lb_policy;
@@ -63,27 +72,19 @@ static void pf_destroy(grpc_lb_policy* pol) {
   }
 }
 
-static void pf_shutdown_locked(grpc_lb_policy* pol,
-                               grpc_lb_policy* new_policy) {
+static void pf_shutdown_locked(grpc_lb_policy* pol) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
   grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   if (grpc_lb_pick_first_trace.enabled()) {
     gpr_log(GPR_DEBUG, "Pick First %p Shutting down", p);
   }
   p->shutdown = true;
-  grpc_lb_policy_pick_state* pick;
-  while ((pick = p->pending_picks) != nullptr) {
-    p->pending_picks = pick->next;
-    if (new_policy != nullptr) {
-      // Hand off to new LB policy.
-      if (grpc_lb_policy_pick_locked(new_policy, pick)) {
-        // Synchronous return, schedule closure.
-        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
-      }
-    } else {
-      pick->connected_subchannel = nullptr;
-      GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error));
-    }
+  pending_pick* pp;
+  while ((pp = p->pending_picks) != nullptr) {
+    p->pending_picks = pp->next;
+    *pp->target = nullptr;
+    GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_REF(error));
+    gpr_free(pp);
   }
   grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
                               GRPC_ERROR_REF(error), "shutdown");
@@ -103,18 +104,19 @@ static void pf_shutdown_locked(grpc_lb_policy* pol,
 }
 
 static void pf_cancel_pick_locked(grpc_lb_policy* pol,
-                                  grpc_lb_policy_pick_state* pick,
+                                  grpc_connected_subchannel** target,
                                   grpc_error* error) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
-  grpc_lb_policy_pick_state* pp = p->pending_picks;
+  pending_pick* pp = p->pending_picks;
   p->pending_picks = nullptr;
   while (pp != nullptr) {
-    grpc_lb_policy_pick_state* next = pp->next;
-    if (pp == pick) {
-      pick->connected_subchannel = nullptr;
-      GRPC_CLOSURE_SCHED(pick->on_complete,
+    pending_pick* next = pp->next;
+    if (pp->target == target) {
+      *target = nullptr;
+      GRPC_CLOSURE_SCHED(pp->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
+      gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
       p->pending_picks = pp;
@@ -129,20 +131,21 @@ static void pf_cancel_picks_locked(grpc_lb_policy* pol,
                                    uint32_t initial_metadata_flags_eq,
                                    grpc_error* error) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
-  grpc_lb_policy_pick_state* pick = p->pending_picks;
+  pending_pick* pp = p->pending_picks;
   p->pending_picks = nullptr;
-  while (pick != nullptr) {
-    grpc_lb_policy_pick_state* next = pick->next;
-    if ((pick->initial_metadata_flags & initial_metadata_flags_mask) ==
+  while (pp != nullptr) {
+    pending_pick* next = pp->next;
+    if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      GRPC_CLOSURE_SCHED(pick->on_complete,
+      GRPC_CLOSURE_SCHED(pp->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
+      gpr_free(pp);
     } else {
-      pick->next = p->pending_picks;
-      p->pending_picks = pick;
+      pp->next = p->pending_picks;
+      p->pending_picks = pp;
     }
-    pick = next;
+    pp = next;
   }
   GRPC_ERROR_UNREF(error);
 }
@@ -172,20 +175,27 @@ static void pf_exit_idle_locked(grpc_lb_policy* pol) {
 }
 
 static int pf_pick_locked(grpc_lb_policy* pol,
-                          grpc_lb_policy_pick_state* pick) {
+                          const grpc_lb_policy_pick_args* pick_args,
+                          grpc_connected_subchannel** target,
+                          grpc_call_context_element* context, void** user_data,
+                          grpc_closure* on_complete) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
   // If we have a selected subchannel already, return synchronously.
   if (p->selected != nullptr) {
-    pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
-        p->selected->connected_subchannel, "picked");
+    *target = GRPC_CONNECTED_SUBCHANNEL_REF(p->selected->connected_subchannel,
+                                            "picked");
     return 1;
   }
   // No subchannel selected yet, so handle asynchronously.
   if (!p->started_picking) {
     start_picking_locked(p);
   }
-  pick->next = p->pending_picks;
-  p->pending_picks = pick;
+  pending_pick* pp = (pending_pick*)gpr_malloc(sizeof(*pp));
+  pp->next = p->pending_picks;
+  pp->target = target;
+  pp->initial_metadata_flags = pick_args->initial_metadata_flags;
+  pp->on_complete = on_complete;
+  p->pending_picks = pp;
   return 0;
 }
 
@@ -471,17 +481,18 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
       // Drop all other subchannels, since we are now connected.
       destroy_unselected_subchannels_locked(p);
       // Update any calls that were waiting for a pick.
-      grpc_lb_policy_pick_state* pick;
-      while ((pick = p->pending_picks)) {
-        p->pending_picks = pick->next;
-        pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
+      pending_pick* pp;
+      while ((pp = p->pending_picks)) {
+        p->pending_picks = pp->next;
+        *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF(
             p->selected->connected_subchannel, "picked");
         if (grpc_lb_pick_first_trace.enabled()) {
           gpr_log(GPR_INFO,
                   "Servicing pending pick with selected subchannel %p",
                   (void*)p->selected);
         }
-        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
+        GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_NONE);
+        gpr_free(pp);
       }
       // Renew notification.
       grpc_lb_subchannel_data_start_connectivity_watch(sd);
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
index 92c7d5bd5d0..a964af06270 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
@@ -41,6 +41,31 @@
 
 grpc_core::TraceFlag grpc_lb_round_robin_trace(false, "round_robin");
 
+namespace {
+/** List of entities waiting for a pick.
+ *
+ * Once a pick is available, \a target is updated and \a on_complete called. */
+struct pending_pick {
+  pending_pick* next;
+
+  /* output argument where to store the pick()ed user_data. It'll be NULL if no
+   * such data is present or there's an error (the definite test for errors is
+   * \a target being NULL). */
+  void** user_data;
+
+  /* bitmask passed to pick() and used for selective cancelling. See
+   * grpc_lb_policy_cancel_picks() */
+  uint32_t initial_metadata_flags;
+
+  /* output argument where to store the pick()ed connected subchannel, or NULL
+   * upon error. */
+  grpc_connected_subchannel** target;
+
+  /* to be invoked once the pick() has completed (regardless of success) */
+  grpc_closure* on_complete;
+};
+}  // namespace
+
 typedef struct round_robin_lb_policy {
   /** base policy: must be first */
   grpc_lb_policy base;
@@ -52,7 +77,7 @@ typedef struct round_robin_lb_policy {
   /** are we shutting down? */
   bool shutdown;
   /** List of picks that are waiting on connectivity */
-  grpc_lb_policy_pick_state* pending_picks;
+  pending_pick* pending_picks;
 
   /** our connectivity state tracker */
   grpc_connectivity_state_tracker state_tracker;
@@ -144,27 +169,19 @@ static void rr_destroy(grpc_lb_policy* pol) {
   gpr_free(p);
 }
 
-static void rr_shutdown_locked(grpc_lb_policy* pol,
-                               grpc_lb_policy* new_policy) {
+static void rr_shutdown_locked(grpc_lb_policy* pol) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
   grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   if (grpc_lb_round_robin_trace.enabled()) {
     gpr_log(GPR_DEBUG, "[RR %p] Shutting down", p);
   }
   p->shutdown = true;
-  grpc_lb_policy_pick_state* pick;
-  while ((pick = p->pending_picks) != nullptr) {
-    p->pending_picks = pick->next;
-    if (new_policy != nullptr) {
-      // Hand off to new LB policy.
-      if (grpc_lb_policy_pick_locked(new_policy, pick)) {
-        // Synchronous return; schedule callback.
-        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
-      }
-    } else {
-      pick->connected_subchannel = nullptr;
-      GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error));
-    }
+  pending_pick* pp;
+  while ((pp = p->pending_picks) != nullptr) {
+    p->pending_picks = pp->next;
+    *pp->target = nullptr;
+    GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_REF(error));
+    gpr_free(pp);
   }
   grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
                               GRPC_ERROR_REF(error), "rr_shutdown");
@@ -184,18 +201,19 @@ static void rr_shutdown_locked(grpc_lb_policy* pol,
 }
 
 static void rr_cancel_pick_locked(grpc_lb_policy* pol,
-                                  grpc_lb_policy_pick_state* pick,
+                                  grpc_connected_subchannel** target,
                                   grpc_error* error) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
-  grpc_lb_policy_pick_state* pp = p->pending_picks;
+  pending_pick* pp = p->pending_picks;
   p->pending_picks = nullptr;
   while (pp != nullptr) {
-    grpc_lb_policy_pick_state* next = pp->next;
-    if (pp == pick) {
-      pick->connected_subchannel = nullptr;
-      GRPC_CLOSURE_SCHED(pick->on_complete,
+    pending_pick* next = pp->next;
+    if (pp->target == target) {
+      *target = nullptr;
+      GRPC_CLOSURE_SCHED(pp->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick cancelled", &error, 1));
+      gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
       p->pending_picks = pp;
@@ -210,21 +228,22 @@ static void rr_cancel_picks_locked(grpc_lb_policy* pol,
                                    uint32_t initial_metadata_flags_eq,
                                    grpc_error* error) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
-  grpc_lb_policy_pick_state* pick = p->pending_picks;
+  pending_pick* pp = p->pending_picks;
   p->pending_picks = nullptr;
-  while (pick != nullptr) {
-    grpc_lb_policy_pick_state* next = pick->next;
-    if ((pick->initial_metadata_flags & initial_metadata_flags_mask) ==
+  while (pp != nullptr) {
+    pending_pick* next = pp->next;
+    if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      pick->connected_subchannel = nullptr;
-      GRPC_CLOSURE_SCHED(pick->on_complete,
+      *pp->target = nullptr;
+      GRPC_CLOSURE_SCHED(pp->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick cancelled", &error, 1));
+      gpr_free(pp);
     } else {
-      pick->next = p->pending_picks;
-      p->pending_picks = pick;
+      pp->next = p->pending_picks;
+      p->pending_picks = pp;
     }
-    pick = next;
+    pp = next;
   }
   GRPC_ERROR_UNREF(error);
 }
@@ -249,10 +268,13 @@ static void rr_exit_idle_locked(grpc_lb_policy* pol) {
 }
 
 static int rr_pick_locked(grpc_lb_policy* pol,
-                          grpc_lb_policy_pick_state* pick) {
+                          const grpc_lb_policy_pick_args* pick_args,
+                          grpc_connected_subchannel** target,
+                          grpc_call_context_element* context, void** user_data,
+                          grpc_closure* on_complete) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
   if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", pol,
+    gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", (void*)pol,
             p->shutdown);
   }
   GPR_ASSERT(!p->shutdown);
@@ -262,18 +284,18 @@ static int rr_pick_locked(grpc_lb_policy* pol,
       /* readily available, report right away */
       grpc_lb_subchannel_data* sd =
           &p->subchannel_list->subchannels[next_ready_index];
-      pick->connected_subchannel =
+      *target =
           GRPC_CONNECTED_SUBCHANNEL_REF(sd->connected_subchannel, "rr_picked");
-      if (pick->user_data != nullptr) {
-        *pick->user_data = sd->user_data;
+      if (user_data != nullptr) {
+        *user_data = sd->user_data;
       }
       if (grpc_lb_round_robin_trace.enabled()) {
         gpr_log(
             GPR_DEBUG,
             "[RR %p] Picked target <-- Subchannel %p (connected %p) (sl %p, "
-            "index %" PRIuPTR ")",
-            p, sd->subchannel, pick->connected_subchannel, sd->subchannel_list,
-            next_ready_index);
+            "index %lu)",
+            (void*)p, (void*)sd->subchannel, (void*)*target,
+            (void*)sd->subchannel_list, (unsigned long)next_ready_index);
       }
       /* only advance the last picked pointer if the selection was used */
       update_last_ready_subchannel_index_locked(p, next_ready_index);
@@ -284,8 +306,13 @@ static int rr_pick_locked(grpc_lb_policy* pol,
   if (!p->started_picking) {
     start_picking_locked(p);
   }
-  pick->next = p->pending_picks;
-  p->pending_picks = pick;
+  pending_pick* pp = (pending_pick*)gpr_malloc(sizeof(*pp));
+  pp->next = p->pending_picks;
+  pp->target = target;
+  pp->on_complete = on_complete;
+  pp->initial_metadata_flags = pick_args->initial_metadata_flags;
+  pp->user_data = user_data;
+  p->pending_picks = pp;
   return 0;
 }
 
@@ -468,13 +495,13 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
         // picks, update the last picked pointer
         update_last_ready_subchannel_index_locked(p, next_ready_index);
       }
-      grpc_lb_policy_pick_state* pick;
-      while ((pick = p->pending_picks)) {
-        p->pending_picks = pick->next;
-        pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
+      pending_pick* pp;
+      while ((pp = p->pending_picks)) {
+        p->pending_picks = pp->next;
+        *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF(
             selected->connected_subchannel, "rr_picked");
-        if (pick->user_data != nullptr) {
-          *pick->user_data = selected->user_data;
+        if (pp->user_data != nullptr) {
+          *pp->user_data = selected->user_data;
         }
         if (grpc_lb_round_robin_trace.enabled()) {
           gpr_log(GPR_DEBUG,
@@ -483,7 +510,8 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
                   (void*)p, (void*)selected->subchannel,
                   (void*)p->subchannel_list, (unsigned long)next_ready_index);
         }
-        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
+        GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_NONE);
+        gpr_free(pp);
       }
     }
     // Renew notification.
diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
index 5ce1298afc4..a3b4c8e524d 100644
--- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
@@ -213,13 +213,13 @@ void grpc_lb_subchannel_list_unref(grpc_lb_subchannel_list* subchannel_list,
 
 void grpc_lb_subchannel_list_ref_for_connectivity_watch(
     grpc_lb_subchannel_list* subchannel_list, const char* reason) {
-  GRPC_LB_POLICY_REF(subchannel_list->policy, reason);
+  GRPC_LB_POLICY_WEAK_REF(subchannel_list->policy, reason);
   grpc_lb_subchannel_list_ref(subchannel_list, reason);
 }
 
 void grpc_lb_subchannel_list_unref_for_connectivity_watch(
     grpc_lb_subchannel_list* subchannel_list, const char* reason) {
-  GRPC_LB_POLICY_UNREF(subchannel_list->policy, reason);
+  GRPC_LB_POLICY_WEAK_UNREF(subchannel_list->policy, reason);
   grpc_lb_subchannel_list_unref(subchannel_list, reason);
 }
 

From 0e50b94b5be32e7bd0bde6fab5ae1997ae9f2464 Mon Sep 17 00:00:00 2001
From: Dan Zhang <danzh@google.com>
Date: Wed, 10 Jan 2018 15:45:39 -0500
Subject: [PATCH 062/161] Adjust receiv buffer via setsockopt for udp_server's
 listening socket. Since this socket is used for all incoming traffic, its
 current buffer 1MB is appearantly too small. Change it to 10 MB for now.

---
 src/core/lib/iomgr/udp_server.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/core/lib/iomgr/udp_server.cc b/src/core/lib/iomgr/udp_server.cc
index 4a97f3353d9..6dde7b96112 100644
--- a/src/core/lib/iomgr/udp_server.cc
+++ b/src/core/lib/iomgr/udp_server.cc
@@ -283,8 +283,8 @@ static int prepare_socket(grpc_socket_factory* socket_factory, int fd,
                           const grpc_resolved_address* addr) {
   grpc_resolved_address sockname_temp;
   struct sockaddr* addr_ptr = (struct sockaddr*)addr->addr;
-  /* Set send/receive socket buffers to 1 MB */
-  int buffer_size_bytes = 1024 * 1024;
+  /* Set send/receive socket buffers to 10 MB */
+  int buffer_size_bytes = 1024 * 1024 * 10;
 
   if (fd < 0) {
     goto error;

From 8616b362fd2d015dd4694d68bc54ff9231d50214 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Wed, 10 Jan 2018 11:37:10 -0800
Subject: [PATCH 063/161] Add guidance on the AUTHORS file to contributors

---
 CONTRIBUTING.md | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index af462468225..26fbc5970e6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -39,12 +39,19 @@ How to get your contributions merged smoothly and quickly.
 - Don't fix code style and formatting unless you are already changing that line to address an issue. PRs with irrelevant changes won't be merged. If you do want to fix formatting or style, do that in a separate PR.
  
 - Unless your PR is trivial, you should expect there will be reviewer comments that you'll need to address before merging. We expect you to be reasonably responsive to those comments, otherwise the PR will be closed after 2-3 weeks of inactivity.
+
+- If you have non-trivial contributions, please consider adding an entry to [the
+  AUTHORS file](https://github.com/grpc/grpc/blob/master/AUTHORS) listing the
+  copyright holder for the contribution (yourself, if you are signing the
+  individual CLA, or your company, for corporate CLAs) in the same PR as your
+  contribution.  This needs to be done only once, for each company, or
+  individual.
  
 - Maintain **clean commit history** and use **meaningful commit messages**. PRs with messy commit history are difficult to review and won't be merged. Use `rebase -i upstream/master` to curate your commit history and/or to bring in latest changes from master (but avoid rebasing in the middle of a code review).
  
 - Keep your PR up to date with upstream/master (if there are merge conflicts, we can't really merge your change).
  
-- if you are regenerating the projects using `tools/buildgen/generate_projects.sh`, make changes to generated files a separate commit with commit message `regenerate projects`. Mixing changes to generated and hand-written files make your PR difficult to review.
+- If you are regenerating the projects using `tools/buildgen/generate_projects.sh`, make changes to generated files a separate commit with commit message `regenerate projects`. Mixing changes to generated and hand-written files make your PR difficult to review.
  
 - **All tests need to be passing** before your change can be merged. We recommend you **run tests locally** before creating your PR to catch breakages early on (see [tools/run_tests](tools/run_tests). Ultimately, the green signal will be provided by our testing infrastructure. The reviewer will help you if there are test failures that seem not related to the change you are making.
  

From 4d001386fd821eca8587b1918bc94ae5c3200139 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Wed, 10 Jan 2018 13:43:28 -0800
Subject: [PATCH 064/161] Reformat CONTRIBUTING.md

---
 CONTRIBUTING.md | 73 ++++++++++++++++++++++++++++++++++---------------
 1 file changed, 51 insertions(+), 22 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 26fbc5970e6..c85027117a6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -2,43 +2,58 @@
 
 We definitely welcome your patches and contributions to gRPC!
 
-If you are new to github, please start by reading [Pull Request howto](https://help.github.com/articles/about-pull-requests/)
+If you are new to github, please start by reading [Pull Request
+howto](https://help.github.com/articles/about-pull-requests/)
 
 ## Legal requirements
 
 In order to protect both you and ourselves, you will need to sign the
-[Contributor License Agreement](https://identity.linuxfoundation.org/projects/cncf).
+[Contributor License
+Agreement](https://identity.linuxfoundation.org/projects/cncf).
 
 ## Running tests
 
-Use `tools/run_tests/run_tests.py` script to run the unit tests.
-See [tools/run_tests](tools/run_tests) for how to run tests for a given language.
+Use `tools/run_tests/run_tests.py` script to run the unit tests.  See
+[tools/run_tests](tools/run_tests) for how to run tests for a given language.
 
-Prerequisites for building and running tests are listed in [INSTALL.md](INSTALL.md)
-and in `src/YOUR-LANGUAGE` (e.g. `src/csharp`)
+Prerequisites for building and running tests are listed in
+[INSTALL.md](INSTALL.md) and in `src/YOUR-LANGUAGE` (e.g. `src/csharp`)
 
 ## Generated project files
 
-To ease maintenance of language- and platform- specific build systems,
-many projects files are generated using templates and should not be edited
-by hand.
-Run `tools/buildgen/generate_projects.sh` to regenerate.
-See [templates](templates) for details.
+To ease maintenance of language- and platform- specific build systems, many
+projects files are generated using templates and should not be edited by hand.
+Run `tools/buildgen/generate_projects.sh` to regenerate.  See
+[templates](templates) for details.
 
-As a rule of thumb, if you see the "sanity tests" failing you've most likely edited generated files or you didn't regenerate the projects properly (or your code formatting doesn't match our code style).
+As a rule of thumb, if you see the "sanity tests" failing you've most likely
+edited generated files or you didn't regenerate the projects properly (or your
+code formatting doesn't match our code style).
 
 ## Guidelines for Pull Requests
 How to get your contributions merged smoothly and quickly.
  
-- Create **small PRs** that are narrowly focused on **addressing a single concern**. We often times receive PRs that are trying to fix several things at a time, but only one fix is considered acceptable, nothing gets merged and both author's & review's time is wasted. Create more PRs to address different concerns and everyone will be happy.
+- Create **small PRs** that are narrowly focused on **addressing a single
+  concern**.  We often times receive PRs that are trying to fix several things
+  at a time, but only one fix is considered acceptable, nothing gets merged and
+  both author's & review's time is wasted.  Create more PRs to address different
+  concerns and everyone will be happy.
  
-- For speculative changes, consider opening an issue and discussing it first. If you are suggesting a behavioral or API change, consider starting with a [gRFC proposal](https://github.com/grpc/proposal). 
+- For speculative changes, consider opening an issue and discussing it first.
+  If you are suggesting a behavioral or API change, consider starting with a
+  [gRFC proposal](https://github.com/grpc/proposal).
  
-- Provide a good **PR description** as a record of **what** change is being made and **why** it was made. Link to a github issue if it exists.
+- Provide a good **PR description** as a record of **what** change is being made
+  and **why** it was made.  Link to a GitHub issue if it exists.
  
-- Don't fix code style and formatting unless you are already changing that line to address an issue. PRs with irrelevant changes won't be merged. If you do want to fix formatting or style, do that in a separate PR.
+- Don't fix code style and formatting unless you are already changing that line
+  to address an issue.  PRs with irrelevant changes won't be merged.  If you do
+  want to fix formatting or style, do that in a separate PR.
  
-- Unless your PR is trivial, you should expect there will be reviewer comments that you'll need to address before merging. We expect you to be reasonably responsive to those comments, otherwise the PR will be closed after 2-3 weeks of inactivity.
+- Unless your PR is trivial, you should expect there will be reviewer comments
+  that you'll need to address before merging.  We expect you to be reasonably
+  responsive to those comments, otherwise the PR will be closed after 2-3 weeks
+  of inactivity.
 
 - If you have non-trivial contributions, please consider adding an entry to [the
   AUTHORS file](https://github.com/grpc/grpc/blob/master/AUTHORS) listing the
@@ -47,15 +62,29 @@ How to get your contributions merged smoothly and quickly.
   contribution.  This needs to be done only once, for each company, or
   individual.
  
-- Maintain **clean commit history** and use **meaningful commit messages**. PRs with messy commit history are difficult to review and won't be merged. Use `rebase -i upstream/master` to curate your commit history and/or to bring in latest changes from master (but avoid rebasing in the middle of a code review).
+- Maintain **clean commit history** and use **meaningful commit messages**.
+  PRs with messy commit history are difficult to review and won't be merged.
+  Use `rebase -i upstream/master` to curate your commit history and/or to
+  bring in latest changes from master (but avoid rebasing in the middle of
+  a code review).
  
-- Keep your PR up to date with upstream/master (if there are merge conflicts, we can't really merge your change).
+- Keep your PR up to date with upstream/master (if there are merge conflicts,
+  we can't really merge your change).
  
-- If you are regenerating the projects using `tools/buildgen/generate_projects.sh`, make changes to generated files a separate commit with commit message `regenerate projects`. Mixing changes to generated and hand-written files make your PR difficult to review.
+- If you are regenerating the projects using
+  `tools/buildgen/generate_projects.sh`, make changes to generated files a
+  separate commit with commit message `regenerate projects`.  Mixing changes
+  to generated and hand-written files make your PR difficult to review.
  
-- **All tests need to be passing** before your change can be merged. We recommend you **run tests locally** before creating your PR to catch breakages early on (see [tools/run_tests](tools/run_tests). Ultimately, the green signal will be provided by our testing infrastructure. The reviewer will help you if there are test failures that seem not related to the change you are making.
+- **All tests need to be passing** before your change can be merged.
+  We recommend you **run tests locally** before creating your PR to catch
+  breakages early on (see [tools/run_tests](tools/run_tests).  Ultimately, the
+  green signal will be provided by our testing infrastructure.  The reviewer
+  will help you if there are test failures that seem not related to the change
+  you are making.
  
-- Exceptions to the rules can be made if there's a compelling reason for doing so.
+- Exceptions to the rules can be made if there's a compelling reason for doing
+  so.
 
 
 

From c052f363bfc72be8e274df3f70687ec96502f8e5 Mon Sep 17 00:00:00 2001
From: Craig Tiller <ctiller@google.com>
Date: Wed, 10 Jan 2018 14:39:52 -0800
Subject: [PATCH 065/161] Substitute a11r for ctiller in all OWNERS files

---
 .github/CODEOWNERS                         | 6 +++---
 OWNERS                                     | 2 +-
 bazel/OWNERS                               | 2 +-
 src/core/ext/filters/client_channel/OWNERS | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index cb322814fe6..093e9396cf4 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1,7 +1,7 @@
 # Auto-generated by the tools/mkowners/mkowners.py tool
 # Uses OWNERS files in different modules throughout the
 # repository as the source of truth for module ownership.
-/**/OWNERS @markdroth @nicolasnoble @ctiller
-/bazel/** @nicolasnoble @dgquintas @ctiller
-/src/core/ext/filters/client_channel/** @markdroth @dgquintas @ctiller
+/**/OWNERS @markdroth @nicolasnoble @a11r
+/bazel/** @nicolasnoble @dgquintas @a11r
+/src/core/ext/filters/client_channel/** @markdroth @dgquintas @a11r
 /tools/run_tests/performance/** @ncteisen @matt-kwong @ctiller
diff --git a/OWNERS b/OWNERS
index 650d58f42ce..33f319803ba 100644
--- a/OWNERS
+++ b/OWNERS
@@ -13,5 +13,5 @@
 #         lead to a bus factor of one to changes to that code
 @markdroth **/OWNERS
 @nicolasnoble **/OWNERS
-@ctiller **/OWNERS
+@a11r **/OWNERS
 
diff --git a/bazel/OWNERS b/bazel/OWNERS
index 8fc7502a915..f6ad849be30 100644
--- a/bazel/OWNERS
+++ b/bazel/OWNERS
@@ -1,5 +1,5 @@
 set noparent
 @nicolasnoble
 @dgquintas
-@ctiller
+@a11r
 
diff --git a/src/core/ext/filters/client_channel/OWNERS b/src/core/ext/filters/client_channel/OWNERS
index 773bc73179e..8f5e92808e6 100644
--- a/src/core/ext/filters/client_channel/OWNERS
+++ b/src/core/ext/filters/client_channel/OWNERS
@@ -1,4 +1,4 @@
 set noparent
 @markdroth
 @dgquintas
-@ctiller
+@a11r

From b4b0ac704984be21d128924433cbe9bcd568ef83 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Wed, 10 Jan 2018 22:02:37 +0000
Subject: [PATCH 066/161] Resolve leak by freeing request payload if resources
 exhausted

---
 include/grpc++/impl/codegen/byte_buffer.h         | 4 ++++
 include/grpc++/impl/codegen/method_handler_impl.h | 5 +++++
 src/cpp/server/server_cc.cc                       | 3 ++-
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h
index fe73ce7a83d..9c0246e6174 100644
--- a/include/grpc++/impl/codegen/byte_buffer.h
+++ b/include/grpc++/impl/codegen/byte_buffer.h
@@ -41,6 +41,8 @@ template <class ServiceType, class RequestType, class ResponseType>
 class RpcMethodHandler;
 template <class ServiceType, class RequestType, class ResponseType>
 class ServerStreamingHandler;
+template <StatusCode code>
+class ErrorMethodHandler;
 template <class R>
 class DeserializeFuncType;
 }  // namespace internal
@@ -107,6 +109,8 @@ class ByteBuffer final {
   friend class internal::RpcMethodHandler;
   template <class ServiceType, class RequestType, class ResponseType>
   friend class internal::ServerStreamingHandler;
+  template <StatusCode code>
+  friend class internal::ErrorMethodHandler;
   template <class R>
   friend class internal::DeserializeFuncType;
 
diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h
index d98ab7938cf..93b7826e8fc 100644
--- a/include/grpc++/impl/codegen/method_handler_impl.h
+++ b/include/grpc++/impl/codegen/method_handler_impl.h
@@ -266,6 +266,11 @@ class ErrorMethodHandler : public MethodHandler {
     FillOps(param.server_context, &ops);
     param.call->PerformOps(&ops);
     param.call->cq()->Pluck(&ops);
+    // We also have to destroy any request payload in the handler parameter
+    ByteBuffer* payload = param.request.bbuf_ptr();
+    if (payload != nullptr) {
+      payload->Clear();
+    }
   }
 };
 
diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc
index 6ab76a287e5..02a663d660e 100644
--- a/src/cpp/server/server_cc.cc
+++ b/src/cpp/server/server_cc.cc
@@ -196,7 +196,8 @@ class Server::SyncRequest final : public internal::CompletionQueueTag {
           call_(mrd->call_, server, &cq_, server->max_receive_message_size()),
           ctx_(mrd->deadline_, &mrd->request_metadata_),
           has_request_payload_(mrd->has_request_payload_),
-          request_payload_(mrd->request_payload_),
+          request_payload_(has_request_payload_ ? mrd->request_payload_
+                                                : nullptr),
           method_(mrd->method_),
           server_(server) {
       ctx_.set_call(mrd->call_);

From c6406f32f626bcad9aee9582132f06bc68f4e0e5 Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Wed, 10 Jan 2018 14:47:37 -0800
Subject: [PATCH 067/161] Implement InlinedVector independently of absl.

---
 CMakeLists.txt                                | 40 +++++++++
 Makefile                                      | 48 +++++++++++
 build.yaml                                    | 15 ++++
 gRPC-Core.podspec                             |  2 +
 grpc.gemspec                                  |  1 +
 package.xml                                   |  1 +
 src/core/lib/support/vector.h                 | 84 ++++++++++++++++++-
 test/core/support/vector_test.cc              | 42 ++++++++--
 tools/doxygen/Doxyfile.c++.internal           |  1 +
 tools/doxygen/Doxyfile.core.internal          |  1 +
 .../generated/sources_and_headers.json        | 21 +++++
 tools/run_tests/generated/tests.json          | 24 ++++++
 12 files changed, 273 insertions(+), 7 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index eed12052685..a53624978cf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -594,6 +594,7 @@ add_dependencies(buildtests_cxx stress_test)
 add_dependencies(buildtests_cxx thread_manager_test)
 add_dependencies(buildtests_cxx thread_stress_test)
 add_dependencies(buildtests_cxx transport_pid_controller_test)
+add_dependencies(buildtests_cxx vector_test)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_cxx writes_per_rpc_test)
 endif()
@@ -12495,6 +12496,45 @@ target_link_libraries(transport_pid_controller_test
   ${_gRPC_GFLAGS_LIBRARIES}
 )
 
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
+add_executable(vector_test
+  test/core/support/vector_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+
+target_include_directories(vector_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${BENCHMARK_ROOT_DIR}/include
+  PRIVATE ${ZLIB_ROOT_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CARES_INCLUDE_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(vector_test
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc_test_util
+  grpc++
+  grpc
+  gpr_test_util
+  gpr
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
diff --git a/Makefile b/Makefile
index 38b40804d64..a5a9f9508d3 100644
--- a/Makefile
+++ b/Makefile
@@ -1180,6 +1180,7 @@ stress_test: $(BINDIR)/$(CONFIG)/stress_test
 thread_manager_test: $(BINDIR)/$(CONFIG)/thread_manager_test
 thread_stress_test: $(BINDIR)/$(CONFIG)/thread_stress_test
 transport_pid_controller_test: $(BINDIR)/$(CONFIG)/transport_pid_controller_test
+vector_test: $(BINDIR)/$(CONFIG)/vector_test
 writes_per_rpc_test: $(BINDIR)/$(CONFIG)/writes_per_rpc_test
 public_headers_must_be_c89: $(BINDIR)/$(CONFIG)/public_headers_must_be_c89
 gen_hpack_tables: $(BINDIR)/$(CONFIG)/gen_hpack_tables
@@ -1620,6 +1621,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/thread_manager_test \
   $(BINDIR)/$(CONFIG)/thread_stress_test \
   $(BINDIR)/$(CONFIG)/transport_pid_controller_test \
+  $(BINDIR)/$(CONFIG)/vector_test \
   $(BINDIR)/$(CONFIG)/writes_per_rpc_test \
   $(BINDIR)/$(CONFIG)/boringssl_aes_test \
   $(BINDIR)/$(CONFIG)/boringssl_asn1_test \
@@ -1749,6 +1751,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/thread_manager_test \
   $(BINDIR)/$(CONFIG)/thread_stress_test \
   $(BINDIR)/$(CONFIG)/transport_pid_controller_test \
+  $(BINDIR)/$(CONFIG)/vector_test \
   $(BINDIR)/$(CONFIG)/writes_per_rpc_test \
   $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure \
   $(BINDIR)/$(CONFIG)/resolver_component_test \
@@ -2167,6 +2170,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/thread_stress_test || ( echo test thread_stress_test failed ; exit 1 )
 	$(E) "[RUN]     Testing transport_pid_controller_test"
 	$(Q) $(BINDIR)/$(CONFIG)/transport_pid_controller_test || ( echo test transport_pid_controller_test failed ; exit 1 )
+	$(E) "[RUN]     Testing vector_test"
+	$(Q) $(BINDIR)/$(CONFIG)/vector_test || ( echo test vector_test failed ; exit 1 )
 	$(E) "[RUN]     Testing writes_per_rpc_test"
 	$(Q) $(BINDIR)/$(CONFIG)/writes_per_rpc_test || ( echo test writes_per_rpc_test failed ; exit 1 )
 	$(E) "[RUN]     Testing resolver_component_tests_runner_invoker_unsecure"
@@ -17270,6 +17275,49 @@ endif
 endif
 
 
+VECTOR_TEST_SRC = \
+    test/core/support/vector_test.cc \
+
+VECTOR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(VECTOR_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/vector_test: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
+
+$(BINDIR)/$(CONFIG)/vector_test: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/vector_test: $(PROTOBUF_DEP) $(VECTOR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(VECTOR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/vector_test
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/support/vector_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_vector_test: $(VECTOR_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(VECTOR_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 WRITES_PER_RPC_TEST_SRC = \
     test/cpp/performance/writes_per_rpc_test.cc \
 
diff --git a/build.yaml b/build.yaml
index fef7d6189f7..a753253a2e9 100644
--- a/build.yaml
+++ b/build.yaml
@@ -399,6 +399,7 @@ filegroups:
   - src/core/lib/support/debug_location.h
   - src/core/lib/support/ref_counted.h
   - src/core/lib/support/ref_counted_ptr.h
+  - src/core/lib/support/vector.h
   - src/core/lib/surface/alarm_internal.h
   - src/core/lib/surface/api_trace.h
   - src/core/lib/surface/call.h
@@ -4798,6 +4799,20 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
+- name: vector_test
+  gtest: true
+  build: test
+  language: c++
+  src:
+  - test/core/support/vector_test.cc
+  deps:
+  - grpc_test_util
+  - grpc++
+  - grpc
+  - gpr_test_util
+  - gpr
+  uses:
+  - grpc++_test
 - name: writes_per_rpc_test
   gtest: true
   cpu_cost: 0.5
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index c127660dd50..a64f5e45592 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -423,6 +423,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/support/debug_location.h',
                       'src/core/lib/support/ref_counted.h',
                       'src/core/lib/support/ref_counted_ptr.h',
+                      'src/core/lib/support/vector.h',
                       'src/core/lib/surface/alarm_internal.h',
                       'src/core/lib/surface/api_trace.h',
                       'src/core/lib/surface/call.h',
@@ -903,6 +904,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/support/debug_location.h',
                               'src/core/lib/support/ref_counted.h',
                               'src/core/lib/support/ref_counted_ptr.h',
+                              'src/core/lib/support/vector.h',
                               'src/core/lib/surface/alarm_internal.h',
                               'src/core/lib/surface/api_trace.h',
                               'src/core/lib/surface/call.h',
diff --git a/grpc.gemspec b/grpc.gemspec
index d1859952619..95836a5029d 100644
--- a/grpc.gemspec
+++ b/grpc.gemspec
@@ -349,6 +349,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/support/debug_location.h )
   s.files += %w( src/core/lib/support/ref_counted.h )
   s.files += %w( src/core/lib/support/ref_counted_ptr.h )
+  s.files += %w( src/core/lib/support/vector.h )
   s.files += %w( src/core/lib/surface/alarm_internal.h )
   s.files += %w( src/core/lib/surface/api_trace.h )
   s.files += %w( src/core/lib/surface/call.h )
diff --git a/package.xml b/package.xml
index b4d8c886930..8e34b56ce75 100644
--- a/package.xml
+++ b/package.xml
@@ -361,6 +361,7 @@
     <file baseinstalldir="/" name="src/core/lib/support/debug_location.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/ref_counted.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/ref_counted_ptr.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/support/vector.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/surface/alarm_internal.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/surface/api_trace.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/surface/call.h" role="src" />
diff --git a/src/core/lib/support/vector.h b/src/core/lib/support/vector.h
index 4a7db806761..2f249a5b9e8 100644
--- a/src/core/lib/support/vector.h
+++ b/src/core/lib/support/vector.h
@@ -19,13 +19,93 @@
 #ifndef GRPC_CORE_LIB_SUPPORT_VECTOR_H
 #define GRPC_CORE_LIB_SUPPORT_VECTOR_H
 
-#include "absl/container/inlined_vector.h"
+#include <cassert>
+
 #include "src/core/lib/support/memory.h"
 
 namespace grpc_core {
 
+// NOTE: We eventually want to use absl::InlinedVector here.  However,
+// there are currently build problems that prevent us from using absl.
+// In the interim, we define a custom implementation as a place-holder,
+// with the intent to eventually replace this with the absl
+// implementation.
+//
+// This place-holder implementation does not implement the full set of
+// functionality from the absl version; it has just the methods that we
+// currently happen to need in gRPC.  If additional functionality is
+// needed before this gets replaced with the absl version, it can be
+// added, with the following proviso:
+//
+// ANY METHOD ADDED HERE MUST COMPLY WITH THE INTERFACE IN THE absl
+// IMPLEMENTATION!
+//
+// TODO(ctiller, nnoble, roth): Replace this with absl::InlinedVector
+// once we integrate absl into the gRPC build system in a usable way.
 template <typename T, size_t N>
-using InlinedVector = absl::InlinedVector<T, N, Allocator<T>>;
+class InlinedVector {
+ public:
+  InlinedVector() {}
+  ~InlinedVector() {
+    for (size_t i = 0; i < size_ && i < N; ++i) {
+      T& value = *reinterpret_cast<T*>(inline_ + i);
+      value.~T();
+    }
+    if (size_ > N) {  // Avoid subtracting two signed values.
+      for (size_t i = 0; i < size_ - N; ++i) {
+        dynamic_[i].~T();
+      }
+    }
+    gpr_free(dynamic_);
+  }
+
+  // For now, we do not support copying.
+  InlinedVector(const InlinedVector&) = delete;
+  InlinedVector& operator=(const InlinedVector&) = delete;
+
+  T& operator[](size_t offset) {
+    assert(offset < size_);
+    if (offset < N) {
+      return *reinterpret_cast<T*>(inline_ + offset);
+    } else {
+      return dynamic_[offset - N];
+    }
+  }
+
+  template <typename... Args>
+  void emplace_back(Args&&... args) {
+    if (size_ < N) {
+      new (&inline_[size_]) T(std::forward<Args>(args)...);
+    } else {
+      if (size_ - N == dynamic_capacity_) {
+        size_t new_capacity =
+            dynamic_capacity_ == 0 ? 2 : dynamic_capacity_ * 2;
+        T* new_dynamic = static_cast<T*>(gpr_malloc(sizeof(T) * new_capacity));
+        for (size_t i = 0; i < dynamic_capacity_; ++i) {
+          new (&new_dynamic[i]) T(std::move(dynamic_[i]));
+          dynamic_[i].~T();
+        }
+        gpr_free(dynamic_);
+        dynamic_ = new_dynamic;
+        dynamic_capacity_ = new_capacity;
+      }
+      new (&dynamic_[size_ - N]) T(std::forward<Args>(args)...);
+    }
+    ++size_;
+  }
+
+  void push_back(const T& value) { emplace_back(value); }
+
+  void push_back(T&& value) { emplace_back(std::move(value)); }
+
+  size_t size() const { return size_; }
+
+ private:
+  typename std::aligned_storage<sizeof(T)>::type inline_[N];
+  T* dynamic_ = nullptr;
+  size_t size_ = 0;
+  size_t dynamic_capacity_ = 0;
+};
 
 }  // namespace grpc_core
 
diff --git a/test/core/support/vector_test.cc b/test/core/support/vector_test.cc
index aad9f3be907..82607a1b260 100644
--- a/test/core/support/vector_test.cc
+++ b/test/core/support/vector_test.cc
@@ -18,18 +18,50 @@
 
 #include "src/core/lib/support/vector.h"
 #include <gtest/gtest.h>
+#include "src/core/lib/support/memory.h"
 #include "test/core/util/test_config.h"
 
 namespace grpc_core {
 namespace testing {
 
 TEST(InlinedVectorTest, CreateAndIterate) {
-  InlinedVector<int, 1> v{1, 2, 3};
-  int sum = 0;
-  for (auto i : v) {
-    sum += i;
+  const int kNumElements = 9;
+  InlinedVector<int, 2> v;
+  for (int i = 0; i < kNumElements; ++i) {
+    v.push_back(i);
   }
-  EXPECT_EQ(6, sum);
+  EXPECT_EQ(static_cast<size_t>(kNumElements), v.size());
+  for (int i = 0; i < kNumElements; ++i) {
+    EXPECT_EQ(i, v[i]);
+  }
+}
+
+TEST(InlinedVectorTest, ValuesAreInlined) {
+  const int kNumElements = 5;
+  InlinedVector<int, 10> v;
+  for (int i = 0; i < kNumElements; ++i) {
+    v.push_back(i);
+  }
+  EXPECT_EQ(static_cast<size_t>(kNumElements), v.size());
+  for (int i = 0; i < kNumElements; ++i) {
+    EXPECT_EQ(i, v[i]);
+  }
+}
+
+TEST(InlinedVectorTest, PushBackWithMove) {
+  InlinedVector<UniquePtr<int>, 1> v;
+  UniquePtr<int> i = MakeUnique<int>(3);
+  v.push_back(std::move(i));
+  EXPECT_EQ(nullptr, i.get());
+  EXPECT_EQ(1UL, v.size());
+  EXPECT_EQ(3, *v[0]);
+}
+
+TEST(InlinedVectorTest, EmplaceBack) {
+  InlinedVector<UniquePtr<int>, 1> v;
+  v.emplace_back(New<int>(3));
+  EXPECT_EQ(1UL, v.size());
+  EXPECT_EQ(3, *v[0]);
 }
 
 }  // namespace testing
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index d09b325c97a..211149ac261 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -1047,6 +1047,7 @@ src/core/lib/support/string_windows.h \
 src/core/lib/support/thd_internal.h \
 src/core/lib/support/time_precise.h \
 src/core/lib/support/tmpfile.h \
+src/core/lib/support/vector.h \
 src/core/lib/surface/alarm_internal.h \
 src/core/lib/surface/api_trace.h \
 src/core/lib/surface/call.h \
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 1aff0075a6a..5bd707d5451 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -1333,6 +1333,7 @@ src/core/lib/support/tmpfile.h \
 src/core/lib/support/tmpfile_msys.cc \
 src/core/lib/support/tmpfile_posix.cc \
 src/core/lib/support/tmpfile_windows.cc \
+src/core/lib/support/vector.h \
 src/core/lib/support/wrap_memcpy.cc \
 src/core/lib/surface/README.md \
 src/core/lib/surface/alarm.cc \
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index d432bd0e537..80bd0d6185d 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -4262,6 +4262,25 @@
     "third_party": false, 
     "type": "target"
   }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc++", 
+      "grpc++_test", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c++", 
+    "name": "vector_test", 
+    "src": [
+      "test/core/support/vector_test.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
   {
     "deps": [
       "gpr", 
@@ -8250,6 +8269,7 @@
       "src/core/lib/support/debug_location.h", 
       "src/core/lib/support/ref_counted.h", 
       "src/core/lib/support/ref_counted_ptr.h", 
+      "src/core/lib/support/vector.h", 
       "src/core/lib/surface/alarm_internal.h", 
       "src/core/lib/surface/api_trace.h", 
       "src/core/lib/surface/call.h", 
@@ -8389,6 +8409,7 @@
       "src/core/lib/support/debug_location.h", 
       "src/core/lib/support/ref_counted.h", 
       "src/core/lib/support/ref_counted_ptr.h", 
+      "src/core/lib/support/vector.h", 
       "src/core/lib/surface/alarm_internal.h", 
       "src/core/lib/surface/api_trace.h", 
       "src/core/lib/surface/call.h", 
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index 98517cba2e7..6b83cecd41d 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -4504,6 +4504,30 @@
     ], 
     "uses_polling": true
   }, 
+  {
+    "args": [], 
+    "benchmark": false, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": true, 
+    "language": "c++", 
+    "name": "vector_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "uses_polling": true
+  }, 
   {
     "args": [], 
     "benchmark": false, 

From 31ddbff8cfed36d77ad34ec892af812dff614ce6 Mon Sep 17 00:00:00 2001
From: Nathaniel Manista <nathaniel@google.com>
Date: Wed, 10 Jan 2018 22:58:00 +0000
Subject: [PATCH 068/161] Elide cygrpc.Timespec

---
 src/python/grpcio/grpc/_channel.py            | 55 +++++++-------
 .../grpc/_cython/_cygrpc/channel.pyx.pxi      | 26 ++++---
 .../_cython/_cygrpc/completion_queue.pyx.pxi  |  9 +--
 .../grpc/_cython/_cygrpc/records.pxd.pxi      |  5 --
 .../grpc/_cython/_cygrpc/records.pyx.pxi      | 72 +------------------
 .../grpc/_cython/_cygrpc/server.pyx.pxi       |  2 +-
 .../grpcio/grpc/_cython/_cygrpc/time.pxd.pxi  | 19 +++++
 .../grpcio/grpc/_cython/_cygrpc/time.pyx.pxi  | 30 ++++++++
 src/python/grpcio/grpc/_cython/cygrpc.pxd     |  1 +
 src/python/grpcio/grpc/_cython/cygrpc.pyx     |  1 +
 src/python/grpcio/grpc/_server.py             |  3 +-
 .../unit/_cython/_cancel_many_calls_test.py   |  7 +-
 .../tests/unit/_cython/_channel_test.py       |  6 +-
 .../tests/unit/_cython/_common.py             |  1 -
 ...s_server_completion_queue_per_call_test.py |  6 +-
 ...ges_single_server_completion_queue_test.py |  6 +-
 .../_read_some_but_not_all_responses_test.py  |  3 +-
 .../tests/unit/_cython/cygrpc_test.py         | 38 +++-------
 .../tests/unit/_cython/test_utilities.py      |  2 +-
 19 files changed, 120 insertions(+), 172 deletions(-)
 create mode 100644 src/python/grpcio/grpc/_cython/_cygrpc/time.pxd.pxi
 create mode 100644 src/python/grpcio/grpc/_cython/_cygrpc/time.pyx.pxi

diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py
index 24be042f61f..bfc7208310c 100644
--- a/src/python/grpcio/grpc/_channel.py
+++ b/src/python/grpcio/grpc/_channel.py
@@ -27,7 +27,6 @@ from grpc.framework.foundation import callable_util
 _USER_AGENT = 'grpc-python/{}'.format(_grpcio_metadata.__version__)
 
 _EMPTY_FLAGS = 0
-_INFINITE_FUTURE = cygrpc.Timespec(float('+inf'))
 
 _UNARY_UNARY_INITIAL_DUE = (
     cygrpc.OperationType.send_initial_metadata,
@@ -61,11 +60,7 @@ _CHANNEL_SUBSCRIPTION_CALLBACK_ERROR_LOG_MESSAGE = (
 
 
 def _deadline(timeout):
-    if timeout is None:
-        return None, _INFINITE_FUTURE
-    else:
-        deadline = time.time() + timeout
-        return deadline, cygrpc.Timespec(deadline)
+    return None if timeout is None else time.time() + timeout
 
 
 def _unknown_code_details(unknown_cygrpc_code, details):
@@ -420,15 +415,15 @@ class _Rendezvous(grpc.RpcError, grpc.Future, grpc.Call):
 
 
 def _start_unary_request(request, timeout, request_serializer):
-    deadline, deadline_timespec = _deadline(timeout)
+    deadline = _deadline(timeout)
     serialized_request = _common.serialize(request, request_serializer)
     if serialized_request is None:
         state = _RPCState((), (), (), grpc.StatusCode.INTERNAL,
                           'Exception serializing request!')
         rendezvous = _Rendezvous(state, None, None, deadline)
-        return deadline, deadline_timespec, None, rendezvous
+        return deadline, None, rendezvous
     else:
-        return deadline, deadline_timespec, serialized_request, None
+        return deadline, serialized_request, None
 
 
 def _end_unary_response_blocking(state, call, with_call, deadline):
@@ -453,10 +448,10 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable):
         self._response_deserializer = response_deserializer
 
     def _prepare(self, request, timeout, metadata):
-        deadline, deadline_timespec, serialized_request, rendezvous = (
-            _start_unary_request(request, timeout, self._request_serializer))
+        deadline, serialized_request, rendezvous = (_start_unary_request(
+            request, timeout, self._request_serializer))
         if serialized_request is None:
-            return None, None, None, None, rendezvous
+            return None, None, None, rendezvous
         else:
             state = _RPCState(_UNARY_UNARY_INITIAL_DUE, None, None, None, None)
             operations = (
@@ -467,18 +462,17 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable):
                 cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS),
                 cygrpc.ReceiveStatusOnClientOperation(_EMPTY_FLAGS),
             )
-            return state, operations, deadline, deadline_timespec, None
+            return state, operations, deadline, None
 
     def _blocking(self, request, timeout, metadata, credentials):
-        state, operations, deadline, deadline_timespec, rendezvous = self._prepare(
+        state, operations, deadline, rendezvous = self._prepare(
             request, timeout, metadata)
         if rendezvous:
             raise rendezvous
         else:
             completion_queue = cygrpc.CompletionQueue()
             call = self._channel.create_call(None, 0, completion_queue,
-                                             self._method, None,
-                                             deadline_timespec)
+                                             self._method, None, deadline)
             if credentials is not None:
                 call.set_credentials(credentials._credentials)
             call_error = call.start_client_batch(operations, None)
@@ -498,13 +492,13 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable):
         return _end_unary_response_blocking(state, call, True, deadline)
 
     def future(self, request, timeout=None, metadata=None, credentials=None):
-        state, operations, deadline, deadline_timespec, rendezvous = self._prepare(
+        state, operations, deadline, rendezvous = self._prepare(
             request, timeout, metadata)
         if rendezvous:
             return rendezvous
         else:
             call, drive_call = self._managed_call(None, 0, self._method, None,
-                                                  deadline_timespec)
+                                                  deadline)
             if credentials is not None:
                 call.set_credentials(credentials._credentials)
             event_handler = _event_handler(state, call,
@@ -530,14 +524,14 @@ class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable):
         self._response_deserializer = response_deserializer
 
     def __call__(self, request, timeout=None, metadata=None, credentials=None):
-        deadline, deadline_timespec, serialized_request, rendezvous = (
-            _start_unary_request(request, timeout, self._request_serializer))
+        deadline, serialized_request, rendezvous = (_start_unary_request(
+            request, timeout, self._request_serializer))
         if serialized_request is None:
             raise rendezvous
         else:
             state = _RPCState(_UNARY_STREAM_INITIAL_DUE, None, None, None, None)
             call, drive_call = self._managed_call(None, 0, self._method, None,
-                                                  deadline_timespec)
+                                                  deadline)
             if credentials is not None:
                 call.set_credentials(credentials._credentials)
             event_handler = _event_handler(state, call,
@@ -573,11 +567,11 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable):
         self._response_deserializer = response_deserializer
 
     def _blocking(self, request_iterator, timeout, metadata, credentials):
-        deadline, deadline_timespec = _deadline(timeout)
+        deadline = _deadline(timeout)
         state = _RPCState(_STREAM_UNARY_INITIAL_DUE, None, None, None, None)
         completion_queue = cygrpc.CompletionQueue()
         call = self._channel.create_call(None, 0, completion_queue,
-                                         self._method, None, deadline_timespec)
+                                         self._method, None, deadline)
         if credentials is not None:
             call.set_credentials(credentials._credentials)
         with state.condition:
@@ -624,10 +618,10 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable):
                timeout=None,
                metadata=None,
                credentials=None):
-        deadline, deadline_timespec = _deadline(timeout)
+        deadline = _deadline(timeout)
         state = _RPCState(_STREAM_UNARY_INITIAL_DUE, None, None, None, None)
         call, drive_call = self._managed_call(None, 0, self._method, None,
-                                              deadline_timespec)
+                                              deadline)
         if credentials is not None:
             call.set_credentials(credentials._credentials)
         event_handler = _event_handler(state, call, self._response_deserializer)
@@ -665,10 +659,10 @@ class _StreamStreamMultiCallable(grpc.StreamStreamMultiCallable):
                  timeout=None,
                  metadata=None,
                  credentials=None):
-        deadline, deadline_timespec = _deadline(timeout)
+        deadline = _deadline(timeout)
         state = _RPCState(_STREAM_STREAM_INITIAL_DUE, None, None, None, None)
         call, drive_call = self._managed_call(None, 0, self._method, None,
-                                              deadline_timespec)
+                                              deadline)
         if credentials is not None:
             call.set_credentials(credentials._credentials)
         event_handler = _event_handler(state, call, self._response_deserializer)
@@ -737,7 +731,8 @@ def _channel_managed_call_management(state):
       flags: An integer bitfield of call flags.
       method: The RPC method.
       host: A host string for the created call.
-      deadline: A cygrpc.Timespec to be the deadline of the created call.
+      deadline: A float to be the deadline of the created call or None if the
+        call is to have an infinite deadline.
 
     Returns:
       A cygrpc.Call with which to conduct an RPC and a function to call if
@@ -827,8 +822,8 @@ def _poll_connectivity(state, channel, initial_try_to_connect):
     completion_queue = cygrpc.CompletionQueue()
     while True:
         channel.watch_connectivity_state(connectivity,
-                                         cygrpc.Timespec(time.time() + 0.2),
-                                         completion_queue, None)
+                                         time.time() + 0.2, completion_queue,
+                                         None)
         event = completion_queue.poll()
         with state.lock:
             if not state.callbacks_and_connectivities and not state.try_to_connect:
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
index 443d534d7eb..efe5f2e0db2 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
@@ -42,7 +42,7 @@ cdef class Channel:
 
   def create_call(self, Call parent, int flags,
                   CompletionQueue queue not None,
-                  method, host, Timespec deadline not None):
+                  method, host, object deadline):
     if queue.is_shutting_down:
       raise ValueError("queue must not be shutting down or shutdown")
     cdef grpc_slice method_slice = _slice_from_bytes(method)
@@ -56,14 +56,13 @@ cdef class Channel:
     cdef grpc_call *parent_call = NULL
     if parent is not None:
       parent_call = parent.c_call
-    with nogil:
-      operation_call.c_call = grpc_channel_create_call(
-          self.c_channel, parent_call, flags,
-          queue.c_completion_queue, method_slice, host_slice_ptr,
-          deadline.c_time, NULL)
-      grpc_slice_unref(method_slice)
-      if host_slice_ptr:
-        grpc_slice_unref(host_slice)
+    operation_call.c_call = grpc_channel_create_call(
+        self.c_channel, parent_call, flags,
+        queue.c_completion_queue, method_slice, host_slice_ptr,
+        _timespec_from_time(deadline), NULL)
+    grpc_slice_unref(method_slice)
+    if host_slice_ptr:
+      grpc_slice_unref(host_slice)
     return operation_call
 
   def check_connectivity_state(self, bint try_to_connect):
@@ -75,13 +74,12 @@ cdef class Channel:
 
   def watch_connectivity_state(
       self, grpc_connectivity_state last_observed_state,
-      Timespec deadline not None, CompletionQueue queue not None, tag):
+      object deadline, CompletionQueue queue not None, tag):
     cdef _ConnectivityTag connectivity_tag = _ConnectivityTag(tag)
     cpython.Py_INCREF(connectivity_tag)
-    with nogil:
-      grpc_channel_watch_connectivity_state(
-          self.c_channel, last_observed_state, deadline.c_time,
-          queue.c_completion_queue, <cpython.PyObject *>connectivity_tag)
+    grpc_channel_watch_connectivity_state(
+        self.c_channel, last_observed_state, _timespec_from_time(deadline),
+        queue.c_completion_queue, <cpython.PyObject *>connectivity_tag)
 
   def target(self):
     cdef char *target = NULL
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
index e259789b35f..40496d11243 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
@@ -52,17 +52,18 @@ cdef class CompletionQueue:
       cpython.Py_DECREF(tag)
       return tag.event(event)
 
-  def poll(self, Timespec deadline=None):
+  def poll(self, deadline=None):
     # We name this 'poll' to avoid problems with CPython's expectations for
     # 'special' methods (like next and __next__).
     cdef gpr_timespec c_increment
     cdef gpr_timespec c_timeout
     cdef gpr_timespec c_deadline
+    if deadline is None:
+      c_deadline = gpr_inf_future(GPR_CLOCK_REALTIME)
+    else:
+      c_deadline = _timespec_from_time(deadline)
     with nogil:
       c_increment = gpr_time_from_millis(_INTERRUPT_CHECK_PERIOD_MS, GPR_TIMESPAN)
-      c_deadline = gpr_inf_future(GPR_CLOCK_REALTIME)
-      if deadline is not None:
-        c_deadline = deadline.c_time
 
       while True:
         c_timeout = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c_increment)
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi
index 7b2482d9479..297bbadfe08 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi
@@ -18,11 +18,6 @@ cdef grpc_slice _copy_slice(grpc_slice slice) nogil
 cdef grpc_slice _slice_from_bytes(bytes value) nogil
 
 
-cdef class Timespec:
-
-  cdef gpr_timespec c_time
-
-
 cdef class CallDetails:
 
   cdef grpc_call_details c_details
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
index bc2cd0338e0..b2343b53d6a 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
@@ -123,74 +123,6 @@ class CompressionLevel:
   high = GRPC_COMPRESS_LEVEL_HIGH
 
 
-cdef class Timespec:
-
-  def __cinit__(self, time):
-    if time is None:
-      with nogil:
-        self.c_time = gpr_now(GPR_CLOCK_REALTIME)
-      return
-    if isinstance(time, int):
-      time = float(time)
-    if isinstance(time, float):
-      if time == float("+inf"):
-        with nogil:
-          self.c_time = gpr_inf_future(GPR_CLOCK_REALTIME)
-      elif time == float("-inf"):
-        with nogil:
-          self.c_time = gpr_inf_past(GPR_CLOCK_REALTIME)
-      else:
-        self.c_time.seconds = time
-        self.c_time.nanoseconds = (time - float(self.c_time.seconds)) * 1e9
-        self.c_time.clock_type = GPR_CLOCK_REALTIME
-    elif isinstance(time, Timespec):
-      self.c_time = (<Timespec>time).c_time
-    else:
-      raise TypeError("expected time to be float, int, or Timespec, not {}"
-                          .format(type(time)))
-
-  @property
-  def seconds(self):
-    # TODO(atash) ensure that everywhere a Timespec is created that it's
-    # converted to GPR_CLOCK_REALTIME then and not every time someone wants to
-    # read values off in Python.
-    cdef gpr_timespec real_time
-    with nogil:
-      real_time = (
-          gpr_convert_clock_type(self.c_time, GPR_CLOCK_REALTIME))
-    return real_time.seconds
-
-  @property
-  def nanoseconds(self):
-    cdef gpr_timespec real_time = (
-        gpr_convert_clock_type(self.c_time, GPR_CLOCK_REALTIME))
-    return real_time.nanoseconds
-
-  def __float__(self):
-    cdef gpr_timespec real_time = (
-        gpr_convert_clock_type(self.c_time, GPR_CLOCK_REALTIME))
-    return <double>real_time.seconds + <double>real_time.nanoseconds / 1e9
-
-  def __richcmp__(Timespec self not None, Timespec other not None, int op):
-    cdef gpr_timespec self_c_time = self.c_time
-    cdef gpr_timespec other_c_time = other.c_time
-    cdef int result = gpr_time_cmp(self_c_time, other_c_time)
-    if op == 0:  # <
-      return result < 0
-    elif op == 2:  # ==
-      return result == 0
-    elif op == 4:  # >
-      return result > 0
-    elif op == 1:  # <=
-      return result <= 0
-    elif op == 3:  # !=
-      return result != 0
-    elif op == 5:  # >=
-      return result >= 0
-    else:
-      raise ValueError('__richcmp__ `op` contract violated')
-
-
 cdef class CallDetails:
 
   def __cinit__(self):
@@ -213,9 +145,7 @@ cdef class CallDetails:
 
   @property
   def deadline(self):
-    timespec = Timespec(float("-inf"))
-    timespec.c_time = self.c_details.deadline
-    return timespec
+    return _time_from_timespec(self.c_details.deadline)
 
 
 cdef class SslPemKeyCertPair:
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
index c19beccde63..e5d28a85d58 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
@@ -106,7 +106,7 @@ cdef class Server:
     with nogil:
       grpc_server_start(self.c_server)
     # Ensure the core has gotten a chance to do the start-up work
-    self.backup_shutdown_queue.poll(Timespec(None))
+    self.backup_shutdown_queue.poll(deadline=time.time())
 
   def add_http2_port(self, bytes address,
                      ServerCredentials server_credentials=None):
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/time.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/time.pxd.pxi
new file mode 100644
index 00000000000..ce67c61eaf6
--- /dev/null
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/time.pxd.pxi
@@ -0,0 +1,19 @@
+# Copyright 2018 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+cdef gpr_timespec _timespec_from_time(object time)
+
+
+cdef double _time_from_timespec(gpr_timespec timespec)
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/time.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/time.pyx.pxi
new file mode 100644
index 00000000000..7a668680b8f
--- /dev/null
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/time.pyx.pxi
@@ -0,0 +1,30 @@
+# Copyright 2018 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+cdef gpr_timespec _timespec_from_time(object time):
+  cdef gpr_timespec timespec
+  if time is None:
+    return gpr_inf_future(GPR_CLOCK_REALTIME)
+  else:
+    timespec.seconds = time
+    timespec.nanoseconds = (time - float(timespec.seconds)) * 1e9
+    timespec.clock_type = GPR_CLOCK_REALTIME
+    return timespec
+
+
+cdef double _time_from_timespec(gpr_timespec timespec):
+  cdef gpr_timespec real_timespec = gpr_convert_clock_type(
+      timespec, GPR_CLOCK_REALTIME)
+  return <double>real_timespec.seconds + <double>real_timespec.nanoseconds / 1e9
diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pxd b/src/python/grpcio/grpc/_cython/cygrpc.pxd
index b32fa518fc2..01e2da6d542 100644
--- a/src/python/grpcio/grpc/_cython/cygrpc.pxd
+++ b/src/python/grpcio/grpc/_cython/cygrpc.pxd
@@ -25,3 +25,4 @@ include "_cygrpc/records.pxd.pxi"
 include "_cygrpc/security.pxd.pxi"
 include "_cygrpc/server.pxd.pxi"
 include "_cygrpc/tag.pxd.pxi"
+include "_cygrpc/time.pxd.pxi"
diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pyx b/src/python/grpcio/grpc/_cython/cygrpc.pyx
index 5106394708b..d8ac84a317d 100644
--- a/src/python/grpcio/grpc/_cython/cygrpc.pyx
+++ b/src/python/grpcio/grpc/_cython/cygrpc.pyx
@@ -32,6 +32,7 @@ include "_cygrpc/records.pyx.pxi"
 include "_cygrpc/security.pyx.pxi"
 include "_cygrpc/server.pyx.pxi"
 include "_cygrpc/tag.pyx.pxi"
+include "_cygrpc/time.pyx.pxi"
 
 #
 # initialize gRPC
diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py
index 1cdb2d45b6c..9402941bab1 100644
--- a/src/python/grpcio/grpc/_server.py
+++ b/src/python/grpcio/grpc/_server.py
@@ -220,8 +220,7 @@ class _Context(grpc.ServicerContext):
             return self._state.client is not _CANCELLED and not self._state.statused
 
     def time_remaining(self):
-        return max(
-            float(self._rpc_event.call_details.deadline) - time.time(), 0)
+        return max(self._rpc_event.call_details.deadline - time.time(), 0)
 
     def cancel(self):
         self._rpc_event.call.cancel()
diff --git a/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py b/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py
index b81d6fbc617..2ca1fa82f4f 100644
--- a/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py
@@ -20,7 +20,6 @@ from grpc._cython import cygrpc
 from grpc.framework.foundation import logging_pool
 from tests.unit.framework.common import test_constants
 
-_INFINITE_FUTURE = cygrpc.Timespec(float('+inf'))
 _EMPTY_FLAGS = 0
 _EMPTY_METADATA = ()
 
@@ -171,9 +170,9 @@ class CancelManyCallsTest(unittest.TestCase):
         with client_condition:
             client_calls = []
             for index in range(test_constants.RPC_CONCURRENCY):
-                client_call = channel.create_call(
-                    None, _EMPTY_FLAGS, client_completion_queue, b'/twinkies',
-                    None, _INFINITE_FUTURE)
+                client_call = channel.create_call(None, _EMPTY_FLAGS,
+                                                  client_completion_queue,
+                                                  b'/twinkies', None, None)
                 operations = (
                     cygrpc.SendInitialMetadataOperation(_EMPTY_METADATA,
                                                         _EMPTY_FLAGS),
diff --git a/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py b/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py
index 4eeb34b92ec..c22c77ddbd2 100644
--- a/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py
@@ -31,9 +31,9 @@ def _connectivity_loop(channel, completion_queue):
     for _ in range(100):
         connectivity = channel.check_connectivity_state(True)
         channel.watch_connectivity_state(connectivity,
-                                         cygrpc.Timespec(time.time() + 0.2),
-                                         completion_queue, None)
-        completion_queue.poll(deadline=cygrpc.Timespec(float('+inf')))
+                                         time.time() + 0.2, completion_queue,
+                                         None)
+        completion_queue.poll()
 
 
 def _create_loop_destroy():
diff --git a/src/python/grpcio_tests/tests/unit/_cython/_common.py b/src/python/grpcio_tests/tests/unit/_cython/_common.py
index ffd226fa95a..d4b01ca38b4 100644
--- a/src/python/grpcio_tests/tests/unit/_cython/_common.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/_common.py
@@ -20,7 +20,6 @@ from grpc._cython import cygrpc
 
 RPC_COUNT = 4000
 
-INFINITE_FUTURE = cygrpc.Timespec(float('+inf'))
 EMPTY_FLAGS = 0
 
 INVOCATION_METADATA = (
diff --git a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py
index 4ef4ad33e56..7caa98f72da 100644
--- a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py
@@ -41,9 +41,9 @@ class Test(_common.RpcTest, unittest.TestCase):
                 server_request_call_tag,
             })
 
-        client_call = self.channel.create_call(
-            None, _common.EMPTY_FLAGS, self.client_completion_queue,
-            b'/twinkies', None, _common.INFINITE_FUTURE)
+        client_call = self.channel.create_call(None, _common.EMPTY_FLAGS,
+                                               self.client_completion_queue,
+                                               b'/twinkies', None, None)
         client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag'
         client_complete_rpc_tag = 'client_complete_rpc_tag'
         with self.client_condition:
diff --git a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py
index 85395c9680f..8582a39c010 100644
--- a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py
@@ -36,9 +36,9 @@ class Test(_common.RpcTest, unittest.TestCase):
                 server_request_call_tag,
             })
 
-        client_call = self.channel.create_call(
-            None, _common.EMPTY_FLAGS, self.client_completion_queue,
-            b'/twinkies', None, _common.INFINITE_FUTURE)
+        client_call = self.channel.create_call(None, _common.EMPTY_FLAGS,
+                                               self.client_completion_queue,
+                                               b'/twinkies', None, None)
         client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag'
         client_complete_rpc_tag = 'client_complete_rpc_tag'
         with self.client_condition:
diff --git a/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py b/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py
index 82ef25b2a7f..ecd23afda71 100644
--- a/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py
@@ -18,7 +18,6 @@ import unittest
 
 from grpc._cython import cygrpc
 
-_INFINITE_FUTURE = cygrpc.Timespec(float('+inf'))
 _EMPTY_FLAGS = 0
 _EMPTY_METADATA = ()
 
@@ -156,7 +155,7 @@ class ReadSomeButNotAllResponsesTest(unittest.TestCase):
 
         client_call = channel.create_call(None, _EMPTY_FLAGS,
                                           client_completion_queue, b'/twinkies',
-                                          None, _INFINITE_FUTURE)
+                                          None, None)
         client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag'
         client_complete_rpc_tag = 'client_complete_rpc_tag'
         with client_condition:
diff --git a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py
index 5f9b74ba989..561adf7dff0 100644
--- a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py
@@ -37,21 +37,6 @@ def _metadata_plugin(context, callback):
 
 class TypeSmokeTest(unittest.TestCase):
 
-    def testTimespec(self):
-        now = time.time()
-        now_timespec_a = cygrpc.Timespec(now)
-        now_timespec_b = cygrpc.Timespec(now)
-        self.assertAlmostEqual(now, float(now_timespec_a), places=8)
-        self.assertEqual(now_timespec_a, now_timespec_b)
-        self.assertLess(cygrpc.Timespec(now - 1), cygrpc.Timespec(now))
-        self.assertGreater(cygrpc.Timespec(now + 1), cygrpc.Timespec(now))
-        self.assertGreaterEqual(cygrpc.Timespec(now + 1), cygrpc.Timespec(now))
-        self.assertGreaterEqual(cygrpc.Timespec(now), cygrpc.Timespec(now))
-        self.assertLessEqual(cygrpc.Timespec(now - 1), cygrpc.Timespec(now))
-        self.assertLessEqual(cygrpc.Timespec(now), cygrpc.Timespec(now))
-        self.assertNotEqual(cygrpc.Timespec(now - 1), cygrpc.Timespec(now))
-        self.assertNotEqual(cygrpc.Timespec(now + 1), cygrpc.Timespec(now))
-
     def testCompletionQueueUpDown(self):
         completion_queue = cygrpc.CompletionQueue()
         del completion_queue
@@ -147,7 +132,7 @@ class ServerClientMixin(object):
             try:
                 call_result = call.start_client_batch(operations, tag)
                 self.assertEqual(cygrpc.CallError.ok, call_result)
-                event = queue.poll(deadline)
+                event = queue.poll(deadline=deadline)
                 self.assertEqual(cygrpc.CompletionType.operation_complete,
                                  event.completion_type)
                 self.assertTrue(event.success)
@@ -176,8 +161,6 @@ class ServerClientMixin(object):
         RESPONSE = b'his name is robert paulson'
         METHOD = b'twinkies'
 
-        cygrpc_deadline = cygrpc.Timespec(DEADLINE)
-
         server_request_tag = object()
         request_call_result = self.server.request_call(
             self.server_completion_queue, self.server_completion_queue,
@@ -188,7 +171,7 @@ class ServerClientMixin(object):
         client_call_tag = object()
         client_call = self.client_channel.create_call(
             None, 0, self.client_completion_queue, METHOD, self.host_argument,
-            cygrpc_deadline)
+            DEADLINE)
         client_initial_metadata = (
             (
                 CLIENT_METADATA_ASCII_KEY,
@@ -210,9 +193,9 @@ class ServerClientMixin(object):
         ], client_call_tag)
         self.assertEqual(cygrpc.CallError.ok, client_start_batch_result)
         client_event_future = test_utilities.CompletionQueuePollFuture(
-            self.client_completion_queue, cygrpc_deadline)
+            self.client_completion_queue, DEADLINE)
 
-        request_event = self.server_completion_queue.poll(cygrpc_deadline)
+        request_event = self.server_completion_queue.poll(deadline=DEADLINE)
         self.assertEqual(cygrpc.CompletionType.operation_complete,
                          request_event.completion_type)
         self.assertIsInstance(request_event.call, cygrpc.Call)
@@ -223,7 +206,7 @@ class ServerClientMixin(object):
         self.assertEqual(METHOD, request_event.call_details.method)
         self.assertEqual(self.expected_host, request_event.call_details.host)
         self.assertLess(
-            abs(DEADLINE - float(request_event.call_details.deadline)),
+            abs(DEADLINE - request_event.call_details.deadline),
             DEADLINE_TOLERANCE)
 
         server_call_tag = object()
@@ -248,7 +231,7 @@ class ServerClientMixin(object):
         ], server_call_tag)
         self.assertEqual(cygrpc.CallError.ok, server_start_batch_result)
 
-        server_event = self.server_completion_queue.poll(cygrpc_deadline)
+        server_event = self.server_completion_queue.poll(deadline=DEADLINE)
         client_event = client_event_future.result()
 
         self.assertEqual(6, len(client_event.batch_operations))
@@ -310,7 +293,6 @@ class ServerClientMixin(object):
         DEADLINE_TOLERANCE = 0.25
         METHOD = b'twinkies'
 
-        cygrpc_deadline = cygrpc.Timespec(DEADLINE)
         empty_metadata = ()
 
         server_request_tag = object()
@@ -319,26 +301,26 @@ class ServerClientMixin(object):
                                  server_request_tag)
         client_call = self.client_channel.create_call(
             None, 0, self.client_completion_queue, METHOD, self.host_argument,
-            cygrpc_deadline)
+            DEADLINE)
 
         # Prologue
         def perform_client_operations(operations, description):
             return self._perform_operations(operations, client_call,
                                             self.client_completion_queue,
-                                            cygrpc_deadline, description)
+                                            DEADLINE, description)
 
         client_event_future = perform_client_operations([
             cygrpc.SendInitialMetadataOperation(empty_metadata, _EMPTY_FLAGS),
             cygrpc.ReceiveInitialMetadataOperation(_EMPTY_FLAGS),
         ], "Client prologue")
 
-        request_event = self.server_completion_queue.poll(cygrpc_deadline)
+        request_event = self.server_completion_queue.poll(deadline=DEADLINE)
         server_call = request_event.call
 
         def perform_server_operations(operations, description):
             return self._perform_operations(operations, server_call,
                                             self.server_completion_queue,
-                                            cygrpc_deadline, description)
+                                            DEADLINE, description)
 
         server_event_future = perform_server_operations([
             cygrpc.SendInitialMetadataOperation(empty_metadata, _EMPTY_FLAGS),
diff --git a/src/python/grpcio_tests/tests/unit/_cython/test_utilities.py b/src/python/grpcio_tests/tests/unit/_cython/test_utilities.py
index 8e91161f803..4a00b9ef2f1 100644
--- a/src/python/grpcio_tests/tests/unit/_cython/test_utilities.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/test_utilities.py
@@ -49,4 +49,4 @@ class CompletionQueuePollFuture(SimpleFuture):
 
     def __init__(self, completion_queue, deadline):
         super(CompletionQueuePollFuture,
-              self).__init__(lambda: completion_queue.poll(deadline))
+              self).__init__(lambda: completion_queue.poll(deadline=deadline))

From 9ee9c924d8f23604a8ab78089b1706b5b00e971a Mon Sep 17 00:00:00 2001
From: Dan Zhang <danzh@google.com>
Date: Wed, 10 Jan 2018 18:04:25 -0500
Subject: [PATCH 069/161] change to pass in value

---
 src/core/lib/iomgr/udp_server.cc   | 38 ++++++++++++++++++++----------
 src/core/lib/iomgr/udp_server.h    |  1 +
 test/core/iomgr/udp_server_test.cc | 23 +++++++++++-------
 3 files changed, 41 insertions(+), 21 deletions(-)

diff --git a/src/core/lib/iomgr/udp_server.cc b/src/core/lib/iomgr/udp_server.cc
index 6dde7b96112..8deb0ea544d 100644
--- a/src/core/lib/iomgr/udp_server.cc
+++ b/src/core/lib/iomgr/udp_server.cc
@@ -280,11 +280,10 @@ static int bind_socket(grpc_socket_factory* socket_factory, int sockfd,
 
 /* Prepare a recently-created socket for listening. */
 static int prepare_socket(grpc_socket_factory* socket_factory, int fd,
-                          const grpc_resolved_address* addr) {
+                          const grpc_resolved_address* addr,
+                          size_t rcv_buf_size, size_t snd_buf_size) {
   grpc_resolved_address sockname_temp;
   struct sockaddr* addr_ptr = (struct sockaddr*)addr->addr;
-  /* Set send/receive socket buffers to 10 MB */
-  int buffer_size_bytes = 1024 * 1024 * 10;
 
   if (fd < 0) {
     goto error;
@@ -325,18 +324,25 @@ static int prepare_socket(grpc_socket_factory* socket_factory, int fd,
     goto error;
   }
 
-  if (grpc_set_socket_sndbuf(fd, buffer_size_bytes) != GRPC_ERROR_NONE) {
-    gpr_log(GPR_ERROR, "Failed to set send buffer size to %d bytes",
-            buffer_size_bytes);
+  if (grpc_set_socket_sndbuf(fd, snd_buf_size) != GRPC_ERROR_NONE) {
+    gpr_log(GPR_ERROR, "Failed to set send buffer size to %lu bytes",
+            snd_buf_size);
     goto error;
   }
 
-  if (grpc_set_socket_rcvbuf(fd, buffer_size_bytes) != GRPC_ERROR_NONE) {
-    gpr_log(GPR_ERROR, "Failed to set receive buffer size to %d bytes",
-            buffer_size_bytes);
+  if (grpc_set_socket_rcvbuf(fd, rcv_buf_size) != GRPC_ERROR_NONE) {
+    gpr_log(GPR_ERROR, "Failed to set receive buffer size to %lu bytes",
+            rcv_buf_size);
     goto error;
   }
 
+  {
+    int get_overflow = 1;
+    if (0 != setsockopt(fd, SOL_SOCKET, SO_RXQ_OVFL, &get_overflow,
+                        sizeof(get_overflow))) {
+      gpr_log(GPR_INFO, "Failed to set socket overflow support");
+    }
+  }
   return grpc_sockaddr_get_port(&sockname_temp);
 
 error:
@@ -451,6 +457,8 @@ static void on_write(void* arg, grpc_error* error) {
 
 static int add_socket_to_server(grpc_udp_server* s, int fd,
                                 const grpc_resolved_address* addr,
+                                size_t rcv_buf_size,
+                                size_t snd_buf_size,
                                 grpc_udp_server_start_cb start_cb,
                                 grpc_udp_server_read_cb read_cb,
                                 grpc_udp_server_write_cb write_cb,
@@ -460,7 +468,8 @@ static int add_socket_to_server(grpc_udp_server* s, int fd,
   char* addr_str;
   char* name;
 
-  port = prepare_socket(s->socket_factory, fd, addr);
+  port =
+      prepare_socket(s->socket_factory, fd, addr, rcv_buf_size, snd_buf_size);
   if (port >= 0) {
     grpc_sockaddr_to_string(&addr_str, addr, 1);
     gpr_asprintf(&name, "udp-server-listener:%s", addr_str);
@@ -495,6 +504,7 @@ static int add_socket_to_server(grpc_udp_server* s, int fd,
 
 int grpc_udp_server_add_port(grpc_udp_server* s,
                              const grpc_resolved_address* addr,
+                             size_t rcv_buf_size, size_t snd_buf_size,
                              grpc_udp_server_start_cb start_cb,
                              grpc_udp_server_read_cb read_cb,
                              grpc_udp_server_write_cb write_cb,
@@ -545,8 +555,9 @@ int grpc_udp_server_add_port(grpc_udp_server* s,
     // TODO(rjshade): Test and propagate the returned grpc_error*:
     GRPC_ERROR_UNREF(grpc_create_dualstack_socket_using_factory(
         s->socket_factory, addr, SOCK_DGRAM, IPPROTO_UDP, &dsmode, &fd));
-    allocated_port1 = add_socket_to_server(s, fd, addr, start_cb, read_cb,
-                                           write_cb, orphan_cb);
+    allocated_port1 =
+        add_socket_to_server(s, fd, addr, rcv_buf_size, snd_buf_size, start_cb,
+                             read_cb, write_cb, orphan_cb);
     if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) {
       goto done;
     }
@@ -569,7 +580,8 @@ int grpc_udp_server_add_port(grpc_udp_server* s,
     addr = &addr4_copy;
   }
   allocated_port2 =
-      add_socket_to_server(s, fd, addr, start_cb, read_cb, write_cb, orphan_cb);
+      add_socket_to_server(s, fd, addr, rcv_buf_size, snd_buf_size, start_cb,
+                           read_cb, write_cb, orphan_cb);
 
 done:
   gpr_free(allocated_addr);
diff --git a/src/core/lib/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h
index a469ab9be57..ec18716f396 100644
--- a/src/core/lib/iomgr/udp_server.h
+++ b/src/core/lib/iomgr/udp_server.h
@@ -68,6 +68,7 @@ int grpc_udp_server_get_fd(grpc_udp_server* s, unsigned port_index);
                   all of the multiple socket port matching logic in one place */
 int grpc_udp_server_add_port(grpc_udp_server* s,
                              const grpc_resolved_address* addr,
+                             size_t rcv_buf_size, size_t snd_buf_size,
                              grpc_udp_server_start_cb start_cb,
                              grpc_udp_server_read_cb read_cb,
                              grpc_udp_server_write_cb write_cb,
diff --git a/test/core/iomgr/udp_server_test.cc b/test/core/iomgr/udp_server_test.cc
index dc1248bc1c9..5245840ba95 100644
--- a/test/core/iomgr/udp_server_test.cc
+++ b/test/core/iomgr/udp_server_test.cc
@@ -51,6 +51,9 @@ static int g_number_of_bytes_read = 0;
 static int g_number_of_orphan_calls = 0;
 static int g_number_of_starts = 0;
 
+size_t rcv_buf_size = 1024;
+size_t snd_buf_size = 1024;
+
 static void on_start(grpc_fd* emfd, void* user_data) { g_number_of_starts++; }
 
 static bool on_read(grpc_fd* emfd) {
@@ -177,8 +180,9 @@ static void test_no_op_with_port(void) {
   memset(&resolved_addr, 0, sizeof(resolved_addr));
   resolved_addr.len = sizeof(struct sockaddr_in);
   addr->sin_family = AF_INET;
-  GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, on_start, on_read,
-                                      on_write, on_fd_orphaned));
+  GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size,
+                                      snd_buf_size, on_start, on_read, on_write,
+                                      on_fd_orphaned));
 
   grpc_udp_server_destroy(s, nullptr);
 
@@ -207,8 +211,9 @@ static void test_no_op_with_port_and_socket_factory(void) {
   memset(&resolved_addr, 0, sizeof(resolved_addr));
   resolved_addr.len = sizeof(struct sockaddr_in);
   addr->sin_family = AF_INET;
-  GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, on_start, on_read,
-                                      on_write, on_fd_orphaned));
+  GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size,
+                                      snd_buf_size, on_start, on_read, on_write,
+                                      on_fd_orphaned));
   GPR_ASSERT(socket_factory->number_of_socket_calls == 1);
   GPR_ASSERT(socket_factory->number_of_bind_calls == 1);
 
@@ -233,8 +238,9 @@ static void test_no_op_with_port_and_start(void) {
   memset(&resolved_addr, 0, sizeof(resolved_addr));
   resolved_addr.len = sizeof(struct sockaddr_in);
   addr->sin_family = AF_INET;
-  GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, on_start, on_read,
-                                      on_write, on_fd_orphaned));
+  GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size,
+                                      snd_buf_size, on_start, on_read, on_write,
+                                      on_fd_orphaned));
 
   grpc_udp_server_start(s, nullptr, 0, nullptr);
   GPR_ASSERT(g_number_of_starts == 1);
@@ -265,8 +271,9 @@ static void test_receive(int number_of_clients) {
   memset(&resolved_addr, 0, sizeof(resolved_addr));
   resolved_addr.len = sizeof(struct sockaddr_storage);
   addr->ss_family = AF_INET;
-  GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, on_start, on_read,
-                                      on_write, on_fd_orphaned));
+  GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size,
+                                      snd_buf_size, on_start, on_read, on_write,
+                                      on_fd_orphaned));
 
   svrfd = grpc_udp_server_get_fd(s, 0);
   GPR_ASSERT(svrfd >= 0);

From 1d91362f8124751ecfc1929df207006cabb41dae Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Wed, 10 Jan 2018 15:59:05 -0800
Subject: [PATCH 070/161] exec_ctx_fwd.h should never have been in public
 headers

---
 BUILD                                         |  1 -
 CMakeLists.txt                                | 10 -------
 Makefile                                      | 10 -------
 build.yaml                                    |  1 -
 gRPC-Core.podspec                             |  1 -
 grpc.gemspec                                  |  1 -
 include/grpc/impl/codegen/exec_ctx_fwd.h      | 26 -------------------
 include/grpc/impl/codegen/grpc_types.h        |  1 -
 include/grpc/impl/codegen/slice.h             |  1 -
 include/grpc/module.modulemap                 |  1 -
 package.xml                                   |  1 -
 src/core/lib/iomgr/closure.h                  |  1 -
 src/core/lib/iomgr/iomgr.h                    |  1 -
 .../core/surface/public_headers_must_be_c89.c |  1 -
 tools/doxygen/Doxyfile.c++                    |  1 -
 tools/doxygen/Doxyfile.c++.internal           |  1 -
 tools/doxygen/Doxyfile.core                   |  1 -
 tools/doxygen/Doxyfile.core.internal          |  1 -
 .../generated/sources_and_headers.json        |  2 --
 19 files changed, 63 deletions(-)
 delete mode 100644 include/grpc/impl/codegen/exec_ctx_fwd.h

diff --git a/BUILD b/BUILD
index dba6592f172..804c6cee02f 100644
--- a/BUILD
+++ b/BUILD
@@ -1006,7 +1006,6 @@ grpc_cc_library(
         "include/grpc/impl/codegen/byte_buffer_reader.h",
         "include/grpc/impl/codegen/compression_types.h",
         "include/grpc/impl/codegen/connectivity_state.h",
-        "include/grpc/impl/codegen/exec_ctx_fwd.h",
         "include/grpc/impl/codegen/grpc_types.h",
         "include/grpc/impl/codegen/propagation_bits.h",
         "include/grpc/impl/codegen/status.h",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index eed12052685..fa020e009dc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1080,7 +1080,6 @@ foreach(_hdr
   include/grpc/impl/codegen/byte_buffer_reader.h
   include/grpc/impl/codegen/compression_types.h
   include/grpc/impl/codegen/connectivity_state.h
-  include/grpc/impl/codegen/exec_ctx_fwd.h
   include/grpc/impl/codegen/grpc_types.h
   include/grpc/impl/codegen/propagation_bits.h
   include/grpc/impl/codegen/slice.h
@@ -1394,7 +1393,6 @@ foreach(_hdr
   include/grpc/impl/codegen/byte_buffer_reader.h
   include/grpc/impl/codegen/compression_types.h
   include/grpc/impl/codegen/connectivity_state.h
-  include/grpc/impl/codegen/exec_ctx_fwd.h
   include/grpc/impl/codegen/grpc_types.h
   include/grpc/impl/codegen/propagation_bits.h
   include/grpc/impl/codegen/slice.h
@@ -1680,7 +1678,6 @@ foreach(_hdr
   include/grpc/impl/codegen/byte_buffer_reader.h
   include/grpc/impl/codegen/compression_types.h
   include/grpc/impl/codegen/connectivity_state.h
-  include/grpc/impl/codegen/exec_ctx_fwd.h
   include/grpc/impl/codegen/grpc_types.h
   include/grpc/impl/codegen/propagation_bits.h
   include/grpc/impl/codegen/slice.h
@@ -1950,7 +1947,6 @@ foreach(_hdr
   include/grpc/impl/codegen/byte_buffer_reader.h
   include/grpc/impl/codegen/compression_types.h
   include/grpc/impl/codegen/connectivity_state.h
-  include/grpc/impl/codegen/exec_ctx_fwd.h
   include/grpc/impl/codegen/grpc_types.h
   include/grpc/impl/codegen/propagation_bits.h
   include/grpc/impl/codegen/slice.h
@@ -2239,7 +2235,6 @@ foreach(_hdr
   include/grpc/impl/codegen/byte_buffer_reader.h
   include/grpc/impl/codegen/compression_types.h
   include/grpc/impl/codegen/connectivity_state.h
-  include/grpc/impl/codegen/exec_ctx_fwd.h
   include/grpc/impl/codegen/grpc_types.h
   include/grpc/impl/codegen/propagation_bits.h
   include/grpc/impl/codegen/slice.h
@@ -2552,7 +2547,6 @@ foreach(_hdr
   include/grpc/impl/codegen/byte_buffer_reader.h
   include/grpc/impl/codegen/compression_types.h
   include/grpc/impl/codegen/connectivity_state.h
-  include/grpc/impl/codegen/exec_ctx_fwd.h
   include/grpc/impl/codegen/grpc_types.h
   include/grpc/impl/codegen/propagation_bits.h
   include/grpc/impl/codegen/slice.h
@@ -3038,7 +3032,6 @@ foreach(_hdr
   include/grpc/impl/codegen/byte_buffer_reader.h
   include/grpc/impl/codegen/compression_types.h
   include/grpc/impl/codegen/connectivity_state.h
-  include/grpc/impl/codegen/exec_ctx_fwd.h
   include/grpc/impl/codegen/grpc_types.h
   include/grpc/impl/codegen/propagation_bits.h
   include/grpc/impl/codegen/slice.h
@@ -3438,7 +3431,6 @@ foreach(_hdr
   include/grpc/impl/codegen/byte_buffer_reader.h
   include/grpc/impl/codegen/compression_types.h
   include/grpc/impl/codegen/connectivity_state.h
-  include/grpc/impl/codegen/exec_ctx_fwd.h
   include/grpc/impl/codegen/grpc_types.h
   include/grpc/impl/codegen/propagation_bits.h
   include/grpc/impl/codegen/slice.h
@@ -3579,7 +3571,6 @@ foreach(_hdr
   include/grpc/impl/codegen/byte_buffer_reader.h
   include/grpc/impl/codegen/compression_types.h
   include/grpc/impl/codegen/connectivity_state.h
-  include/grpc/impl/codegen/exec_ctx_fwd.h
   include/grpc/impl/codegen/grpc_types.h
   include/grpc/impl/codegen/propagation_bits.h
   include/grpc/impl/codegen/slice.h
@@ -3784,7 +3775,6 @@ foreach(_hdr
   include/grpc/impl/codegen/byte_buffer_reader.h
   include/grpc/impl/codegen/compression_types.h
   include/grpc/impl/codegen/connectivity_state.h
-  include/grpc/impl/codegen/exec_ctx_fwd.h
   include/grpc/impl/codegen/grpc_types.h
   include/grpc/impl/codegen/propagation_bits.h
   include/grpc/impl/codegen/slice.h
diff --git a/Makefile b/Makefile
index 38b40804d64..10c65303139 100644
--- a/Makefile
+++ b/Makefile
@@ -3226,7 +3226,6 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/byte_buffer_reader.h \
     include/grpc/impl/codegen/compression_types.h \
     include/grpc/impl/codegen/connectivity_state.h \
-    include/grpc/impl/codegen/exec_ctx_fwd.h \
     include/grpc/impl/codegen/grpc_types.h \
     include/grpc/impl/codegen/propagation_bits.h \
     include/grpc/impl/codegen/slice.h \
@@ -3540,7 +3539,6 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/byte_buffer_reader.h \
     include/grpc/impl/codegen/compression_types.h \
     include/grpc/impl/codegen/connectivity_state.h \
-    include/grpc/impl/codegen/exec_ctx_fwd.h \
     include/grpc/impl/codegen/grpc_types.h \
     include/grpc/impl/codegen/propagation_bits.h \
     include/grpc/impl/codegen/slice.h \
@@ -3827,7 +3825,6 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/byte_buffer_reader.h \
     include/grpc/impl/codegen/compression_types.h \
     include/grpc/impl/codegen/connectivity_state.h \
-    include/grpc/impl/codegen/exec_ctx_fwd.h \
     include/grpc/impl/codegen/grpc_types.h \
     include/grpc/impl/codegen/propagation_bits.h \
     include/grpc/impl/codegen/slice.h \
@@ -4088,7 +4085,6 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/byte_buffer_reader.h \
     include/grpc/impl/codegen/compression_types.h \
     include/grpc/impl/codegen/connectivity_state.h \
-    include/grpc/impl/codegen/exec_ctx_fwd.h \
     include/grpc/impl/codegen/grpc_types.h \
     include/grpc/impl/codegen/propagation_bits.h \
     include/grpc/impl/codegen/slice.h \
@@ -4354,7 +4350,6 @@ PUBLIC_HEADERS_C += \
     include/grpc/impl/codegen/byte_buffer_reader.h \
     include/grpc/impl/codegen/compression_types.h \
     include/grpc/impl/codegen/connectivity_state.h \
-    include/grpc/impl/codegen/exec_ctx_fwd.h \
     include/grpc/impl/codegen/grpc_types.h \
     include/grpc/impl/codegen/propagation_bits.h \
     include/grpc/impl/codegen/slice.h \
@@ -4646,7 +4641,6 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/impl/codegen/byte_buffer_reader.h \
     include/grpc/impl/codegen/compression_types.h \
     include/grpc/impl/codegen/connectivity_state.h \
-    include/grpc/impl/codegen/exec_ctx_fwd.h \
     include/grpc/impl/codegen/grpc_types.h \
     include/grpc/impl/codegen/propagation_bits.h \
     include/grpc/impl/codegen/slice.h \
@@ -5133,7 +5127,6 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/impl/codegen/byte_buffer_reader.h \
     include/grpc/impl/codegen/compression_types.h \
     include/grpc/impl/codegen/connectivity_state.h \
-    include/grpc/impl/codegen/exec_ctx_fwd.h \
     include/grpc/impl/codegen/grpc_types.h \
     include/grpc/impl/codegen/propagation_bits.h \
     include/grpc/impl/codegen/slice.h \
@@ -5526,7 +5519,6 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/impl/codegen/byte_buffer_reader.h \
     include/grpc/impl/codegen/compression_types.h \
     include/grpc/impl/codegen/connectivity_state.h \
-    include/grpc/impl/codegen/exec_ctx_fwd.h \
     include/grpc/impl/codegen/grpc_types.h \
     include/grpc/impl/codegen/propagation_bits.h \
     include/grpc/impl/codegen/slice.h \
@@ -5644,7 +5636,6 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/impl/codegen/byte_buffer_reader.h \
     include/grpc/impl/codegen/compression_types.h \
     include/grpc/impl/codegen/connectivity_state.h \
-    include/grpc/impl/codegen/exec_ctx_fwd.h \
     include/grpc/impl/codegen/grpc_types.h \
     include/grpc/impl/codegen/propagation_bits.h \
     include/grpc/impl/codegen/slice.h \
@@ -5854,7 +5845,6 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/impl/codegen/byte_buffer_reader.h \
     include/grpc/impl/codegen/compression_types.h \
     include/grpc/impl/codegen/connectivity_state.h \
-    include/grpc/impl/codegen/exec_ctx_fwd.h \
     include/grpc/impl/codegen/grpc_types.h \
     include/grpc/impl/codegen/propagation_bits.h \
     include/grpc/impl/codegen/slice.h \
diff --git a/build.yaml b/build.yaml
index fef7d6189f7..9aff5585e01 100644
--- a/build.yaml
+++ b/build.yaml
@@ -485,7 +485,6 @@ filegroups:
   - include/grpc/impl/codegen/byte_buffer_reader.h
   - include/grpc/impl/codegen/compression_types.h
   - include/grpc/impl/codegen/connectivity_state.h
-  - include/grpc/impl/codegen/exec_ctx_fwd.h
   - include/grpc/impl/codegen/grpc_types.h
   - include/grpc/impl/codegen/propagation_bits.h
   - include/grpc/impl/codegen/slice.h
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index c127660dd50..c007348c818 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -152,7 +152,6 @@ Pod::Spec.new do |s|
                       'include/grpc/impl/codegen/byte_buffer_reader.h',
                       'include/grpc/impl/codegen/compression_types.h',
                       'include/grpc/impl/codegen/connectivity_state.h',
-                      'include/grpc/impl/codegen/exec_ctx_fwd.h',
                       'include/grpc/impl/codegen/grpc_types.h',
                       'include/grpc/impl/codegen/propagation_bits.h',
                       'include/grpc/impl/codegen/slice.h',
diff --git a/grpc.gemspec b/grpc.gemspec
index d1859952619..c7302785c74 100644
--- a/grpc.gemspec
+++ b/grpc.gemspec
@@ -149,7 +149,6 @@ Gem::Specification.new do |s|
   s.files += %w( include/grpc/impl/codegen/byte_buffer_reader.h )
   s.files += %w( include/grpc/impl/codegen/compression_types.h )
   s.files += %w( include/grpc/impl/codegen/connectivity_state.h )
-  s.files += %w( include/grpc/impl/codegen/exec_ctx_fwd.h )
   s.files += %w( include/grpc/impl/codegen/grpc_types.h )
   s.files += %w( include/grpc/impl/codegen/propagation_bits.h )
   s.files += %w( include/grpc/impl/codegen/slice.h )
diff --git a/include/grpc/impl/codegen/exec_ctx_fwd.h b/include/grpc/impl/codegen/exec_ctx_fwd.h
deleted file mode 100644
index 005ff14e7e4..00000000000
--- a/include/grpc/impl/codegen/exec_ctx_fwd.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- *
- * Copyright 2016 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef GRPC_IMPL_CODEGEN_EXEC_CTX_FWD_H
-#define GRPC_IMPL_CODEGEN_EXEC_CTX_FWD_H
-
-/* forward declaration for exec_ctx.h */
-struct grpc_exec_ctx;
-typedef struct grpc_exec_ctx grpc_exec_ctx;
-
-#endif /* GRPC_IMPL_CODEGEN_EXEC_CTX_FWD_H */
diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h
index fcbc8ac5a1e..d481a70ab90 100644
--- a/include/grpc/impl/codegen/grpc_types.h
+++ b/include/grpc/impl/codegen/grpc_types.h
@@ -22,7 +22,6 @@
 #include <grpc/impl/codegen/port_platform.h>
 
 #include <grpc/impl/codegen/compression_types.h>
-#include <grpc/impl/codegen/exec_ctx_fwd.h>
 #include <grpc/impl/codegen/gpr_types.h>
 #include <grpc/impl/codegen/slice.h>
 #include <grpc/impl/codegen/status.h>
diff --git a/include/grpc/impl/codegen/slice.h b/include/grpc/impl/codegen/slice.h
index ad026b685ed..a3cd1f1bbeb 100644
--- a/include/grpc/impl/codegen/slice.h
+++ b/include/grpc/impl/codegen/slice.h
@@ -23,7 +23,6 @@
 
 #include <stddef.h>
 
-#include <grpc/impl/codegen/exec_ctx_fwd.h>
 #include <grpc/impl/codegen/gpr_slice.h>
 
 typedef struct grpc_slice grpc_slice;
diff --git a/include/grpc/module.modulemap b/include/grpc/module.modulemap
index 67136cba8ad..da95515d8e7 100644
--- a/include/grpc/module.modulemap
+++ b/include/grpc/module.modulemap
@@ -30,7 +30,6 @@ framework module grpc {
   header "impl/codegen/byte_buffer_reader.h"
   header "impl/codegen/compression_types.h"
   header "impl/codegen/connectivity_state.h"
-  header "impl/codegen/exec_ctx_fwd.h"
   header "impl/codegen/grpc_types.h"
   header "impl/codegen/propagation_bits.h"
   header "impl/codegen/slice.h"
diff --git a/package.xml b/package.xml
index b4d8c886930..adc98330de9 100644
--- a/package.xml
+++ b/package.xml
@@ -161,7 +161,6 @@
     <file baseinstalldir="/" name="include/grpc/impl/codegen/byte_buffer_reader.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/compression_types.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/connectivity_state.h" role="src" />
-    <file baseinstalldir="/" name="include/grpc/impl/codegen/exec_ctx_fwd.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/grpc_types.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/propagation_bits.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/slice.h" role="src" />
diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h
index 88af76006aa..4c58c0e4bf3 100644
--- a/src/core/lib/iomgr/closure.h
+++ b/src/core/lib/iomgr/closure.h
@@ -22,7 +22,6 @@
 #include <grpc/support/port_platform.h>
 
 #include <assert.h>
-#include <grpc/impl/codegen/exec_ctx_fwd.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <stdbool.h>
diff --git a/src/core/lib/iomgr/iomgr.h b/src/core/lib/iomgr/iomgr.h
index 3f238c660a7..c7cde7ea59c 100644
--- a/src/core/lib/iomgr/iomgr.h
+++ b/src/core/lib/iomgr/iomgr.h
@@ -19,7 +19,6 @@
 #ifndef GRPC_CORE_LIB_IOMGR_IOMGR_H
 #define GRPC_CORE_LIB_IOMGR_IOMGR_H
 
-#include <grpc/impl/codegen/exec_ctx_fwd.h>
 #include "src/core/lib/iomgr/port.h"
 
 /** Initializes the iomgr. */
diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c
index 8d2384ba61f..7fd36a241a1 100644
--- a/test/core/surface/public_headers_must_be_c89.c
+++ b/test/core/surface/public_headers_must_be_c89.c
@@ -29,7 +29,6 @@
 #include <grpc/impl/codegen/byte_buffer_reader.h>
 #include <grpc/impl/codegen/compression_types.h>
 #include <grpc/impl/codegen/connectivity_state.h>
-#include <grpc/impl/codegen/exec_ctx_fwd.h>
 #include <grpc/impl/codegen/fork.h>
 #include <grpc/impl/codegen/gpr_slice.h>
 #include <grpc/impl/codegen/gpr_types.h>
diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++
index e62278cb9f6..5bdbcd71f57 100644
--- a/tools/doxygen/Doxyfile.c++
+++ b/tools/doxygen/Doxyfile.c++
@@ -887,7 +887,6 @@ include/grpc/impl/codegen/byte_buffer.h \
 include/grpc/impl/codegen/byte_buffer_reader.h \
 include/grpc/impl/codegen/compression_types.h \
 include/grpc/impl/codegen/connectivity_state.h \
-include/grpc/impl/codegen/exec_ctx_fwd.h \
 include/grpc/impl/codegen/fork.h \
 include/grpc/impl/codegen/gpr_slice.h \
 include/grpc/impl/codegen/gpr_types.h \
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index d09b325c97a..b57674ba8e1 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -888,7 +888,6 @@ include/grpc/impl/codegen/byte_buffer.h \
 include/grpc/impl/codegen/byte_buffer_reader.h \
 include/grpc/impl/codegen/compression_types.h \
 include/grpc/impl/codegen/connectivity_state.h \
-include/grpc/impl/codegen/exec_ctx_fwd.h \
 include/grpc/impl/codegen/fork.h \
 include/grpc/impl/codegen/gpr_slice.h \
 include/grpc/impl/codegen/gpr_types.h \
diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core
index 6ce90417475..916d3b1e49a 100644
--- a/tools/doxygen/Doxyfile.core
+++ b/tools/doxygen/Doxyfile.core
@@ -818,7 +818,6 @@ include/grpc/impl/codegen/byte_buffer.h \
 include/grpc/impl/codegen/byte_buffer_reader.h \
 include/grpc/impl/codegen/compression_types.h \
 include/grpc/impl/codegen/connectivity_state.h \
-include/grpc/impl/codegen/exec_ctx_fwd.h \
 include/grpc/impl/codegen/fork.h \
 include/grpc/impl/codegen/fork.h \
 include/grpc/impl/codegen/gpr_slice.h \
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 1aff0075a6a..e41123abf68 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -818,7 +818,6 @@ include/grpc/impl/codegen/byte_buffer.h \
 include/grpc/impl/codegen/byte_buffer_reader.h \
 include/grpc/impl/codegen/compression_types.h \
 include/grpc/impl/codegen/connectivity_state.h \
-include/grpc/impl/codegen/exec_ctx_fwd.h \
 include/grpc/impl/codegen/fork.h \
 include/grpc/impl/codegen/fork.h \
 include/grpc/impl/codegen/gpr_slice.h \
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index d432bd0e537..f7898a90a93 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -8505,7 +8505,6 @@
       "include/grpc/impl/codegen/byte_buffer_reader.h", 
       "include/grpc/impl/codegen/compression_types.h", 
       "include/grpc/impl/codegen/connectivity_state.h", 
-      "include/grpc/impl/codegen/exec_ctx_fwd.h", 
       "include/grpc/impl/codegen/grpc_types.h", 
       "include/grpc/impl/codegen/propagation_bits.h", 
       "include/grpc/impl/codegen/slice.h", 
@@ -8519,7 +8518,6 @@
       "include/grpc/impl/codegen/byte_buffer_reader.h", 
       "include/grpc/impl/codegen/compression_types.h", 
       "include/grpc/impl/codegen/connectivity_state.h", 
-      "include/grpc/impl/codegen/exec_ctx_fwd.h", 
       "include/grpc/impl/codegen/grpc_types.h", 
       "include/grpc/impl/codegen/propagation_bits.h", 
       "include/grpc/impl/codegen/slice.h", 

From 157815d5cd5fc257d568bcfa87baa18325608000 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Wed, 10 Jan 2018 17:58:46 -0800
Subject: [PATCH 071/161] Reviewer feedback

---
 .../chttp2/transport/chttp2_plugin.cc         | 10 +++++++++-
 .../chttp2/transport/chttp2_transport.cc      | 19 +++++--------------
 .../chttp2/transport/chttp2_transport.h       |  2 ++
 3 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc b/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc
index 97c1878f340..3aca61fdac7 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc
@@ -18,8 +18,16 @@
 
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/support/env.h"
 #include "src/core/lib/transport/metadata.h"
 
-void grpc_chttp2_plugin_init(void) {}
+void grpc_chttp2_plugin_init(void) {
+  g_flow_control_enabled = true;
+  char* env_variable = gpr_getenv("GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL");
+  if (env_variable != nullptr) {
+    g_flow_control_enabled = false;
+    gpr_free(env_variable);
+  }
+}
 
 void grpc_chttp2_plugin_shutdown(void) {}
diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 97f79d9bdf3..835de6aa0f4 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -152,6 +152,10 @@ static void keepalive_watchdog_fired_locked(void* arg, grpc_error* error);
 
 static void reset_byte_stream(void* arg, grpc_error* error);
 
+// Flow control default enabled. Can be disabled by setting
+// GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL
+bool g_flow_control_enabled = true;
+
 /*******************************************************************************
  * CONSTRUCTION/DESTRUCTION/REFCOUNTING
  */
@@ -232,9 +236,6 @@ void grpc_chttp2_ref_transport(grpc_chttp2_transport* t) { gpr_ref(&t->refs); }
 
 static const grpc_transport_vtable* get_vtable(void);
 
-// -1 == unset, 0 == disabled, 1 == enabled
-static gpr_atm flow_control_enabled = -1;
-
 static void init_transport(grpc_chttp2_transport* t,
                            const grpc_channel_args* channel_args,
                            grpc_endpoint* ep, bool is_client) {
@@ -520,17 +521,7 @@ static void init_transport(grpc_chttp2_transport* t,
     }
   }
 
-  if (gpr_atm_no_barrier_load(&flow_control_enabled) == -1) {
-    char* env_variable = gpr_getenv("GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL");
-    if (env_variable != nullptr) {
-      gpr_atm_no_barrier_store(&flow_control_enabled, 0);
-    } else {
-      gpr_atm_no_barrier_store(&flow_control_enabled, 1);
-    }
-    gpr_free(env_variable);
-  }
-
-  if (gpr_atm_no_barrier_load(&flow_control_enabled)) {
+  if (g_flow_control_enabled) {
     t->flow_control.Init<grpc_core::chttp2::TransportFlowControl>(t,
                                                                   enable_bdp);
   } else {
diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h
index 596ababb194..34519ceec9c 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.h
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h
@@ -27,6 +27,8 @@ extern grpc_core::TraceFlag grpc_http_trace;
 extern grpc_core::TraceFlag grpc_trace_http2_stream_state;
 extern grpc_core::DebugOnlyTraceFlag grpc_trace_chttp2_refcount;
 
+extern bool g_flow_control_enabled;
+
 grpc_transport* grpc_create_chttp2_transport(
     const grpc_channel_args* channel_args, grpc_endpoint* ep, bool is_client);
 

From 52338fb9eeec74da8bf10078de2f2b4b635412f2 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Thu, 11 Jan 2018 00:25:00 -0800
Subject: [PATCH 072/161] Avoid linker warnings by removing extraneous
 directories from -L and making at right time

---
 Makefile                    | 4 +---
 templates/Makefile.template | 4 +---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 10c65303139..58818f33b32 100644
--- a/Makefile
+++ b/Makefile
@@ -643,7 +643,6 @@ ZLIB_DEP = $(LIBDIR)/$(CONFIG)/libz.a
 ZLIB_MERGE_LIBS = $(LIBDIR)/$(CONFIG)/libz.a
 ZLIB_MERGE_OBJS = $(LIBZ_OBJS)
 CPPFLAGS += -Ithird_party/zlib
-LDFLAGS += -L$(LIBDIR)/$(CONFIG)/zlib
 else
 ifeq ($(HAS_PKG_CONFIG),true)
 CPPFLAGS += $(shell $(PKG_CONFIG) --cflags zlib)
@@ -674,7 +673,6 @@ CARES_DEP = $(LIBDIR)/$(CONFIG)/libares.a
 CARES_MERGE_OBJS = $(LIBARES_OBJS)
 CARES_MERGE_LIBS = $(LIBDIR)/$(CONFIG)/libares.a
 CPPFLAGS := -Ithird_party/cares -Ithird_party/cares/cares $(CPPFLAGS)
-LDFLAGS := -L$(LIBDIR)/$(CONFIG)/c-ares $(LDFLAGS)
 else
 ifeq ($(HAS_PKG_CONFIG),true)
 PC_REQUIRES_GRPC += libcares
@@ -1301,10 +1299,10 @@ third_party/protobuf/configure:
 
 $(LIBDIR)/$(CONFIG)/protobuf/libprotobuf.a: third_party/protobuf/configure
 	$(E) "[MAKE]    Building protobuf"
+	$(Q)mkdir -p $(LIBDIR)/$(CONFIG)/protobuf
 	$(Q)(cd third_party/protobuf ; CC="$(CC)" CXX="$(CXX)" LDFLAGS="$(LDFLAGS_$(CONFIG)) -g $(PROTOBUF_LDFLAGS_EXTRA)" CPPFLAGS="$(PIC_CPPFLAGS) $(CPPFLAGS_$(CONFIG)) -g $(PROTOBUF_CPPFLAGS_EXTRA)" ./configure --disable-shared --enable-static $(PROTOBUF_CONFIG_OPTS))
 	$(Q)$(MAKE) -C third_party/protobuf clean
 	$(Q)$(MAKE) -C third_party/protobuf
-	$(Q)mkdir -p $(LIBDIR)/$(CONFIG)/protobuf
 	$(Q)mkdir -p $(BINDIR)/$(CONFIG)/protobuf
 	$(Q)cp third_party/protobuf/src/.libs/libprotoc.a $(LIBDIR)/$(CONFIG)/protobuf
 	$(Q)cp third_party/protobuf/src/.libs/libprotobuf.a $(LIBDIR)/$(CONFIG)/protobuf
diff --git a/templates/Makefile.template b/templates/Makefile.template
index 954bea7e17a..b8e26b646b0 100644
--- a/templates/Makefile.template
+++ b/templates/Makefile.template
@@ -563,7 +563,6 @@
   ZLIB_MERGE_LIBS = $(LIBDIR)/$(CONFIG)/libz.a
   ZLIB_MERGE_OBJS = $(LIBZ_OBJS)
   CPPFLAGS += -Ithird_party/zlib
-  LDFLAGS += -L$(LIBDIR)/$(CONFIG)/zlib
   else
   ifeq ($(HAS_PKG_CONFIG),true)
   CPPFLAGS += $(shell $(PKG_CONFIG) --cflags zlib)
@@ -594,7 +593,6 @@
   CARES_MERGE_OBJS = $(LIBARES_OBJS)
   CARES_MERGE_LIBS = $(LIBDIR)/$(CONFIG)/libares.a
   CPPFLAGS := -Ithird_party/cares -Ithird_party/cares/cares $(CPPFLAGS)
-  LDFLAGS := -L$(LIBDIR)/$(CONFIG)/c-ares $(LDFLAGS)
   else
   ifeq ($(HAS_PKG_CONFIG),true)
   PC_REQUIRES_GRPC += libcares
@@ -896,10 +894,10 @@
 
   $(LIBDIR)/$(CONFIG)/protobuf/libprotobuf.a: third_party/protobuf/configure
   	$(E) "[MAKE]    Building protobuf"
+  	$(Q)mkdir -p $(LIBDIR)/$(CONFIG)/protobuf
   	$(Q)(cd third_party/protobuf ; CC="$(CC)" CXX="$(CXX)" LDFLAGS="$(LDFLAGS_$(CONFIG)) -g $(PROTOBUF_LDFLAGS_EXTRA)" CPPFLAGS="$(PIC_CPPFLAGS) $(CPPFLAGS_$(CONFIG)) -g $(PROTOBUF_CPPFLAGS_EXTRA)" ./configure --disable-shared --enable-static $(PROTOBUF_CONFIG_OPTS))
   	$(Q)$(MAKE) -C third_party/protobuf clean
   	$(Q)$(MAKE) -C third_party/protobuf
-  	$(Q)mkdir -p $(LIBDIR)/$(CONFIG)/protobuf
   	$(Q)mkdir -p $(BINDIR)/$(CONFIG)/protobuf
   	$(Q)cp third_party/protobuf/src/.libs/libprotoc.a $(LIBDIR)/$(CONFIG)/protobuf
   	$(Q)cp third_party/protobuf/src/.libs/libprotobuf.a $(LIBDIR)/$(CONFIG)/protobuf

From 324703db51b43e150d9d8ffbcceb9d2096e26a9f Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Thu, 11 Jan 2018 07:41:31 -0800
Subject: [PATCH 073/161] Fix existing ref counting classes and add new ones.

---
 BUILD                                         |  10 ++
 CMakeLists.txt                                |  40 +++++
 Makefile                                      |  48 +++++
 build.yaml                                    |  15 ++
 gRPC-Core.podspec                             |   2 +
 grpc.gemspec                                  |   1 +
 package.xml                                   |   1 +
 src/core/lib/support/abstract.h               |   5 +
 src/core/lib/support/orphanable.h             | 166 ++++++++++++++++++
 src/core/lib/support/ref_counted.h            |  13 +-
 test/core/support/BUILD                       |  13 ++
 test/core/support/orphanable_test.cc          | 114 ++++++++++++
 test/core/support/ref_counted_test.cc         |   4 +-
 tools/doxygen/Doxyfile.c++.internal           |   1 +
 tools/doxygen/Doxyfile.core.internal          |   1 +
 .../generated/sources_and_headers.json        |  21 +++
 tools/run_tests/generated/tests.json          |  24 +++
 17 files changed, 477 insertions(+), 2 deletions(-)
 create mode 100644 src/core/lib/support/orphanable.h
 create mode 100644 test/core/support/orphanable_test.cc

diff --git a/BUILD b/BUILD
index 804c6cee02f..f478652df2b 100644
--- a/BUILD
+++ b/BUILD
@@ -548,6 +548,16 @@ grpc_cc_library(
     language = "c++",
 )
 
+grpc_cc_library(
+    name = "orphanable",
+    public_hdrs = ["src/core/lib/support/orphanable.h"],
+    language = "c++",
+    deps = [
+        "grpc_trace",
+        "debug_location",
+    ],
+)
+
 grpc_cc_library(
     name = "ref_counted",
     public_hdrs = ["src/core/lib/support/ref_counted.h"],
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 78ccfb21329..863c192a87c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -558,6 +558,7 @@ add_dependencies(buildtests_cxx memory_test)
 add_dependencies(buildtests_cxx metrics_client)
 add_dependencies(buildtests_cxx mock_test)
 add_dependencies(buildtests_cxx noop-benchmark)
+add_dependencies(buildtests_cxx orphanable_test)
 add_dependencies(buildtests_cxx proto_server_reflection_test)
 add_dependencies(buildtests_cxx proto_utils_test)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
@@ -11328,6 +11329,45 @@ target_link_libraries(noop-benchmark
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
+add_executable(orphanable_test
+  test/core/support/orphanable_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+
+target_include_directories(orphanable_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${BENCHMARK_ROOT_DIR}/include
+  PRIVATE ${ZLIB_ROOT_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CARES_INCLUDE_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(orphanable_test
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc_test_util
+  grpc++
+  grpc
+  gpr_test_util
+  gpr
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
 add_executable(proto_server_reflection_test
   test/cpp/end2end/proto_server_reflection_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
diff --git a/Makefile b/Makefile
index f50163efdce..9e60130f674 100644
--- a/Makefile
+++ b/Makefile
@@ -1154,6 +1154,7 @@ memory_test: $(BINDIR)/$(CONFIG)/memory_test
 metrics_client: $(BINDIR)/$(CONFIG)/metrics_client
 mock_test: $(BINDIR)/$(CONFIG)/mock_test
 noop-benchmark: $(BINDIR)/$(CONFIG)/noop-benchmark
+orphanable_test: $(BINDIR)/$(CONFIG)/orphanable_test
 proto_server_reflection_test: $(BINDIR)/$(CONFIG)/proto_server_reflection_test
 proto_utils_test: $(BINDIR)/$(CONFIG)/proto_utils_test
 qps_interarrival_test: $(BINDIR)/$(CONFIG)/qps_interarrival_test
@@ -1595,6 +1596,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/metrics_client \
   $(BINDIR)/$(CONFIG)/mock_test \
   $(BINDIR)/$(CONFIG)/noop-benchmark \
+  $(BINDIR)/$(CONFIG)/orphanable_test \
   $(BINDIR)/$(CONFIG)/proto_server_reflection_test \
   $(BINDIR)/$(CONFIG)/proto_utils_test \
   $(BINDIR)/$(CONFIG)/qps_interarrival_test \
@@ -1725,6 +1727,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/metrics_client \
   $(BINDIR)/$(CONFIG)/mock_test \
   $(BINDIR)/$(CONFIG)/noop-benchmark \
+  $(BINDIR)/$(CONFIG)/orphanable_test \
   $(BINDIR)/$(CONFIG)/proto_server_reflection_test \
   $(BINDIR)/$(CONFIG)/proto_utils_test \
   $(BINDIR)/$(CONFIG)/qps_interarrival_test \
@@ -2132,6 +2135,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/mock_test || ( echo test mock_test failed ; exit 1 )
 	$(E) "[RUN]     Testing noop-benchmark"
 	$(Q) $(BINDIR)/$(CONFIG)/noop-benchmark || ( echo test noop-benchmark failed ; exit 1 )
+	$(E) "[RUN]     Testing orphanable_test"
+	$(Q) $(BINDIR)/$(CONFIG)/orphanable_test || ( echo test orphanable_test failed ; exit 1 )
 	$(E) "[RUN]     Testing proto_server_reflection_test"
 	$(Q) $(BINDIR)/$(CONFIG)/proto_server_reflection_test || ( echo test proto_server_reflection_test failed ; exit 1 )
 	$(E) "[RUN]     Testing proto_utils_test"
@@ -16088,6 +16093,49 @@ endif
 endif
 
 
+ORPHANABLE_TEST_SRC = \
+    test/core/support/orphanable_test.cc \
+
+ORPHANABLE_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(ORPHANABLE_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/orphanable_test: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
+
+$(BINDIR)/$(CONFIG)/orphanable_test: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/orphanable_test: $(PROTOBUF_DEP) $(ORPHANABLE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(ORPHANABLE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/orphanable_test
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/support/orphanable_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_orphanable_test: $(ORPHANABLE_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(ORPHANABLE_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 PROTO_SERVER_REFLECTION_TEST_SRC = \
     test/cpp/end2end/proto_server_reflection_test.cc \
 
diff --git a/build.yaml b/build.yaml
index db2ff8828b2..8a34ade9598 100644
--- a/build.yaml
+++ b/build.yaml
@@ -397,6 +397,7 @@ filegroups:
   - src/core/lib/slice/slice_internal.h
   - src/core/lib/slice/slice_string_helpers.h
   - src/core/lib/support/debug_location.h
+  - src/core/lib/support/orphanable.h
   - src/core/lib/support/ref_counted.h
   - src/core/lib/support/ref_counted_ptr.h
   - src/core/lib/support/vector.h
@@ -4390,6 +4391,20 @@ targets:
   deps:
   - benchmark
   defaults: benchmark
+- name: orphanable_test
+  gtest: true
+  build: test
+  language: c++
+  src:
+  - test/core/support/orphanable_test.cc
+  deps:
+  - grpc_test_util
+  - grpc++
+  - grpc
+  - gpr_test_util
+  - gpr
+  uses:
+  - grpc++_test
 - name: proto_server_reflection_test
   gtest: true
   build: test
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index 358fad3d983..d064ac80ae6 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -420,6 +420,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/slice/slice_internal.h',
                       'src/core/lib/slice/slice_string_helpers.h',
                       'src/core/lib/support/debug_location.h',
+                      'src/core/lib/support/orphanable.h',
                       'src/core/lib/support/ref_counted.h',
                       'src/core/lib/support/ref_counted_ptr.h',
                       'src/core/lib/support/vector.h',
@@ -901,6 +902,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/slice/slice_internal.h',
                               'src/core/lib/slice/slice_string_helpers.h',
                               'src/core/lib/support/debug_location.h',
+                              'src/core/lib/support/orphanable.h',
                               'src/core/lib/support/ref_counted.h',
                               'src/core/lib/support/ref_counted_ptr.h',
                               'src/core/lib/support/vector.h',
diff --git a/grpc.gemspec b/grpc.gemspec
index 7547bc85ded..f8afaa5803c 100644
--- a/grpc.gemspec
+++ b/grpc.gemspec
@@ -346,6 +346,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/slice/slice_internal.h )
   s.files += %w( src/core/lib/slice/slice_string_helpers.h )
   s.files += %w( src/core/lib/support/debug_location.h )
+  s.files += %w( src/core/lib/support/orphanable.h )
   s.files += %w( src/core/lib/support/ref_counted.h )
   s.files += %w( src/core/lib/support/ref_counted_ptr.h )
   s.files += %w( src/core/lib/support/vector.h )
diff --git a/package.xml b/package.xml
index ff3d0797abe..8c590348b6c 100644
--- a/package.xml
+++ b/package.xml
@@ -358,6 +358,7 @@
     <file baseinstalldir="/" name="src/core/lib/slice/slice_internal.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/slice/slice_string_helpers.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/debug_location.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/support/orphanable.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/ref_counted.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/ref_counted_ptr.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/vector.h" role="src" />
diff --git a/src/core/lib/support/abstract.h b/src/core/lib/support/abstract.h
index 5498769a7d9..1dffa301282 100644
--- a/src/core/lib/support/abstract.h
+++ b/src/core/lib/support/abstract.h
@@ -26,4 +26,9 @@
 #define GRPC_ABSTRACT_BASE_CLASS \
   static void operator delete(void* p) { abort(); }
 
+// gRPC currently can't depend on libstdc++, so we can't use "= 0" for
+// pure virtual methods.  Instead, we use this macro.
+#define GRPC_ABSTRACT \
+  { GPR_ASSERT(false); }
+
 #endif /* GRPC_CORE_LIB_SUPPORT_ABSTRACT_H */
diff --git a/src/core/lib/support/orphanable.h b/src/core/lib/support/orphanable.h
new file mode 100644
index 00000000000..63eda2e08b0
--- /dev/null
+++ b/src/core/lib/support/orphanable.h
@@ -0,0 +1,166 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SUPPORT_ORPHANABLE_H
+#define GRPC_CORE_LIB_SUPPORT_ORPHANABLE_H
+
+#include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+
+#include <memory>
+
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/support/abstract.h"
+#include "src/core/lib/support/debug_location.h"
+#include "src/core/lib/support/memory.h"
+
+namespace grpc_core {
+
+// A base class for orphanable objects.
+class Orphanable {
+ public:
+  // Gives up ownership of the object.  The implementation must arrange
+  // to destroy the object without further interaction from the caller.
+  virtual void Orphan() GRPC_ABSTRACT;
+
+  // Not copyable or movable.
+  Orphanable(const Orphanable&) = delete;
+  Orphanable& operator=(const Orphanable&) = delete;
+
+  GRPC_ABSTRACT_BASE_CLASS
+
+ protected:
+  Orphanable() {}
+  virtual ~Orphanable() {}
+};
+
+template <typename T>
+class OrphanableDelete {
+ public:
+  void operator()(T* p) { p->Orphan(); }
+};
+
+template <typename T, typename Deleter = OrphanableDelete<T>>
+using OrphanablePtr = std::unique_ptr<T, Deleter>;
+
+template <typename T, typename... Args>
+inline OrphanablePtr<T> MakeOrphanable(Args&&... args) {
+  return OrphanablePtr<T>(New<T>(std::forward<Args>(args)...));
+}
+
+// A type of Orphanable with internal ref-counting.
+class InternallyRefCounted : public Orphanable {
+ public:
+  // Not copyable nor movable.
+  InternallyRefCounted(const InternallyRefCounted&) = delete;
+  InternallyRefCounted& operator=(const InternallyRefCounted&) = delete;
+
+  GRPC_ABSTRACT_BASE_CLASS
+
+ protected:
+  InternallyRefCounted() { gpr_ref_init(&refs_, 1); }
+  virtual ~InternallyRefCounted() {}
+
+  void Ref() { gpr_ref(&refs_); }
+
+  void Unref() {
+    if (gpr_unref(&refs_)) {
+      Delete(this);
+    }
+  }
+
+  // Allow Delete() to access destructor.
+  template <typename T>
+  friend void Delete(T*);
+
+ private:
+  gpr_refcount refs_;
+};
+
+// An alternative version of the InternallyRefCounted base class that
+// supports tracing.  This is intended to be used in cases where the
+// object will be handled both by idiomatic C++ code using smart
+// pointers and legacy code that is manually calling Ref() and Unref().
+// Once all of our code is converted to idiomatic C++, we may be able to
+// eliminate this class.
+class InternallyRefCountedWithTracing : public Orphanable {
+ public:
+  // Not copyable nor movable.
+  InternallyRefCountedWithTracing(const InternallyRefCountedWithTracing&) =
+      delete;
+  InternallyRefCountedWithTracing& operator=(
+      const InternallyRefCountedWithTracing&) = delete;
+
+  GRPC_ABSTRACT_BASE_CLASS
+
+ protected:
+  // Allow Delete() to access destructor.
+  template <typename T>
+  friend void Delete(T*);
+
+  InternallyRefCountedWithTracing()
+      : InternallyRefCountedWithTracing(static_cast<TraceFlag*>(nullptr)) {}
+
+  explicit InternallyRefCountedWithTracing(TraceFlag* trace_flag)
+      : trace_flag_(trace_flag) {
+    gpr_ref_init(&refs_, 1);
+  }
+
+#ifdef NDEBUG
+  explicit InternallyRefCountedWithTracing(DebugOnlyTraceFlag* trace_flag)
+      : InternallyRefCountedWithTracing() {}
+#endif
+
+  virtual ~InternallyRefCountedWithTracing() {}
+
+  void Ref() { gpr_ref(&refs_); }
+
+  void Ref(const DebugLocation& location, const char* reason) {
+    if (location.Log() && trace_flag_ != nullptr && trace_flag_->enabled()) {
+      gpr_atm old_refs = gpr_atm_no_barrier_load(&refs_.count);
+      gpr_log(GPR_DEBUG, "%s:%p %s:%d ref %" PRIdPTR " -> %" PRIdPTR " %s",
+              trace_flag_->name(), this, location.file(), location.line(),
+              old_refs, old_refs + 1, reason);
+    }
+    Ref();
+  }
+
+  void Unref() {
+    if (gpr_unref(&refs_)) {
+      Delete(this);
+    }
+  }
+
+  void Unref(const DebugLocation& location, const char* reason) {
+    if (location.Log() && trace_flag_ != nullptr && trace_flag_->enabled()) {
+      gpr_atm old_refs = gpr_atm_no_barrier_load(&refs_.count);
+      gpr_log(GPR_DEBUG, "%s:%p %s:%d unref %" PRIdPTR " -> %" PRIdPTR " %s",
+              trace_flag_->name(), this, location.file(), location.line(),
+              old_refs, old_refs - 1, reason);
+    }
+    Unref();
+  }
+
+ private:
+  TraceFlag* trace_flag_ = nullptr;
+  gpr_refcount refs_;
+};
+
+}  // namespace grpc_core
+
+#endif /* GRPC_CORE_LIB_SUPPORT_ORPHANABLE_H */
diff --git a/src/core/lib/support/ref_counted.h b/src/core/lib/support/ref_counted.h
index 4c662f91190..48c11f7bbfe 100644
--- a/src/core/lib/support/ref_counted.h
+++ b/src/core/lib/support/ref_counted.h
@@ -23,6 +23,7 @@
 #include <grpc/support/sync.h>
 
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/support/abstract.h"
 #include "src/core/lib/support/debug_location.h"
 #include "src/core/lib/support/memory.h"
 
@@ -45,6 +46,8 @@ class RefCounted {
   RefCounted(const RefCounted&) = delete;
   RefCounted& operator=(const RefCounted&) = delete;
 
+  GRPC_ABSTRACT_BASE_CLASS
+
  protected:
   // Allow Delete() to access destructor.
   template <typename T>
@@ -98,18 +101,26 @@ class RefCountedWithTracing {
   RefCountedWithTracing(const RefCountedWithTracing&) = delete;
   RefCountedWithTracing& operator=(const RefCountedWithTracing&) = delete;
 
+  GRPC_ABSTRACT_BASE_CLASS
+
  protected:
   // Allow Delete() to access destructor.
   template <typename T>
   friend void Delete(T*);
 
-  RefCountedWithTracing() : RefCountedWithTracing(nullptr) {}
+  RefCountedWithTracing()
+      : RefCountedWithTracing(static_cast<TraceFlag*>(nullptr)) {}
 
   explicit RefCountedWithTracing(TraceFlag* trace_flag)
       : trace_flag_(trace_flag) {
     gpr_ref_init(&refs_, 1);
   }
 
+#ifdef NDEBUG
+  explicit RefCountedWithTracing(DebugOnlyTraceFlag* trace_flag)
+      : RefCountedWithTracing() {}
+#endif
+
   virtual ~RefCountedWithTracing() {}
 
  private:
diff --git a/test/core/support/BUILD b/test/core/support/BUILD
index 4372b49b545..c8fa046da1d 100644
--- a/test/core/support/BUILD
+++ b/test/core/support/BUILD
@@ -214,6 +214,19 @@ grpc_cc_test(
     ],
 )
 
+grpc_cc_test(
+    name = "orphanable_test",
+    srcs = ["orphanable_test.cc"],
+    language = "C++",
+    deps = [
+        "//:orphanable",
+        "//test/core/util:gpr_test_util",
+    ],
+    external_deps = [
+        "gtest",
+    ],
+)
+
 grpc_cc_test(
     name = "ref_counted_test",
     srcs = ["ref_counted_test.cc"],
diff --git a/test/core/support/orphanable_test.cc b/test/core/support/orphanable_test.cc
new file mode 100644
index 00000000000..e07017ab1ed
--- /dev/null
+++ b/test/core/support/orphanable_test.cc
@@ -0,0 +1,114 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/support/orphanable.h"
+
+#include <gtest/gtest.h>
+
+#include "src/core/lib/support/memory.h"
+#include "test/core/util/test_config.h"
+
+namespace grpc_core {
+namespace testing {
+namespace {
+
+class Foo : public Orphanable {
+ public:
+  Foo() : Foo(0) {}
+  explicit Foo(int value) : value_(value) {}
+  void Orphan() override { Delete(this); }
+  int value() const { return value_; }
+
+ private:
+  int value_;
+};
+
+TEST(Orphanable, Basic) {
+  Foo* foo = New<Foo>();
+  foo->Orphan();
+}
+
+TEST(OrphanablePtr, Basic) {
+  OrphanablePtr<Foo> foo(New<Foo>());
+  EXPECT_EQ(0, foo->value());
+}
+
+TEST(MakeOrphanable, DefaultConstructor) {
+  auto foo = MakeOrphanable<Foo>();
+  EXPECT_EQ(0, foo->value());
+}
+
+TEST(MakeOrphanable, WithParameters) {
+  auto foo = MakeOrphanable<Foo>(5);
+  EXPECT_EQ(5, foo->value());
+}
+
+class Bar : public InternallyRefCounted {
+ public:
+  Bar() : Bar(0) {}
+  explicit Bar(int value) : value_(value) {}
+  void Orphan() override { Unref(); }
+  int value() const { return value_; }
+
+  void StartWork() { Ref(); }
+  void FinishWork() { Unref(); }
+
+ private:
+  int value_;
+};
+
+TEST(OrphanablePtr, InternallyRefCounted) {
+  auto bar = MakeOrphanable<Bar>();
+  bar->StartWork();
+  bar->FinishWork();
+}
+
+// Note: We use DebugOnlyTraceFlag instead of TraceFlag to ensure that
+// things build properly in both debug and non-debug cases.
+DebugOnlyTraceFlag baz_tracer(true, "baz");
+
+class Baz : public InternallyRefCountedWithTracing {
+ public:
+  Baz() : Baz(0) {}
+  explicit Baz(int value)
+      : InternallyRefCountedWithTracing(&baz_tracer), value_(value) {}
+  void Orphan() override { Unref(); }
+  int value() const { return value_; }
+
+  void StartWork() { Ref(DEBUG_LOCATION, "work"); }
+  void FinishWork() { Unref(DEBUG_LOCATION, "work"); }
+
+ private:
+  int value_;
+};
+
+TEST(OrphanablePtr, InternallyRefCountedWithTracing) {
+  auto baz = MakeOrphanable<Baz>();
+  baz->StartWork();
+  baz->FinishWork();
+}
+
+}  // namespace
+}  // namespace testing
+}  // namespace grpc_core
+
+int main(int argc, char** argv) {
+  grpc_test_init(argc, argv);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/test/core/support/ref_counted_test.cc b/test/core/support/ref_counted_test.cc
index be9b6ff7c2d..0629e3ff5fd 100644
--- a/test/core/support/ref_counted_test.cc
+++ b/test/core/support/ref_counted_test.cc
@@ -44,7 +44,9 @@ TEST(RefCounted, ExtraRef) {
   foo->Unref();
 }
 
-TraceFlag foo_tracer(true, "foo");
+// Note: We use DebugOnlyTraceFlag instead of TraceFlag to ensure that
+// things build properly in both debug and non-debug cases.
+DebugOnlyTraceFlag foo_tracer(true, "foo");
 
 class FooWithTracing : public RefCountedWithTracing {
  public:
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index d9184f49a22..85bbeed088e 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -1038,6 +1038,7 @@ src/core/lib/support/manual_constructor.h \
 src/core/lib/support/memory.h \
 src/core/lib/support/mpscq.h \
 src/core/lib/support/murmur_hash.h \
+src/core/lib/support/orphanable.h \
 src/core/lib/support/ref_counted.h \
 src/core/lib/support/ref_counted_ptr.h \
 src/core/lib/support/spinlock.h \
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 3d3c6711d0c..4bf0fc74d1d 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -1304,6 +1304,7 @@ src/core/lib/support/mpscq.cc \
 src/core/lib/support/mpscq.h \
 src/core/lib/support/murmur_hash.cc \
 src/core/lib/support/murmur_hash.h \
+src/core/lib/support/orphanable.h \
 src/core/lib/support/ref_counted.h \
 src/core/lib/support/ref_counted_ptr.h \
 src/core/lib/support/spinlock.h \
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 51f0ac7ca61..a20acb1feff 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -3688,6 +3688,25 @@
     "third_party": false, 
     "type": "target"
   }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc++", 
+      "grpc++_test", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c++", 
+    "name": "orphanable_test", 
+    "src": [
+      "test/core/support/orphanable_test.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
   {
     "deps": [
       "gpr", 
@@ -8267,6 +8286,7 @@
       "src/core/lib/slice/slice_internal.h", 
       "src/core/lib/slice/slice_string_helpers.h", 
       "src/core/lib/support/debug_location.h", 
+      "src/core/lib/support/orphanable.h", 
       "src/core/lib/support/ref_counted.h", 
       "src/core/lib/support/ref_counted_ptr.h", 
       "src/core/lib/support/vector.h", 
@@ -8407,6 +8427,7 @@
       "src/core/lib/slice/slice_internal.h", 
       "src/core/lib/slice/slice_string_helpers.h", 
       "src/core/lib/support/debug_location.h", 
+      "src/core/lib/support/orphanable.h", 
       "src/core/lib/support/ref_counted.h", 
       "src/core/lib/support/ref_counted_ptr.h", 
       "src/core/lib/support/vector.h", 
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index 6b83cecd41d..57b934d9c66 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -4053,6 +4053,30 @@
     ], 
     "uses_polling": true
   }, 
+  {
+    "args": [], 
+    "benchmark": false, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": true, 
+    "language": "c++", 
+    "name": "orphanable_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "uses_polling": true
+  }, 
   {
     "args": [], 
     "benchmark": false, 

From 262d59e07c6243bce1357091472e90060e9aa78d Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Thu, 11 Jan 2018 07:47:52 -0800
Subject: [PATCH 074/161] Update CONTRIBUTING.md

Use 2 white spaces after period for consistency with rest of document
---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c2dc4eba57b..2f90ccf2814 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -73,7 +73,7 @@ How to get your contributions merged smoothly and quickly.
  
 - If you are regenerating the projects using
   `tools/buildgen/generate_projects.sh`, make changes to generated files a
-  separate commit with commit message `regenerate projects`. Mixing changes
+  separate commit with commit message `regenerate projects`.  Mixing changes
   to generated and hand-written files make your PR difficult to review.
   Note that running this script requires the installation of Python packages
   `pyyaml` and `mako` (typically installed using `pip`) as well as a recent

From a6efb9cec7a81b6d2a26d1c5466458b432ebfbd9 Mon Sep 17 00:00:00 2001
From: Dan Zhang <danzh@google.com>
Date: Thu, 11 Jan 2018 11:01:30 -0500
Subject: [PATCH 075/161] fix portability errors

---
 src/core/lib/iomgr/udp_server.cc | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/core/lib/iomgr/udp_server.cc b/src/core/lib/iomgr/udp_server.cc
index 8deb0ea544d..2df10ee09a9 100644
--- a/src/core/lib/iomgr/udp_server.cc
+++ b/src/core/lib/iomgr/udp_server.cc
@@ -21,6 +21,10 @@
 #define _GNU_SOURCE
 #endif
 
+#ifndef SO_RXQ_OVFL
+#define SO_RXQ_OVFL 40
+#endif
+
 #include "src/core/lib/iomgr/port.h"
 
 #ifdef GRPC_POSIX_SOCKET
@@ -325,13 +329,13 @@ static int prepare_socket(grpc_socket_factory* socket_factory, int fd,
   }
 
   if (grpc_set_socket_sndbuf(fd, snd_buf_size) != GRPC_ERROR_NONE) {
-    gpr_log(GPR_ERROR, "Failed to set send buffer size to %lu bytes",
+    gpr_log(GPR_ERROR, "Failed to set send buffer size to %zd bytes",
             snd_buf_size);
     goto error;
   }
 
   if (grpc_set_socket_rcvbuf(fd, rcv_buf_size) != GRPC_ERROR_NONE) {
-    gpr_log(GPR_ERROR, "Failed to set receive buffer size to %lu bytes",
+    gpr_log(GPR_ERROR, "Failed to set receive buffer size to %zd bytes",
             rcv_buf_size);
     goto error;
   }
@@ -457,8 +461,7 @@ static void on_write(void* arg, grpc_error* error) {
 
 static int add_socket_to_server(grpc_udp_server* s, int fd,
                                 const grpc_resolved_address* addr,
-                                size_t rcv_buf_size,
-                                size_t snd_buf_size,
+                                size_t rcv_buf_size, size_t snd_buf_size,
                                 grpc_udp_server_start_cb start_cb,
                                 grpc_udp_server_read_cb read_cb,
                                 grpc_udp_server_write_cb write_cb,

From 473267b7e83cbedfa8902d00de8e2f12dbe62a0d Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Thu, 11 Jan 2018 08:53:53 -0800
Subject: [PATCH 076/161] Revert "Merge pull request #13970 from
 grpc/revert-13857-lb_policy_ref_simplification"

This reverts commit 61b32965bec11f4106c729bb0a428ff03d2d03ab, reversing
changes made to 2eb22fd67d73a210c1f41d79efcfe52285ccb2ec.
---
 .../filters/client_channel/client_channel.cc  |  71 +-
 .../ext/filters/client_channel/lb_policy.cc   |  95 +--
 .../ext/filters/client_channel/lb_policy.h    |  90 ++-
 .../client_channel/lb_policy/grpclb/grpclb.cc | 607 +++++++-----------
 .../lb_policy/pick_first/pick_first.cc        |  91 ++-
 .../lb_policy/round_robin/round_robin.cc      | 124 ++--
 .../lb_policy/subchannel_list.cc              |   4 +-
 7 files changed, 415 insertions(+), 667 deletions(-)

diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc
index e99022a91b8..3f3334d44a3 100644
--- a/src/core/ext/filters/client_channel/client_channel.cc
+++ b/src/core/ext/filters/client_channel/client_channel.cc
@@ -553,6 +553,7 @@ static void on_resolver_result_changed_locked(void* arg, grpc_error* error) {
       }
       grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties,
                                        chand->interested_parties);
+      grpc_lb_policy_shutdown_locked(chand->lb_policy, new_lb_policy);
       GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel");
     }
     chand->lb_policy = new_lb_policy;
@@ -658,6 +659,7 @@ static void start_transport_op_locked(void* arg, grpc_error* error_ignored) {
       if (chand->lb_policy != nullptr) {
         grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties,
                                          chand->interested_parties);
+        grpc_lb_policy_shutdown_locked(chand->lb_policy, nullptr);
         GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel");
         chand->lb_policy = nullptr;
       }
@@ -792,6 +794,7 @@ static void cc_destroy_channel_elem(grpc_channel_element* elem) {
   if (chand->lb_policy != nullptr) {
     grpc_pollset_set_del_pollset_set(chand->lb_policy->interested_parties,
                                      chand->interested_parties);
+    grpc_lb_policy_shutdown_locked(chand->lb_policy, nullptr);
     GRPC_LB_POLICY_UNREF(chand->lb_policy, "channel");
   }
   gpr_free(chand->info_lb_policy_name);
@@ -852,12 +855,10 @@ typedef struct client_channel_call_data {
   grpc_subchannel_call* subchannel_call;
   grpc_error* error;
 
-  grpc_lb_policy* lb_policy;  // Holds ref while LB pick is pending.
+  grpc_lb_policy_pick_state pick;
   grpc_closure lb_pick_closure;
   grpc_closure lb_pick_cancel_closure;
 
-  grpc_connected_subchannel* connected_subchannel;
-  grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
   grpc_polling_entity* pollent;
 
   grpc_transport_stream_op_batch* waiting_for_pick_batches[MAX_WAITING_BATCHES];
@@ -866,8 +867,6 @@ typedef struct client_channel_call_data {
 
   grpc_transport_stream_op_batch* initial_metadata_batch;
 
-  grpc_linked_mdelem lb_token_mdelem;
-
   grpc_closure on_complete;
   grpc_closure* original_on_complete;
 } call_data;
@@ -1005,16 +1004,16 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
   channel_data* chand = (channel_data*)elem->channel_data;
   call_data* calld = (call_data*)elem->call_data;
   const grpc_connected_subchannel_call_args call_args = {
-      calld->pollent,                  // pollent
-      calld->path,                     // path
-      calld->call_start_time,          // start_time
-      calld->deadline,                 // deadline
-      calld->arena,                    // arena
-      calld->subchannel_call_context,  // context
-      calld->call_combiner             // call_combiner
+      calld->pollent,                       // pollent
+      calld->path,                          // path
+      calld->call_start_time,               // start_time
+      calld->deadline,                      // deadline
+      calld->arena,                         // arena
+      calld->pick.subchannel_call_context,  // context
+      calld->call_combiner                  // call_combiner
   };
   grpc_error* new_error = grpc_connected_subchannel_create_call(
-      calld->connected_subchannel, &call_args, &calld->subchannel_call);
+      calld->pick.connected_subchannel, &call_args, &calld->subchannel_call);
   if (grpc_client_channel_trace.enabled()) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: create subchannel_call=%p: error=%s",
             chand, calld, calld->subchannel_call, grpc_error_string(new_error));
@@ -1032,7 +1031,7 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
 static void pick_done_locked(grpc_call_element* elem, grpc_error* error) {
   call_data* calld = (call_data*)elem->call_data;
   channel_data* chand = (channel_data*)elem->channel_data;
-  if (calld->connected_subchannel == nullptr) {
+  if (calld->pick.connected_subchannel == nullptr) {
     // Failed to create subchannel.
     GRPC_ERROR_UNREF(calld->error);
     calld->error = error == GRPC_ERROR_NONE
@@ -1071,13 +1070,16 @@ static void pick_callback_cancel_locked(void* arg, grpc_error* error) {
   grpc_call_element* elem = (grpc_call_element*)arg;
   channel_data* chand = (channel_data*)elem->channel_data;
   call_data* calld = (call_data*)elem->call_data;
-  if (calld->lb_policy != nullptr) {
+  // Note: chand->lb_policy may have changed since we started our pick,
+  // in which case we will be cancelling the pick on a policy other than
+  // the one we started it on.  However, this will just be a no-op.
+  if (error != GRPC_ERROR_NONE && chand->lb_policy != nullptr) {
     if (grpc_client_channel_trace.enabled()) {
       gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p",
-              chand, calld, calld->lb_policy);
+              chand, calld, chand->lb_policy);
     }
-    grpc_lb_policy_cancel_pick_locked(
-        calld->lb_policy, &calld->connected_subchannel, GRPC_ERROR_REF(error));
+    grpc_lb_policy_cancel_pick_locked(chand->lb_policy, &calld->pick,
+                                      GRPC_ERROR_REF(error));
   }
   GRPC_CALL_STACK_UNREF(calld->owning_call, "pick_callback_cancel");
 }
@@ -1092,9 +1094,6 @@ static void pick_callback_done_locked(void* arg, grpc_error* error) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously",
             chand, calld);
   }
-  GPR_ASSERT(calld->lb_policy != nullptr);
-  GRPC_LB_POLICY_UNREF(calld->lb_policy, "pick_subchannel");
-  calld->lb_policy = nullptr;
   async_pick_done_locked(elem, GRPC_ERROR_REF(error));
 }
 
@@ -1128,26 +1127,21 @@ static bool pick_callback_start_locked(grpc_call_element* elem) {
       initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY;
     }
   }
-  const grpc_lb_policy_pick_args inputs = {
+  calld->pick.initial_metadata =
       calld->initial_metadata_batch->payload->send_initial_metadata
-          .send_initial_metadata,
-      initial_metadata_flags, &calld->lb_token_mdelem};
-  // Keep a ref to the LB policy in calld while the pick is pending.
-  GRPC_LB_POLICY_REF(chand->lb_policy, "pick_subchannel");
-  calld->lb_policy = chand->lb_policy;
+          .send_initial_metadata;
+  calld->pick.initial_metadata_flags = initial_metadata_flags;
   GRPC_CLOSURE_INIT(&calld->lb_pick_closure, pick_callback_done_locked, elem,
                     grpc_combiner_scheduler(chand->combiner));
-  const bool pick_done = grpc_lb_policy_pick_locked(
-      chand->lb_policy, &inputs, &calld->connected_subchannel,
-      calld->subchannel_call_context, nullptr, &calld->lb_pick_closure);
+  calld->pick.on_complete = &calld->lb_pick_closure;
+  const bool pick_done =
+      grpc_lb_policy_pick_locked(chand->lb_policy, &calld->pick);
   if (pick_done) {
     /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */
     if (grpc_client_channel_trace.enabled()) {
       gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed synchronously",
               chand, calld);
     }
-    GRPC_LB_POLICY_UNREF(calld->lb_policy, "pick_subchannel");
-    calld->lb_policy = nullptr;
   } else {
     GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel");
     grpc_call_combiner_set_notify_on_cancel(
@@ -1289,7 +1283,7 @@ static void start_pick_locked(void* arg, grpc_error* ignored) {
   grpc_call_element* elem = (grpc_call_element*)arg;
   call_data* calld = (call_data*)elem->call_data;
   channel_data* chand = (channel_data*)elem->channel_data;
-  GPR_ASSERT(calld->connected_subchannel == nullptr);
+  GPR_ASSERT(calld->pick.connected_subchannel == nullptr);
   if (chand->lb_policy != nullptr) {
     // We already have an LB policy, so ask it for a pick.
     if (pick_callback_start_locked(elem)) {
@@ -1467,15 +1461,14 @@ static void cc_destroy_call_elem(grpc_call_element* elem,
     GRPC_SUBCHANNEL_CALL_UNREF(calld->subchannel_call,
                                "client_channel_destroy_call");
   }
-  GPR_ASSERT(calld->lb_policy == nullptr);
   GPR_ASSERT(calld->waiting_for_pick_batches_count == 0);
-  if (calld->connected_subchannel != nullptr) {
-    GRPC_CONNECTED_SUBCHANNEL_UNREF(calld->connected_subchannel, "picked");
+  if (calld->pick.connected_subchannel != nullptr) {
+    GRPC_CONNECTED_SUBCHANNEL_UNREF(calld->pick.connected_subchannel, "picked");
   }
   for (size_t i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
-    if (calld->subchannel_call_context[i].value != nullptr) {
-      calld->subchannel_call_context[i].destroy(
-          calld->subchannel_call_context[i].value);
+    if (calld->pick.subchannel_call_context[i].value != nullptr) {
+      calld->pick.subchannel_call_context[i].destroy(
+          calld->pick.subchannel_call_context[i].value);
     }
   }
   GRPC_CLOSURE_SCHED(then_schedule_closure, GRPC_ERROR_NONE);
diff --git a/src/core/ext/filters/client_channel/lb_policy.cc b/src/core/ext/filters/client_channel/lb_policy.cc
index 7a5a8dec34b..cc4fe7ec627 100644
--- a/src/core/ext/filters/client_channel/lb_policy.cc
+++ b/src/core/ext/filters/client_channel/lb_policy.cc
@@ -19,8 +19,6 @@
 #include "src/core/ext/filters/client_channel/lb_policy.h"
 #include "src/core/lib/iomgr/combiner.h"
 
-#define WEAK_REF_BITS 16
-
 grpc_core::DebugOnlyTraceFlag grpc_trace_lb_policy_refcount(
     false, "lb_policy_refcount");
 
@@ -28,91 +26,60 @@ void grpc_lb_policy_init(grpc_lb_policy* policy,
                          const grpc_lb_policy_vtable* vtable,
                          grpc_combiner* combiner) {
   policy->vtable = vtable;
-  gpr_atm_no_barrier_store(&policy->ref_pair, 1 << WEAK_REF_BITS);
+  gpr_ref_init(&policy->refs, 1);
   policy->interested_parties = grpc_pollset_set_create();
   policy->combiner = GRPC_COMBINER_REF(combiner, "lb_policy");
 }
 
 #ifndef NDEBUG
-#define REF_FUNC_EXTRA_ARGS , const char *file, int line, const char *reason
-#define REF_MUTATE_EXTRA_ARGS REF_FUNC_EXTRA_ARGS, const char* purpose
-#define REF_FUNC_PASS_ARGS(new_reason) , file, line, new_reason
-#define REF_MUTATE_PASS_ARGS(purpose) , file, line, reason, purpose
+void grpc_lb_policy_ref(grpc_lb_policy* lb_policy, const char* file, int line,
+                        const char* reason) {
+  if (grpc_trace_lb_policy_refcount.enabled()) {
+    gpr_atm old_refs = gpr_atm_no_barrier_load(&lb_policy->refs.count);
+    gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+            "LB_POLICY:%p   ref %" PRIdPTR " -> %" PRIdPTR " %s", lb_policy,
+            old_refs, old_refs + 1, reason);
+  }
 #else
-#define REF_FUNC_EXTRA_ARGS
-#define REF_MUTATE_EXTRA_ARGS
-#define REF_FUNC_PASS_ARGS(new_reason)
-#define REF_MUTATE_PASS_ARGS(x)
+void grpc_lb_policy_ref(grpc_lb_policy* lb_policy) {
 #endif
+  gpr_ref(&lb_policy->refs);
+}
 
-static gpr_atm ref_mutate(grpc_lb_policy* c, gpr_atm delta,
-                          int barrier REF_MUTATE_EXTRA_ARGS) {
-  gpr_atm old_val = barrier ? gpr_atm_full_fetch_add(&c->ref_pair, delta)
-                            : gpr_atm_no_barrier_fetch_add(&c->ref_pair, delta);
 #ifndef NDEBUG
+void grpc_lb_policy_unref(grpc_lb_policy* lb_policy, const char* file, int line,
+                          const char* reason) {
   if (grpc_trace_lb_policy_refcount.enabled()) {
+    gpr_atm old_refs = gpr_atm_no_barrier_load(&lb_policy->refs.count);
     gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-            "LB_POLICY: %p %12s 0x%" PRIxPTR " -> 0x%" PRIxPTR " [%s]", c,
-            purpose, old_val, old_val + delta, reason);
+            "LB_POLICY:%p unref %" PRIdPTR " -> %" PRIdPTR " %s", lb_policy,
+            old_refs, old_refs - 1, reason);
   }
+#else
+void grpc_lb_policy_unref(grpc_lb_policy* lb_policy) {
 #endif
-  return old_val;
-}
-
-void grpc_lb_policy_ref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
-  ref_mutate(policy, 1 << WEAK_REF_BITS, 0 REF_MUTATE_PASS_ARGS("STRONG_REF"));
-}
-
-static void shutdown_locked(void* arg, grpc_error* error) {
-  grpc_lb_policy* policy = (grpc_lb_policy*)arg;
-  policy->vtable->shutdown_locked(policy);
-  GRPC_LB_POLICY_WEAK_UNREF(policy, "strong-unref");
-}
-
-void grpc_lb_policy_unref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
-  gpr_atm old_val =
-      ref_mutate(policy, (gpr_atm)1 - (gpr_atm)(1 << WEAK_REF_BITS),
-                 1 REF_MUTATE_PASS_ARGS("STRONG_UNREF"));
-  gpr_atm mask = ~(gpr_atm)((1 << WEAK_REF_BITS) - 1);
-  gpr_atm check = 1 << WEAK_REF_BITS;
-  if ((old_val & mask) == check) {
-    GRPC_CLOSURE_SCHED(
-        GRPC_CLOSURE_CREATE(shutdown_locked, policy,
-                            grpc_combiner_scheduler(policy->combiner)),
-        GRPC_ERROR_NONE);
-  } else {
-    grpc_lb_policy_weak_unref(policy REF_FUNC_PASS_ARGS("strong-unref"));
+  if (gpr_unref(&lb_policy->refs)) {
+    grpc_pollset_set_destroy(lb_policy->interested_parties);
+    grpc_combiner* combiner = lb_policy->combiner;
+    lb_policy->vtable->destroy(lb_policy);
+    GRPC_COMBINER_UNREF(combiner, "lb_policy");
   }
 }
 
-void grpc_lb_policy_weak_ref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
-  ref_mutate(policy, 1, 0 REF_MUTATE_PASS_ARGS("WEAK_REF"));
-}
-
-void grpc_lb_policy_weak_unref(grpc_lb_policy* policy REF_FUNC_EXTRA_ARGS) {
-  gpr_atm old_val =
-      ref_mutate(policy, -(gpr_atm)1, 1 REF_MUTATE_PASS_ARGS("WEAK_UNREF"));
-  if (old_val == 1) {
-    grpc_pollset_set_destroy(policy->interested_parties);
-    grpc_combiner* combiner = policy->combiner;
-    policy->vtable->destroy(policy);
-    GRPC_COMBINER_UNREF(combiner, "lb_policy");
-  }
+void grpc_lb_policy_shutdown_locked(grpc_lb_policy* policy,
+                                    grpc_lb_policy* new_policy) {
+  policy->vtable->shutdown_locked(policy, new_policy);
 }
 
 int grpc_lb_policy_pick_locked(grpc_lb_policy* policy,
-                               const grpc_lb_policy_pick_args* pick_args,
-                               grpc_connected_subchannel** target,
-                               grpc_call_context_element* context,
-                               void** user_data, grpc_closure* on_complete) {
-  return policy->vtable->pick_locked(policy, pick_args, target, context,
-                                     user_data, on_complete);
+                               grpc_lb_policy_pick_state* pick) {
+  return policy->vtable->pick_locked(policy, pick);
 }
 
 void grpc_lb_policy_cancel_pick_locked(grpc_lb_policy* policy,
-                                       grpc_connected_subchannel** target,
+                                       grpc_lb_policy_pick_state* pick,
                                        grpc_error* error) {
-  policy->vtable->cancel_pick_locked(policy, target, error);
+  policy->vtable->cancel_pick_locked(policy, pick, error);
 }
 
 void grpc_lb_policy_cancel_picks_locked(grpc_lb_policy* policy,
diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h
index 3572c97ed14..1176a05b780 100644
--- a/src/core/ext/filters/client_channel/lb_policy.h
+++ b/src/core/ext/filters/client_channel/lb_policy.h
@@ -33,7 +33,7 @@ extern grpc_core::DebugOnlyTraceFlag grpc_trace_lb_policy_refcount;
 
 struct grpc_lb_policy {
   const grpc_lb_policy_vtable* vtable;
-  gpr_atm ref_pair;
+  gpr_refcount refs;
   /* owned pointer to interested parties in load balancing decisions */
   grpc_pollset_set* interested_parties;
   /* combiner under which lb_policy actions take place */
@@ -42,32 +42,42 @@ struct grpc_lb_policy {
   grpc_closure* request_reresolution;
 };
 
-/** Extra arguments for an LB pick */
-typedef struct grpc_lb_policy_pick_args {
-  /** Initial metadata associated with the picking call. */
+/// State used for an LB pick.
+typedef struct grpc_lb_policy_pick_state {
+  /// Initial metadata associated with the picking call.
   grpc_metadata_batch* initial_metadata;
-  /** Bitmask used for selective cancelling. See \a
-   * grpc_lb_policy_cancel_picks() and \a GRPC_INITIAL_METADATA_* in
-   * grpc_types.h */
+  /// Bitmask used for selective cancelling. See \a
+  /// grpc_lb_policy_cancel_picks() and \a GRPC_INITIAL_METADATA_* in
+  /// grpc_types.h.
   uint32_t initial_metadata_flags;
-  /** Storage for LB token in \a initial_metadata, or NULL if not used */
-  grpc_linked_mdelem* lb_token_mdelem_storage;
-} grpc_lb_policy_pick_args;
+  /// Storage for LB token in \a initial_metadata, or NULL if not used.
+  grpc_linked_mdelem lb_token_mdelem_storage;
+  /// Closure to run when pick is complete, if not completed synchronously.
+  grpc_closure* on_complete;
+  /// Will be set to the selected subchannel, or NULL on failure or when
+  /// the LB policy decides to drop the call.
+  grpc_connected_subchannel* connected_subchannel;
+  /// Will be populated with context to pass to the subchannel call, if needed.
+  grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
+  /// Upon success, \a *user_data will be set to whatever opaque information
+  /// may need to be propagated from the LB policy, or NULL if not needed.
+  void** user_data;
+  /// Next pointer.  For internal use by LB policy.
+  struct grpc_lb_policy_pick_state* next;
+} grpc_lb_policy_pick_state;
 
 struct grpc_lb_policy_vtable {
   void (*destroy)(grpc_lb_policy* policy);
-  void (*shutdown_locked)(grpc_lb_policy* policy);
+
+  /// \see grpc_lb_policy_shutdown_locked().
+  void (*shutdown_locked)(grpc_lb_policy* policy, grpc_lb_policy* new_policy);
 
   /** \see grpc_lb_policy_pick */
-  int (*pick_locked)(grpc_lb_policy* policy,
-                     const grpc_lb_policy_pick_args* pick_args,
-                     grpc_connected_subchannel** target,
-                     grpc_call_context_element* context, void** user_data,
-                     grpc_closure* on_complete);
+  int (*pick_locked)(grpc_lb_policy* policy, grpc_lb_policy_pick_state* pick);
 
   /** \see grpc_lb_policy_cancel_pick */
   void (*cancel_pick_locked)(grpc_lb_policy* policy,
-                             grpc_connected_subchannel** target,
+                             grpc_lb_policy_pick_state* pick,
                              grpc_error* error);
 
   /** \see grpc_lb_policy_cancel_picks */
@@ -103,37 +113,19 @@ struct grpc_lb_policy_vtable {
 };
 
 #ifndef NDEBUG
-
-/* Strong references: the policy will shutdown when they reach zero */
 #define GRPC_LB_POLICY_REF(p, r) \
   grpc_lb_policy_ref((p), __FILE__, __LINE__, (r))
 #define GRPC_LB_POLICY_UNREF(p, r) \
   grpc_lb_policy_unref((p), __FILE__, __LINE__, (r))
-
-/* Weak references: they don't prevent the shutdown of the LB policy. When no
- * strong references are left but there are still weak ones, shutdown is called.
- * Once the weak reference also reaches zero, the LB policy is destroyed. */
-#define GRPC_LB_POLICY_WEAK_REF(p, r) \
-  grpc_lb_policy_weak_ref((p), __FILE__, __LINE__, (r))
-#define GRPC_LB_POLICY_WEAK_UNREF(p, r) \
-  grpc_lb_policy_weak_unref((p), __FILE__, __LINE__, (r))
 void grpc_lb_policy_ref(grpc_lb_policy* policy, const char* file, int line,
                         const char* reason);
 void grpc_lb_policy_unref(grpc_lb_policy* policy, const char* file, int line,
                           const char* reason);
-void grpc_lb_policy_weak_ref(grpc_lb_policy* policy, const char* file, int line,
-                             const char* reason);
-void grpc_lb_policy_weak_unref(grpc_lb_policy* policy, const char* file,
-                               int line, const char* reason);
-#else
+#else  // !NDEBUG
 #define GRPC_LB_POLICY_REF(p, r) grpc_lb_policy_ref((p))
 #define GRPC_LB_POLICY_UNREF(p, r) grpc_lb_policy_unref((p))
-#define GRPC_LB_POLICY_WEAK_REF(p, r) grpc_lb_policy_weak_ref((p))
-#define GRPC_LB_POLICY_WEAK_UNREF(p, r) grpc_lb_policy_weak_unref((p))
 void grpc_lb_policy_ref(grpc_lb_policy* policy);
 void grpc_lb_policy_unref(grpc_lb_policy* policy);
-void grpc_lb_policy_weak_ref(grpc_lb_policy* policy);
-void grpc_lb_policy_weak_unref(grpc_lb_policy* policy);
 #endif
 
 /** called by concrete implementations to initialize the base struct */
@@ -141,28 +133,24 @@ void grpc_lb_policy_init(grpc_lb_policy* policy,
                          const grpc_lb_policy_vtable* vtable,
                          grpc_combiner* combiner);
 
-/** Finds an appropriate subchannel for a call, based on \a pick_args.
-
-    \a target will be set to the selected subchannel, or NULL on failure
-    or when the LB policy decides to drop the call.
+/// Shuts down \a policy.
+/// If \a new_policy is non-null, any pending picks will be restarted
+/// on that policy; otherwise, they will be failed.
+void grpc_lb_policy_shutdown_locked(grpc_lb_policy* policy,
+                                    grpc_lb_policy* new_policy);
 
-    Upon success, \a user_data will be set to whatever opaque information
-    may need to be propagated from the LB policy, or NULL if not needed.
-    \a context will be populated with context to pass to the subchannel
-    call, if needed.
+/** Finds an appropriate subchannel for a call, based on data in \a pick.
+    \a pick must remain alive until the pick is complete.
 
     If the pick succeeds and a result is known immediately, a non-zero
-    value will be returned.  Otherwise, \a on_complete will be invoked
+    value will be returned.  Otherwise, \a pick->on_complete will be invoked
     once the pick is complete with its error argument set to indicate
     success or failure.
 
     Any IO should be done under the \a interested_parties \a grpc_pollset_set
     in the \a grpc_lb_policy struct. */
 int grpc_lb_policy_pick_locked(grpc_lb_policy* policy,
-                               const grpc_lb_policy_pick_args* pick_args,
-                               grpc_connected_subchannel** target,
-                               grpc_call_context_element* context,
-                               void** user_data, grpc_closure* on_complete);
+                               grpc_lb_policy_pick_state* pick);
 
 /** Perform a connected subchannel ping (see \a grpc_connected_subchannel_ping)
     against one of the connected subchannels managed by \a policy. */
@@ -170,11 +158,11 @@ void grpc_lb_policy_ping_one_locked(grpc_lb_policy* policy,
                                     grpc_closure* on_initiate,
                                     grpc_closure* on_ack);
 
-/** Cancel picks for \a target.
+/** Cancel picks for \a pick.
     The \a on_complete callback of the pending picks will be invoked with \a
     *target set to NULL. */
 void grpc_lb_policy_cancel_pick_locked(grpc_lb_policy* policy,
-                                       grpc_connected_subchannel** target,
+                                       grpc_lb_policy_pick_state* pick,
                                        grpc_error* error);
 
 /** Cancel all pending picks for which their \a initial_metadata_flags (as given
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index 1317cdcf750..5849ac9d2da 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -54,7 +54,7 @@
  *    operations in progress over the old RR instance. This is done by
  *    decreasing the reference count on the old policy. The moment no more
  *    references are held on the old RR policy, it'll be destroyed and \a
- *    glb_rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN
+ *    on_rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN
  *    state. At this point we can transition to a new RR instance safely, which
  *    is done once again via \a rr_handover_locked().
  *
@@ -128,187 +128,48 @@
 
 grpc_core::TraceFlag grpc_lb_glb_trace(false, "glb");
 
-/* add lb_token of selected subchannel (address) to the call's initial
- * metadata */
-static grpc_error* initial_metadata_add_lb_token(
-    grpc_metadata_batch* initial_metadata,
-    grpc_linked_mdelem* lb_token_mdelem_storage, grpc_mdelem lb_token) {
-  GPR_ASSERT(lb_token_mdelem_storage != nullptr);
-  GPR_ASSERT(!GRPC_MDISNULL(lb_token));
-  return grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage,
-                                      lb_token);
-}
+struct glb_lb_policy;
 
-static void destroy_client_stats(void* arg) {
-  grpc_grpclb_client_stats_unref((grpc_grpclb_client_stats*)arg);
-}
-
-typedef struct wrapped_rr_closure_arg {
-  /* the closure instance using this struct as argument */
-  grpc_closure wrapper_closure;
-
-  /* the original closure. Usually a on_complete/notify cb for pick() and ping()
-   * calls against the internal RR instance, respectively. */
-  grpc_closure* wrapped_closure;
-
-  /* the pick's initial metadata, kept in order to append the LB token for the
-   * pick */
-  grpc_metadata_batch* initial_metadata;
-
-  /* the picked target, used to determine which LB token to add to the pick's
-   * initial metadata */
-  grpc_connected_subchannel** target;
-
-  /* the context to be populated for the subchannel call */
-  grpc_call_context_element* context;
+namespace {
 
-  /* Stats for client-side load reporting. Note that this holds a
-   * reference, which must be either passed on via context or unreffed. */
+/// Linked list of pending pick requests. It stores all information needed to
+/// eventually call (Round Robin's) pick() on them. They mainly stay pending
+/// waiting for the RR policy to be created.
+///
+/// Note that when a pick is sent to the RR policy, we inject our own
+/// on_complete callback, so that we can intercept the result before
+/// invoking the original on_complete callback.  This allows us to set the
+/// LB token metadata and add client_stats to the call context.
+/// See \a pending_pick_complete() for details.
+struct pending_pick {
+  // Our on_complete closure and the original one.
+  grpc_closure on_complete;
+  grpc_closure* original_on_complete;
+  // The original pick.
+  grpc_lb_policy_pick_state* pick;
+  // Stats for client-side load reporting. Note that this holds a
+  // reference, which must be either passed on via context or unreffed.
   grpc_grpclb_client_stats* client_stats;
-
-  /* the LB token associated with the pick */
+  // The LB token associated with the pick.  This is set via user_data in
+  // the pick.
   grpc_mdelem lb_token;
-
-  /* storage for the lb token initial metadata mdelem */
-  grpc_linked_mdelem* lb_token_mdelem_storage;
-
-  /* The RR instance related to the closure */
-  grpc_lb_policy* rr_policy;
-
-  /* The grpclb instance that created the wrapping. This instance is not owned,
-   * reference counts are untouched. It's used only for logging purposes. */
-  grpc_lb_policy* glb_policy;
-
-  /* heap memory to be freed upon closure execution. */
-  void* free_when_done;
-} wrapped_rr_closure_arg;
-
-/* The \a on_complete closure passed as part of the pick requires keeping a
- * reference to its associated round robin instance. We wrap this closure in
- * order to unref the round robin instance upon its invocation */
-static void wrapped_rr_closure(void* arg, grpc_error* error) {
-  wrapped_rr_closure_arg* wc_arg = (wrapped_rr_closure_arg*)arg;
-
-  GPR_ASSERT(wc_arg->wrapped_closure != nullptr);
-  GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_REF(error));
-
-  if (wc_arg->rr_policy != nullptr) {
-    /* if *target is nullptr, no pick has been made by the RR policy (eg, all
-     * addresses failed to connect). There won't be any user_data/token
-     * available */
-    if (*wc_arg->target != nullptr) {
-      if (!GRPC_MDISNULL(wc_arg->lb_token)) {
-        initial_metadata_add_lb_token(wc_arg->initial_metadata,
-                                      wc_arg->lb_token_mdelem_storage,
-                                      GRPC_MDELEM_REF(wc_arg->lb_token));
-      } else {
-        gpr_log(
-            GPR_ERROR,
-            "[grpclb %p] No LB token for connected subchannel pick %p (from RR "
-            "instance %p).",
-            wc_arg->glb_policy, *wc_arg->target, wc_arg->rr_policy);
-        abort();
-      }
-      // Pass on client stats via context. Passes ownership of the reference.
-      GPR_ASSERT(wc_arg->client_stats != nullptr);
-      wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].value = wc_arg->client_stats;
-      wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].destroy = destroy_client_stats;
-    } else {
-      grpc_grpclb_client_stats_unref(wc_arg->client_stats);
-    }
-    if (grpc_lb_glb_trace.enabled()) {
-      gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p", wc_arg->glb_policy,
-              wc_arg->rr_policy);
-    }
-    GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "wrapped_rr_closure");
-  }
-  GPR_ASSERT(wc_arg->free_when_done != nullptr);
-  gpr_free(wc_arg->free_when_done);
-}
-
-namespace {
-/* Linked list of pending pick requests. It stores all information needed to
- * eventually call (Round Robin's) pick() on them. They mainly stay pending
- * waiting for the RR policy to be created/updated.
- *
- * One particularity is the wrapping of the user-provided \a on_complete closure
- * (in \a wrapped_on_complete and \a wrapped_on_complete_arg). This is needed in
- * order to correctly unref the RR policy instance upon completion of the pick.
- * See \a wrapped_rr_closure for details. */
-struct pending_pick {
+  // The grpclb instance that created the wrapping. This instance is not owned,
+  // reference counts are untouched. It's used only for logging purposes.
+  glb_lb_policy* glb_policy;
+  // Next pending pick.
   struct pending_pick* next;
-
-  /* original pick()'s arguments */
-  grpc_lb_policy_pick_args pick_args;
-
-  /* output argument where to store the pick()ed connected subchannel, or
-   * nullptr upon error. */
-  grpc_connected_subchannel** target;
-
-  /* args for wrapped_on_complete */
-  wrapped_rr_closure_arg wrapped_on_complete_arg;
 };
-}  // namespace
-
-static void add_pending_pick(pending_pick** root,
-                             const grpc_lb_policy_pick_args* pick_args,
-                             grpc_connected_subchannel** target,
-                             grpc_call_context_element* context,
-                             grpc_closure* on_complete) {
-  pending_pick* pp = (pending_pick*)gpr_zalloc(sizeof(*pp));
-  pp->next = *root;
-  pp->pick_args = *pick_args;
-  pp->target = target;
-  pp->wrapped_on_complete_arg.wrapped_closure = on_complete;
-  pp->wrapped_on_complete_arg.target = target;
-  pp->wrapped_on_complete_arg.context = context;
-  pp->wrapped_on_complete_arg.initial_metadata = pick_args->initial_metadata;
-  pp->wrapped_on_complete_arg.lb_token_mdelem_storage =
-      pick_args->lb_token_mdelem_storage;
-  pp->wrapped_on_complete_arg.free_when_done = pp;
-  GRPC_CLOSURE_INIT(&pp->wrapped_on_complete_arg.wrapper_closure,
-                    wrapped_rr_closure, &pp->wrapped_on_complete_arg,
-                    grpc_schedule_on_exec_ctx);
-  *root = pp;
-}
 
-/* Same as the \a pending_pick struct but for ping operations */
-typedef struct pending_ping {
+/// A linked list of pending pings waiting for the RR policy to be created.
+struct pending_ping {
+  grpc_closure* on_initiate;
+  grpc_closure* on_ack;
   struct pending_ping* next;
+};
 
-  /* args for sending the ping */
-  wrapped_rr_closure_arg* on_initiate;
-  wrapped_rr_closure_arg* on_ack;
-} pending_ping;
-
-static void add_pending_ping(pending_ping** root, grpc_closure* on_initiate,
-                             grpc_closure* on_ack) {
-  pending_ping* pping = (pending_ping*)gpr_zalloc(sizeof(*pping));
-  if (on_initiate != nullptr) {
-    pping->on_initiate =
-        (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(*pping->on_initiate));
-    pping->on_initiate->wrapped_closure = on_initiate;
-    pping->on_initiate->free_when_done = pping->on_initiate;
-    GRPC_CLOSURE_INIT(&pping->on_initiate->wrapper_closure, wrapped_rr_closure,
-                      &pping->on_initiate, grpc_schedule_on_exec_ctx);
-  }
-  if (on_ack != nullptr) {
-    pping->on_ack = (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(*pping->on_ack));
-    pping->on_ack->wrapped_closure = on_ack;
-    pping->on_ack->free_when_done = pping->on_ack;
-    GRPC_CLOSURE_INIT(&pping->on_ack->wrapper_closure, wrapped_rr_closure,
-                      &pping->on_ack, grpc_schedule_on_exec_ctx);
-  }
-  pping->next = *root;
-  *root = pping;
-}
-
-/*
- * glb_lb_policy
- */
-typedef struct rr_connectivity_data rr_connectivity_data;
+}  // namespace
 
-typedef struct glb_lb_policy {
+struct glb_lb_policy {
   /** base policy: must be first */
   grpc_lb_policy base;
 
@@ -333,6 +194,9 @@ typedef struct glb_lb_policy {
   /** the RR policy to use of the backend servers returned by the LB server */
   grpc_lb_policy* rr_policy;
 
+  grpc_closure on_rr_connectivity_changed;
+  grpc_connectivity_state rr_connectivity_state;
+
   bool started_picking;
 
   /** our connectivity state tracker */
@@ -437,15 +301,85 @@ typedef struct glb_lb_policy {
   grpc_closure client_load_report_closure;
   /* Client load report message payload. */
   grpc_byte_buffer* client_load_report_payload;
-} glb_lb_policy;
-
-/* Keeps track and reacts to changes in connectivity of the RR instance */
-struct rr_connectivity_data {
-  grpc_closure on_change;
-  grpc_connectivity_state state;
-  glb_lb_policy* glb_policy;
 };
 
+/* add lb_token of selected subchannel (address) to the call's initial
+ * metadata */
+static grpc_error* initial_metadata_add_lb_token(
+    grpc_metadata_batch* initial_metadata,
+    grpc_linked_mdelem* lb_token_mdelem_storage, grpc_mdelem lb_token) {
+  GPR_ASSERT(lb_token_mdelem_storage != nullptr);
+  GPR_ASSERT(!GRPC_MDISNULL(lb_token));
+  return grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage,
+                                      lb_token);
+}
+
+static void destroy_client_stats(void* arg) {
+  grpc_grpclb_client_stats_unref((grpc_grpclb_client_stats*)arg);
+}
+
+static void pending_pick_set_metadata_and_context(pending_pick* pp) {
+  /* if connected_subchannel is nullptr, no pick has been made by the RR
+   * policy (e.g., all addresses failed to connect). There won't be any
+   * user_data/token available */
+  if (pp->pick->connected_subchannel != nullptr) {
+    if (!GRPC_MDISNULL(pp->lb_token)) {
+      initial_metadata_add_lb_token(pp->pick->initial_metadata,
+                                    &pp->pick->lb_token_mdelem_storage,
+                                    GRPC_MDELEM_REF(pp->lb_token));
+    } else {
+      gpr_log(GPR_ERROR,
+              "[grpclb %p] No LB token for connected subchannel pick %p",
+              pp->glb_policy, pp->pick);
+      abort();
+    }
+    // Pass on client stats via context. Passes ownership of the reference.
+    GPR_ASSERT(pp->client_stats != nullptr);
+    pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].value =
+        pp->client_stats;
+    pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].destroy =
+        destroy_client_stats;
+  } else {
+    grpc_grpclb_client_stats_unref(pp->client_stats);
+  }
+}
+
+/* The \a on_complete closure passed as part of the pick requires keeping a
+ * reference to its associated round robin instance. We wrap this closure in
+ * order to unref the round robin instance upon its invocation */
+static void pending_pick_complete(void* arg, grpc_error* error) {
+  pending_pick* pp = (pending_pick*)arg;
+  pending_pick_set_metadata_and_context(pp);
+  GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_REF(error));
+  gpr_free(pp);
+}
+
+static pending_pick* pending_pick_create(glb_lb_policy* glb_policy,
+                                         grpc_lb_policy_pick_state* pick) {
+  pending_pick* pp = (pending_pick*)gpr_zalloc(sizeof(*pp));
+  pp->pick = pick;
+  pp->glb_policy = glb_policy;
+  GRPC_CLOSURE_INIT(&pp->on_complete, pending_pick_complete, pp,
+                    grpc_schedule_on_exec_ctx);
+  pp->original_on_complete = pick->on_complete;
+  pp->pick->on_complete = &pp->on_complete;
+  return pp;
+}
+
+static void pending_pick_add(pending_pick** root, pending_pick* new_pp) {
+  new_pp->next = *root;
+  *root = new_pp;
+}
+
+static void pending_ping_add(pending_ping** root, grpc_closure* on_initiate,
+                             grpc_closure* on_ack) {
+  pending_ping* pping = (pending_ping*)gpr_zalloc(sizeof(*pping));
+  pping->on_initiate = on_initiate;
+  pping->on_ack = on_ack;
+  pping->next = *root;
+  *root = pping;
+}
+
 static bool is_server_valid(const grpc_grpclb_server* server, size_t idx,
                             bool log) {
   if (server->drop) return false;
@@ -557,7 +491,6 @@ static grpc_lb_addresses* process_serverlist_locked(
       gpr_free(uri);
       user_data = (void*)GRPC_MDELEM_LB_TOKEN_EMPTY.payload;
     }
-
     grpc_lb_addresses_set_address(lb_addresses, addr_idx, &addr.addr, addr.len,
                                   false /* is_balancer */,
                                   nullptr /* balancer_name */, user_data);
@@ -598,7 +531,6 @@ static void update_lb_connectivity_status_locked(
     grpc_error* rr_state_error) {
   const grpc_connectivity_state curr_glb_state =
       grpc_connectivity_state_check(&glb_policy->state_tracker);
-
   /* The new connectivity status is a function of the previous one and the new
    * input coming from the status of the RR policy.
    *
@@ -628,7 +560,6 @@ static void update_lb_connectivity_status_locked(
    *
    *  (*) This function mustn't be called during shutting down. */
   GPR_ASSERT(curr_glb_state != GRPC_CHANNEL_SHUTDOWN);
-
   switch (rr_state) {
     case GRPC_CHANNEL_TRANSIENT_FAILURE:
     case GRPC_CHANNEL_SHUTDOWN:
@@ -639,7 +570,6 @@ static void update_lb_connectivity_status_locked(
     case GRPC_CHANNEL_READY:
       GPR_ASSERT(rr_state_error == GRPC_ERROR_NONE);
   }
-
   if (grpc_lb_glb_trace.enabled()) {
     gpr_log(
         GPR_INFO,
@@ -657,10 +587,8 @@ static void update_lb_connectivity_status_locked(
  * cleanups this callback would otherwise be responsible for.
  * If \a force_async is true, then we will manually schedule the
  * completion callback even if the pick is available immediately. */
-static bool pick_from_internal_rr_locked(
-    glb_lb_policy* glb_policy, const grpc_lb_policy_pick_args* pick_args,
-    bool force_async, grpc_connected_subchannel** target,
-    wrapped_rr_closure_arg* wc_arg) {
+static bool pick_from_internal_rr_locked(glb_lb_policy* glb_policy,
+                                         bool force_async, pending_pick* pp) {
   // Check for drops if we are not using fallback backend addresses.
   if (glb_policy->serverlist != nullptr) {
     // Look at the index into the serverlist to see if we should drop this call.
@@ -670,57 +598,36 @@ static bool pick_from_internal_rr_locked(
       glb_policy->serverlist_index = 0;  // Wrap-around.
     }
     if (server->drop) {
-      // Not using the RR policy, so unref it.
-      if (grpc_lb_glb_trace.enabled()) {
-        gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p for drop", glb_policy,
-                wc_arg->rr_policy);
-      }
-      GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "glb_pick_sync");
       // Update client load reporting stats to indicate the number of
       // dropped calls.  Note that we have to do this here instead of in
       // the client_load_reporting filter, because we do not create a
       // subchannel call (and therefore no client_load_reporting filter)
       // for dropped calls.
-      GPR_ASSERT(wc_arg->client_stats != nullptr);
+      GPR_ASSERT(glb_policy->client_stats != nullptr);
       grpc_grpclb_client_stats_add_call_dropped_locked(
-          server->load_balance_token, wc_arg->client_stats);
-      grpc_grpclb_client_stats_unref(wc_arg->client_stats);
+          server->load_balance_token, glb_policy->client_stats);
       if (force_async) {
-        GPR_ASSERT(wc_arg->wrapped_closure != nullptr);
-        GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_NONE);
-        gpr_free(wc_arg->free_when_done);
+        GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE);
+        gpr_free(pp);
         return false;
       }
-      gpr_free(wc_arg->free_when_done);
+      gpr_free(pp);
       return true;
     }
   }
+  // Set client_stats and user_data.
+  pp->client_stats = grpc_grpclb_client_stats_ref(glb_policy->client_stats);
+  GPR_ASSERT(pp->pick->user_data == nullptr);
+  pp->pick->user_data = (void**)&pp->lb_token;
   // Pick via the RR policy.
-  const bool pick_done = grpc_lb_policy_pick_locked(
-      wc_arg->rr_policy, pick_args, target, wc_arg->context,
-      (void**)&wc_arg->lb_token, &wc_arg->wrapper_closure);
+  bool pick_done = grpc_lb_policy_pick_locked(glb_policy->rr_policy, pp->pick);
   if (pick_done) {
-    /* synchronous grpc_lb_policy_pick call. Unref the RR policy. */
-    if (grpc_lb_glb_trace.enabled()) {
-      gpr_log(GPR_INFO, "[grpclb %p] Unreffing RR %p", glb_policy,
-              wc_arg->rr_policy);
-    }
-    GRPC_LB_POLICY_UNREF(wc_arg->rr_policy, "glb_pick_sync");
-    /* add the load reporting initial metadata */
-    initial_metadata_add_lb_token(pick_args->initial_metadata,
-                                  pick_args->lb_token_mdelem_storage,
-                                  GRPC_MDELEM_REF(wc_arg->lb_token));
-    // Pass on client stats via context. Passes ownership of the reference.
-    GPR_ASSERT(wc_arg->client_stats != nullptr);
-    wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].value = wc_arg->client_stats;
-    wc_arg->context[GRPC_GRPCLB_CLIENT_STATS].destroy = destroy_client_stats;
+    pending_pick_set_metadata_and_context(pp);
     if (force_async) {
-      GPR_ASSERT(wc_arg->wrapped_closure != nullptr);
-      GRPC_CLOSURE_SCHED(wc_arg->wrapped_closure, GRPC_ERROR_NONE);
-      gpr_free(wc_arg->free_when_done);
-      return false;
+      GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE);
+      pick_done = false;
     }
-    gpr_free(wc_arg->free_when_done);
+    gpr_free(pp);
   }
   /* else, the pending pick will be registered and taken care of by the
    * pending pick list inside the RR policy (glb_policy->rr_policy).
@@ -762,7 +669,7 @@ static void lb_policy_args_destroy(grpc_lb_policy_args* args) {
   gpr_free(args);
 }
 
-static void glb_rr_connectivity_changed_locked(void* arg, grpc_error* error);
+static void on_rr_connectivity_changed_locked(void* arg, grpc_error* error);
 static void create_rr_locked(glb_lb_policy* glb_policy,
                              grpc_lb_policy_args* args) {
   GPR_ASSERT(glb_policy->rr_policy == nullptr);
@@ -784,72 +691,46 @@ static void create_rr_locked(glb_lb_policy* glb_policy,
   glb_policy->base.request_reresolution = nullptr;
   glb_policy->rr_policy = new_rr_policy;
   grpc_error* rr_state_error = nullptr;
-  const grpc_connectivity_state rr_state =
-      grpc_lb_policy_check_connectivity_locked(glb_policy->rr_policy,
-                                               &rr_state_error);
+  glb_policy->rr_connectivity_state = grpc_lb_policy_check_connectivity_locked(
+      glb_policy->rr_policy, &rr_state_error);
   /* Connectivity state is a function of the RR policy updated/created */
-  update_lb_connectivity_status_locked(glb_policy, rr_state, rr_state_error);
+  update_lb_connectivity_status_locked(
+      glb_policy, glb_policy->rr_connectivity_state, rr_state_error);
   /* Add the gRPC LB's interested_parties pollset_set to that of the newly
    * created RR policy. This will make the RR policy progress upon activity on
    * gRPC LB, which in turn is tied to the application's call */
   grpc_pollset_set_add_pollset_set(glb_policy->rr_policy->interested_parties,
                                    glb_policy->base.interested_parties);
-
-  /* Allocate the data for the tracking of the new RR policy's connectivity.
-   * It'll be deallocated in glb_rr_connectivity_changed() */
-  rr_connectivity_data* rr_connectivity =
-      (rr_connectivity_data*)gpr_zalloc(sizeof(rr_connectivity_data));
-  GRPC_CLOSURE_INIT(&rr_connectivity->on_change,
-                    glb_rr_connectivity_changed_locked, rr_connectivity,
+  GRPC_CLOSURE_INIT(&glb_policy->on_rr_connectivity_changed,
+                    on_rr_connectivity_changed_locked, glb_policy,
                     grpc_combiner_scheduler(glb_policy->base.combiner));
-  rr_connectivity->glb_policy = glb_policy;
-  rr_connectivity->state = rr_state;
-
   /* Subscribe to changes to the connectivity of the new RR */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "glb_rr_connectivity_cb");
-  grpc_lb_policy_notify_on_state_change_locked(glb_policy->rr_policy,
-                                               &rr_connectivity->state,
-                                               &rr_connectivity->on_change);
+  GRPC_LB_POLICY_REF(&glb_policy->base, "glb_rr_connectivity_cb");
+  grpc_lb_policy_notify_on_state_change_locked(
+      glb_policy->rr_policy, &glb_policy->rr_connectivity_state,
+      &glb_policy->on_rr_connectivity_changed);
   grpc_lb_policy_exit_idle_locked(glb_policy->rr_policy);
-
-  /* Update picks and pings in wait */
+  // Send pending picks to RR policy.
   pending_pick* pp;
   while ((pp = glb_policy->pending_picks)) {
     glb_policy->pending_picks = pp->next;
-    GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_pick");
-    pp->wrapped_on_complete_arg.rr_policy = glb_policy->rr_policy;
-    pp->wrapped_on_complete_arg.client_stats =
-        grpc_grpclb_client_stats_ref(glb_policy->client_stats);
     if (grpc_lb_glb_trace.enabled()) {
       gpr_log(GPR_INFO,
               "[grpclb %p] Pending pick about to (async) PICK from RR %p",
               glb_policy, glb_policy->rr_policy);
     }
-    pick_from_internal_rr_locked(glb_policy, &pp->pick_args,
-                                 true /* force_async */, pp->target,
-                                 &pp->wrapped_on_complete_arg);
+    pick_from_internal_rr_locked(glb_policy, true /* force_async */, pp);
   }
-
+  // Send pending pings to RR policy.
   pending_ping* pping;
   while ((pping = glb_policy->pending_pings)) {
     glb_policy->pending_pings = pping->next;
-    grpc_closure* on_initiate = nullptr;
-    grpc_closure* on_ack = nullptr;
-    if (pping->on_initiate != nullptr) {
-      GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_ping");
-      pping->on_initiate->rr_policy = glb_policy->rr_policy;
-      on_initiate = &pping->on_initiate->wrapper_closure;
-    }
-    if (pping->on_ack != nullptr) {
-      GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_ping");
-      pping->on_ack->rr_policy = glb_policy->rr_policy;
-      on_ack = &pping->on_ack->wrapper_closure;
-    }
     if (grpc_lb_glb_trace.enabled()) {
       gpr_log(GPR_INFO, "[grpclb %p] Pending ping about to PING from RR %p",
               glb_policy, glb_policy->rr_policy);
     }
-    grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, on_initiate, on_ack);
+    grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, pping->on_initiate,
+                                   pping->on_ack);
     gpr_free(pping);
   }
 }
@@ -875,31 +756,28 @@ static void rr_handover_locked(glb_lb_policy* glb_policy) {
   lb_policy_args_destroy(args);
 }
 
-static void glb_rr_connectivity_changed_locked(void* arg, grpc_error* error) {
-  rr_connectivity_data* rr_connectivity = (rr_connectivity_data*)arg;
-  glb_lb_policy* glb_policy = rr_connectivity->glb_policy;
+static void on_rr_connectivity_changed_locked(void* arg, grpc_error* error) {
+  glb_lb_policy* glb_policy = (glb_lb_policy*)arg;
   if (glb_policy->shutting_down) {
-    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
-    gpr_free(rr_connectivity);
+    GRPC_LB_POLICY_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
     return;
   }
-  if (rr_connectivity->state == GRPC_CHANNEL_SHUTDOWN) {
+  if (glb_policy->rr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
     /* An RR policy that has transitioned into the SHUTDOWN connectivity state
      * should not be considered for picks or updates: the SHUTDOWN state is a
      * sink, policies can't transition back from it. .*/
     GRPC_LB_POLICY_UNREF(glb_policy->rr_policy, "rr_connectivity_shutdown");
     glb_policy->rr_policy = nullptr;
-    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
-    gpr_free(rr_connectivity);
+    GRPC_LB_POLICY_UNREF(&glb_policy->base, "glb_rr_connectivity_cb");
     return;
   }
   /* rr state != SHUTDOWN && !glb_policy->shutting down: biz as usual */
-  update_lb_connectivity_status_locked(glb_policy, rr_connectivity->state,
-                                       GRPC_ERROR_REF(error));
-  /* Resubscribe. Reuse the "glb_rr_connectivity_cb" weak ref. */
-  grpc_lb_policy_notify_on_state_change_locked(glb_policy->rr_policy,
-                                               &rr_connectivity->state,
-                                               &rr_connectivity->on_change);
+  update_lb_connectivity_status_locked(
+      glb_policy, glb_policy->rr_connectivity_state, GRPC_ERROR_REF(error));
+  /* Resubscribe. Reuse the "glb_rr_connectivity_cb" ref. */
+  grpc_lb_policy_notify_on_state_change_locked(
+      glb_policy->rr_policy, &glb_policy->rr_connectivity_state,
+      &glb_policy->on_rr_connectivity_changed);
 }
 
 static void destroy_balancer_name(void* balancer_name) {
@@ -1007,22 +885,17 @@ static void glb_destroy(grpc_lb_policy* pol) {
   gpr_free(glb_policy);
 }
 
-static void glb_shutdown_locked(grpc_lb_policy* pol) {
+static void glb_shutdown_locked(grpc_lb_policy* pol,
+                                grpc_lb_policy* new_policy) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)pol;
   grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   glb_policy->shutting_down = true;
-
-  /* We need a copy of the lb_call pointer because we can't cancell the call
-   * while holding glb_policy->mu: lb_on_server_status_received, invoked due to
-   * the cancel, needs to acquire that same lock */
-  grpc_call* lb_call = glb_policy->lb_call;
-
   /* glb_policy->lb_call and this local lb_call must be consistent at this point
    * because glb_policy->lb_call is only assigned in lb_call_init_locked as part
    * of query_for_backends_locked, which can only be invoked while
    * glb_policy->shutting_down is false. */
-  if (lb_call != nullptr) {
-    grpc_call_cancel(lb_call, nullptr);
+  if (glb_policy->lb_call != nullptr) {
+    grpc_call_cancel(glb_policy->lb_call, nullptr);
     /* lb_on_server_status_received will pick up the cancel and clean up */
   }
   if (glb_policy->retry_timer_callback_pending) {
@@ -1031,12 +904,8 @@ static void glb_shutdown_locked(grpc_lb_policy* pol) {
   if (glb_policy->fallback_timer_callback_pending) {
     grpc_timer_cancel(&glb_policy->lb_fallback_timer);
   }
-
-  pending_pick* pp = glb_policy->pending_picks;
-  glb_policy->pending_picks = nullptr;
-  pending_ping* pping = glb_policy->pending_pings;
-  glb_policy->pending_pings = nullptr;
   if (glb_policy->rr_policy != nullptr) {
+    grpc_lb_policy_shutdown_locked(glb_policy->rr_policy, nullptr);
     GRPC_LB_POLICY_UNREF(glb_policy->rr_policy, "glb_shutdown");
   } else {
     grpc_lb_policy_try_reresolve(pol, &grpc_lb_glb_trace, GRPC_ERROR_CANCELLED);
@@ -1051,28 +920,33 @@ static void glb_shutdown_locked(grpc_lb_policy* pol) {
   }
   grpc_connectivity_state_set(&glb_policy->state_tracker, GRPC_CHANNEL_SHUTDOWN,
                               GRPC_ERROR_REF(error), "glb_shutdown");
-
+  // Clear pending picks.
+  pending_pick* pp = glb_policy->pending_picks;
+  glb_policy->pending_picks = nullptr;
   while (pp != nullptr) {
     pending_pick* next = pp->next;
-    *pp->target = nullptr;
-    GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure,
-                       GRPC_ERROR_REF(error));
-    gpr_free(pp);
+    if (new_policy != nullptr) {
+      // Hand pick over to new policy.
+      grpc_grpclb_client_stats_unref(pp->client_stats);
+      pp->pick->on_complete = pp->original_on_complete;
+      if (grpc_lb_policy_pick_locked(new_policy, pp->pick)) {
+        // Synchronous return; schedule callback.
+        GRPC_CLOSURE_SCHED(pp->pick->on_complete, GRPC_ERROR_NONE);
+      }
+      gpr_free(pp);
+    } else {
+      pp->pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_REF(error));
+    }
     pp = next;
   }
-
+  // Clear pending pings.
+  pending_ping* pping = glb_policy->pending_pings;
+  glb_policy->pending_pings = nullptr;
   while (pping != nullptr) {
     pending_ping* next = pping->next;
-    if (pping->on_initiate != nullptr) {
-      GRPC_CLOSURE_SCHED(&pping->on_initiate->wrapper_closure,
-                         GRPC_ERROR_REF(error));
-      gpr_free(pping->on_initiate);
-    }
-    if (pping->on_ack != nullptr) {
-      GRPC_CLOSURE_SCHED(&pping->on_ack->wrapper_closure,
-                         GRPC_ERROR_REF(error));
-      gpr_free(pping->on_ack);
-    }
+    GRPC_CLOSURE_SCHED(pping->on_initiate, GRPC_ERROR_REF(error));
+    GRPC_CLOSURE_SCHED(pping->on_ack, GRPC_ERROR_REF(error));
     gpr_free(pping);
     pping = next;
   }
@@ -1090,16 +964,16 @@ static void glb_shutdown_locked(grpc_lb_policy* pol) {
 //   level (grpclb), inside the glb_policy->pending_picks list. To cancel these,
 //   we invoke the completion closure and set *target to nullptr right here.
 static void glb_cancel_pick_locked(grpc_lb_policy* pol,
-                                   grpc_connected_subchannel** target,
+                                   grpc_lb_policy_pick_state* pick,
                                    grpc_error* error) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)pol;
   pending_pick* pp = glb_policy->pending_picks;
   glb_policy->pending_picks = nullptr;
   while (pp != nullptr) {
     pending_pick* next = pp->next;
-    if (pp->target == target) {
-      *target = nullptr;
-      GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure,
+    if (pp->pick == pick) {
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(&pp->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
     } else {
@@ -1109,7 +983,7 @@ static void glb_cancel_pick_locked(grpc_lb_policy* pol,
     pp = next;
   }
   if (glb_policy->rr_policy != nullptr) {
-    grpc_lb_policy_cancel_pick_locked(glb_policy->rr_policy, target,
+    grpc_lb_policy_cancel_pick_locked(glb_policy->rr_policy, pick,
                                       GRPC_ERROR_REF(error));
   }
   GRPC_ERROR_UNREF(error);
@@ -1134,9 +1008,9 @@ static void glb_cancel_picks_locked(grpc_lb_policy* pol,
   glb_policy->pending_picks = nullptr;
   while (pp != nullptr) {
     pending_pick* next = pp->next;
-    if ((pp->pick_args.initial_metadata_flags & initial_metadata_flags_mask) ==
+    if ((pp->pick->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      GRPC_CLOSURE_SCHED(&pp->wrapped_on_complete_arg.wrapper_closure,
+      GRPC_CLOSURE_SCHED(&pp->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
     } else {
@@ -1162,7 +1036,7 @@ static void start_picking_locked(glb_lb_policy* glb_policy) {
       !glb_policy->fallback_timer_callback_pending) {
     grpc_millis deadline =
         grpc_core::ExecCtx::Get()->Now() + glb_policy->lb_fallback_timeout_ms;
-    GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer");
+    GRPC_LB_POLICY_REF(&glb_policy->base, "grpclb_fallback_timer");
     GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked,
                       glb_policy,
                       grpc_combiner_scheduler(glb_policy->base.combiner));
@@ -1184,19 +1058,9 @@ static void glb_exit_idle_locked(grpc_lb_policy* pol) {
 }
 
 static int glb_pick_locked(grpc_lb_policy* pol,
-                           const grpc_lb_policy_pick_args* pick_args,
-                           grpc_connected_subchannel** target,
-                           grpc_call_context_element* context, void** user_data,
-                           grpc_closure* on_complete) {
-  if (pick_args->lb_token_mdelem_storage == nullptr) {
-    *target = nullptr;
-    GRPC_CLOSURE_SCHED(on_complete,
-                       GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                           "No mdelem storage for the LB token. Load reporting "
-                           "won't work without it. Failing"));
-    return 0;
-  }
+                           grpc_lb_policy_pick_state* pick) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)pol;
+  pending_pick* pp = pending_pick_create(glb_policy, pick);
   bool pick_done = false;
   if (glb_policy->rr_policy != nullptr) {
     const grpc_connectivity_state rr_connectivity_state =
@@ -1204,7 +1068,7 @@ static int glb_pick_locked(grpc_lb_policy* pol,
                                                  nullptr);
     // The glb_policy->rr_policy may have transitioned to SHUTDOWN but the
     // callback registered to capture this event
-    // (glb_rr_connectivity_changed_locked) may not have been invoked yet. We
+    // (on_rr_connectivity_changed_locked) may not have been invoked yet. We
     // need to make sure we aren't trying to pick from a RR policy instance
     // that's in shutdown.
     if (rr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
@@ -1214,32 +1078,16 @@ static int glb_pick_locked(grpc_lb_policy* pol,
                 glb_policy, glb_policy->rr_policy,
                 grpc_connectivity_state_name(rr_connectivity_state));
       }
-      add_pending_pick(&glb_policy->pending_picks, pick_args, target, context,
-                       on_complete);
+      pending_pick_add(&glb_policy->pending_picks, pp);
       pick_done = false;
     } else {  // RR not in shutdown
       if (grpc_lb_glb_trace.enabled()) {
         gpr_log(GPR_INFO, "[grpclb %p] about to PICK from RR %p", glb_policy,
                 glb_policy->rr_policy);
       }
-      GRPC_LB_POLICY_REF(glb_policy->rr_policy, "glb_pick");
-      wrapped_rr_closure_arg* wc_arg =
-          (wrapped_rr_closure_arg*)gpr_zalloc(sizeof(wrapped_rr_closure_arg));
-      GRPC_CLOSURE_INIT(&wc_arg->wrapper_closure, wrapped_rr_closure, wc_arg,
-                        grpc_schedule_on_exec_ctx);
-      wc_arg->rr_policy = glb_policy->rr_policy;
-      wc_arg->target = target;
-      wc_arg->context = context;
       GPR_ASSERT(glb_policy->client_stats != nullptr);
-      wc_arg->client_stats =
-          grpc_grpclb_client_stats_ref(glb_policy->client_stats);
-      wc_arg->wrapped_closure = on_complete;
-      wc_arg->lb_token_mdelem_storage = pick_args->lb_token_mdelem_storage;
-      wc_arg->initial_metadata = pick_args->initial_metadata;
-      wc_arg->free_when_done = wc_arg;
-      wc_arg->glb_policy = pol;
-      pick_done = pick_from_internal_rr_locked(
-          glb_policy, pick_args, false /* force_async */, target, wc_arg);
+      pick_done =
+          pick_from_internal_rr_locked(glb_policy, false /* force_async */, pp);
     }
   } else {  // glb_policy->rr_policy == NULL
     if (grpc_lb_glb_trace.enabled()) {
@@ -1247,8 +1095,7 @@ static int glb_pick_locked(grpc_lb_policy* pol,
               "[grpclb %p] No RR policy. Adding to grpclb's pending picks",
               glb_policy);
     }
-    add_pending_pick(&glb_policy->pending_picks, pick_args, target, context,
-                     on_complete);
+    pending_pick_add(&glb_policy->pending_picks, pp);
     if (!glb_policy->started_picking) {
       start_picking_locked(glb_policy);
     }
@@ -1270,7 +1117,7 @@ static void glb_ping_one_locked(grpc_lb_policy* pol, grpc_closure* on_initiate,
   if (glb_policy->rr_policy) {
     grpc_lb_policy_ping_one_locked(glb_policy->rr_policy, on_initiate, on_ack);
   } else {
-    add_pending_ping(&glb_policy->pending_pings, on_initiate, on_ack);
+    pending_ping_add(&glb_policy->pending_pings, on_initiate, on_ack);
     if (!glb_policy->started_picking) {
       start_picking_locked(glb_policy);
     }
@@ -1295,7 +1142,7 @@ static void lb_call_on_retry_timer_locked(void* arg, grpc_error* error) {
     }
     query_for_backends_locked(glb_policy);
   }
-  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "grpclb_retry_timer");
+  GRPC_LB_POLICY_UNREF(&glb_policy->base, "grpclb_retry_timer");
 }
 
 static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
@@ -1321,7 +1168,7 @@ static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
                 glb_policy);
       }
     }
-    GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_retry_timer");
+    GRPC_LB_POLICY_REF(&glb_policy->base, "grpclb_retry_timer");
     GRPC_CLOSURE_INIT(&glb_policy->lb_on_call_retry,
                       lb_call_on_retry_timer_locked, glb_policy,
                       grpc_combiner_scheduler(glb_policy->base.combiner));
@@ -1329,8 +1176,8 @@ static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
     grpc_timer_init(&glb_policy->lb_call_retry_timer, next_try,
                     &glb_policy->lb_on_call_retry);
   }
-  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
-                            "lb_on_server_status_received_locked");
+  GRPC_LB_POLICY_UNREF(&glb_policy->base,
+                       "lb_on_server_status_received_locked");
 }
 
 static void send_client_load_report_locked(void* arg, grpc_error* error);
@@ -1353,7 +1200,7 @@ static void client_load_report_done_locked(void* arg, grpc_error* error) {
   glb_policy->client_load_report_payload = nullptr;
   if (error != GRPC_ERROR_NONE || glb_policy->lb_call == nullptr) {
     glb_policy->client_load_report_timer_callback_pending = false;
-    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "client_load_report");
+    GRPC_LB_POLICY_UNREF(&glb_policy->base, "client_load_report");
     if (glb_policy->lb_call == nullptr) {
       maybe_restart_lb_call(glb_policy);
     }
@@ -1394,7 +1241,7 @@ static void send_client_load_report_locked(void* arg, grpc_error* error) {
   glb_lb_policy* glb_policy = (glb_lb_policy*)arg;
   if (error == GRPC_ERROR_CANCELLED || glb_policy->lb_call == nullptr) {
     glb_policy->client_load_report_timer_callback_pending = false;
-    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "client_load_report");
+    GRPC_LB_POLICY_UNREF(&glb_policy->base, "client_load_report");
     if (glb_policy->lb_call == nullptr) {
       maybe_restart_lb_call(glb_policy);
     }
@@ -1547,10 +1394,8 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) {
   op->flags = 0;
   op->reserved = nullptr;
   op++;
-  /* take a weak ref (won't prevent calling of \a glb_shutdown if the strong ref
-   * count goes to zero) to be unref'd in lb_on_sent_initial_request_locked() */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base,
-                          "lb_on_sent_initial_request_locked");
+  /* take a ref to be released in lb_on_sent_initial_request_locked() */
+  GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_sent_initial_request_locked");
   call_error = grpc_call_start_batch_and_execute(
       glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_sent_initial_request);
@@ -1566,10 +1411,8 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) {
   op->flags = 0;
   op->reserved = nullptr;
   op++;
-  /* take a weak ref (won't prevent calling of \a glb_shutdown if the strong ref
-   * count goes to zero) to be unref'd in lb_on_server_status_received_locked */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base,
-                          "lb_on_server_status_received_locked");
+  /* take a ref to be released in lb_on_server_status_received_locked() */
+  GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_server_status_received_locked");
   call_error = grpc_call_start_batch_and_execute(
       glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_server_status_received);
@@ -1581,9 +1424,8 @@ static void query_for_backends_locked(glb_lb_policy* glb_policy) {
   op->flags = 0;
   op->reserved = nullptr;
   op++;
-  /* take another weak ref to be unref'd/reused in
-   * lb_on_response_received_locked */
-  GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "lb_on_response_received_locked");
+  /* take a ref to be unref'd/reused in lb_on_response_received_locked() */
+  GRPC_LB_POLICY_REF(&glb_policy->base, "lb_on_response_received_locked");
   call_error = grpc_call_start_batch_and_execute(
       glb_policy->lb_call, ops, (size_t)(op - ops),
       &glb_policy->lb_on_response_received);
@@ -1598,8 +1440,7 @@ static void lb_on_sent_initial_request_locked(void* arg, grpc_error* error) {
   if (glb_policy->client_load_report_payload != nullptr) {
     do_send_client_load_report_locked(glb_policy);
   }
-  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
-                            "lb_on_sent_initial_request_locked");
+  GRPC_LB_POLICY_UNREF(&glb_policy->base, "lb_on_sent_initial_request_locked");
 }
 
 static void lb_on_response_received_locked(void* arg, grpc_error* error) {
@@ -1631,11 +1472,9 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) {
                   "client load reporting interval = %" PRIdPTR " milliseconds",
                   glb_policy, glb_policy->client_stats_report_interval);
         }
-        /* take a weak ref (won't prevent calling of \a glb_shutdown() if the
-         * strong ref count goes to zero) to be unref'd in
-         * send_client_load_report_locked() */
+        /* take a ref to be unref'd in send_client_load_report_locked() */
         glb_policy->client_load_report_timer_callback_pending = true;
-        GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "client_load_report");
+        GRPC_LB_POLICY_REF(&glb_policy->base, "client_load_report");
         schedule_next_client_load_report(glb_policy);
       } else if (grpc_lb_glb_trace.enabled()) {
         gpr_log(GPR_INFO,
@@ -1717,21 +1556,21 @@ static void lb_on_response_received_locked(void* arg, grpc_error* error) {
       op->flags = 0;
       op->reserved = nullptr;
       op++;
-      /* reuse the "lb_on_response_received_locked" weak ref taken in
+      /* reuse the "lb_on_response_received_locked" ref taken in
        * query_for_backends_locked() */
       const grpc_call_error call_error = grpc_call_start_batch_and_execute(
           glb_policy->lb_call, ops, (size_t)(op - ops),
           &glb_policy->lb_on_response_received); /* loop */
       GPR_ASSERT(GRPC_CALL_OK == call_error);
     } else {
-      GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
-                                "lb_on_response_received_locked_shutdown");
+      GRPC_LB_POLICY_UNREF(&glb_policy->base,
+                           "lb_on_response_received_locked_shutdown");
     }
   } else { /* empty payload: call cancelled. */
-           /* dispose of the "lb_on_response_received_locked" weak ref taken in
+           /* dispose of the "lb_on_response_received_locked" ref taken in
             * query_for_backends_locked() and reused in every reception loop */
-    GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
-                              "lb_on_response_received_locked_empty_payload");
+    GRPC_LB_POLICY_UNREF(&glb_policy->base,
+                         "lb_on_response_received_locked_empty_payload");
   }
 }
 
@@ -1751,7 +1590,7 @@ static void lb_on_fallback_timer_locked(void* arg, grpc_error* error) {
       rr_handover_locked(glb_policy);
     }
   }
-  GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base, "grpclb_fallback_timer");
+  GRPC_LB_POLICY_UNREF(&glb_policy->base, "grpclb_fallback_timer");
 }
 
 static void lb_on_server_status_received_locked(void* arg, grpc_error* error) {
@@ -1835,7 +1674,7 @@ static void glb_update_locked(grpc_lb_policy* policy,
         grpc_channel_get_channel_stack(glb_policy->lb_channel));
     GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter);
     glb_policy->watching_lb_channel = true;
-    GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "watch_lb_channel_connectivity");
+    GRPC_LB_POLICY_REF(&glb_policy->base, "watch_lb_channel_connectivity");
     grpc_client_channel_watch_connectivity_state(
         client_channel_elem,
         grpc_polling_entity_create_from_pollset_set(
@@ -1891,8 +1730,8 @@ static void glb_lb_channel_on_connectivity_changed_cb(void* arg,
     case GRPC_CHANNEL_SHUTDOWN:
     done:
       glb_policy->watching_lb_channel = false;
-      GRPC_LB_POLICY_WEAK_UNREF(&glb_policy->base,
-                                "watch_lb_channel_connectivity_cb_shutdown");
+      GRPC_LB_POLICY_UNREF(&glb_policy->base,
+                           "watch_lb_channel_connectivity_cb_shutdown");
       break;
   }
 }
diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
index 9ff40aa53c0..60385272cf5 100644
--- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
@@ -31,15 +31,6 @@
 
 grpc_core::TraceFlag grpc_lb_pick_first_trace(false, "pick_first");
 
-namespace {
-struct pending_pick {
-  struct pending_pick* next;
-  uint32_t initial_metadata_flags;
-  grpc_connected_subchannel** target;
-  grpc_closure* on_complete;
-};
-}  // namespace
-
 typedef struct {
   /** base policy: must be first */
   grpc_lb_policy base;
@@ -54,7 +45,7 @@ typedef struct {
   /** are we shut down? */
   bool shutdown;
   /** list of picks that are waiting on connectivity */
-  pending_pick* pending_picks;
+  grpc_lb_policy_pick_state* pending_picks;
   /** our connectivity state tracker */
   grpc_connectivity_state_tracker state_tracker;
 } pick_first_lb_policy;
@@ -72,19 +63,27 @@ static void pf_destroy(grpc_lb_policy* pol) {
   }
 }
 
-static void pf_shutdown_locked(grpc_lb_policy* pol) {
+static void pf_shutdown_locked(grpc_lb_policy* pol,
+                               grpc_lb_policy* new_policy) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
   grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   if (grpc_lb_pick_first_trace.enabled()) {
     gpr_log(GPR_DEBUG, "Pick First %p Shutting down", p);
   }
   p->shutdown = true;
-  pending_pick* pp;
-  while ((pp = p->pending_picks) != nullptr) {
-    p->pending_picks = pp->next;
-    *pp->target = nullptr;
-    GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_REF(error));
-    gpr_free(pp);
+  grpc_lb_policy_pick_state* pick;
+  while ((pick = p->pending_picks) != nullptr) {
+    p->pending_picks = pick->next;
+    if (new_policy != nullptr) {
+      // Hand off to new LB policy.
+      if (grpc_lb_policy_pick_locked(new_policy, pick)) {
+        // Synchronous return, schedule closure.
+        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
+      }
+    } else {
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error));
+    }
   }
   grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
                               GRPC_ERROR_REF(error), "shutdown");
@@ -104,19 +103,18 @@ static void pf_shutdown_locked(grpc_lb_policy* pol) {
 }
 
 static void pf_cancel_pick_locked(grpc_lb_policy* pol,
-                                  grpc_connected_subchannel** target,
+                                  grpc_lb_policy_pick_state* pick,
                                   grpc_error* error) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
-  pending_pick* pp = p->pending_picks;
+  grpc_lb_policy_pick_state* pp = p->pending_picks;
   p->pending_picks = nullptr;
   while (pp != nullptr) {
-    pending_pick* next = pp->next;
-    if (pp->target == target) {
-      *target = nullptr;
-      GRPC_CLOSURE_SCHED(pp->on_complete,
+    grpc_lb_policy_pick_state* next = pp->next;
+    if (pp == pick) {
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(pick->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
-      gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
       p->pending_picks = pp;
@@ -131,21 +129,20 @@ static void pf_cancel_picks_locked(grpc_lb_policy* pol,
                                    uint32_t initial_metadata_flags_eq,
                                    grpc_error* error) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
-  pending_pick* pp = p->pending_picks;
+  grpc_lb_policy_pick_state* pick = p->pending_picks;
   p->pending_picks = nullptr;
-  while (pp != nullptr) {
-    pending_pick* next = pp->next;
-    if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
+  while (pick != nullptr) {
+    grpc_lb_policy_pick_state* next = pick->next;
+    if ((pick->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      GRPC_CLOSURE_SCHED(pp->on_complete,
+      GRPC_CLOSURE_SCHED(pick->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
-      gpr_free(pp);
     } else {
-      pp->next = p->pending_picks;
-      p->pending_picks = pp;
+      pick->next = p->pending_picks;
+      p->pending_picks = pick;
     }
-    pp = next;
+    pick = next;
   }
   GRPC_ERROR_UNREF(error);
 }
@@ -175,27 +172,20 @@ static void pf_exit_idle_locked(grpc_lb_policy* pol) {
 }
 
 static int pf_pick_locked(grpc_lb_policy* pol,
-                          const grpc_lb_policy_pick_args* pick_args,
-                          grpc_connected_subchannel** target,
-                          grpc_call_context_element* context, void** user_data,
-                          grpc_closure* on_complete) {
+                          grpc_lb_policy_pick_state* pick) {
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
   // If we have a selected subchannel already, return synchronously.
   if (p->selected != nullptr) {
-    *target = GRPC_CONNECTED_SUBCHANNEL_REF(p->selected->connected_subchannel,
-                                            "picked");
+    pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
+        p->selected->connected_subchannel, "picked");
     return 1;
   }
   // No subchannel selected yet, so handle asynchronously.
   if (!p->started_picking) {
     start_picking_locked(p);
   }
-  pending_pick* pp = (pending_pick*)gpr_malloc(sizeof(*pp));
-  pp->next = p->pending_picks;
-  pp->target = target;
-  pp->initial_metadata_flags = pick_args->initial_metadata_flags;
-  pp->on_complete = on_complete;
-  p->pending_picks = pp;
+  pick->next = p->pending_picks;
+  p->pending_picks = pick;
   return 0;
 }
 
@@ -481,18 +471,17 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
       // Drop all other subchannels, since we are now connected.
       destroy_unselected_subchannels_locked(p);
       // Update any calls that were waiting for a pick.
-      pending_pick* pp;
-      while ((pp = p->pending_picks)) {
-        p->pending_picks = pp->next;
-        *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF(
+      grpc_lb_policy_pick_state* pick;
+      while ((pick = p->pending_picks)) {
+        p->pending_picks = pick->next;
+        pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
             p->selected->connected_subchannel, "picked");
         if (grpc_lb_pick_first_trace.enabled()) {
           gpr_log(GPR_INFO,
                   "Servicing pending pick with selected subchannel %p",
                   (void*)p->selected);
         }
-        GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_NONE);
-        gpr_free(pp);
+        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
       }
       // Renew notification.
       grpc_lb_subchannel_data_start_connectivity_watch(sd);
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
index a964af06270..92c7d5bd5d0 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
@@ -41,31 +41,6 @@
 
 grpc_core::TraceFlag grpc_lb_round_robin_trace(false, "round_robin");
 
-namespace {
-/** List of entities waiting for a pick.
- *
- * Once a pick is available, \a target is updated and \a on_complete called. */
-struct pending_pick {
-  pending_pick* next;
-
-  /* output argument where to store the pick()ed user_data. It'll be NULL if no
-   * such data is present or there's an error (the definite test for errors is
-   * \a target being NULL). */
-  void** user_data;
-
-  /* bitmask passed to pick() and used for selective cancelling. See
-   * grpc_lb_policy_cancel_picks() */
-  uint32_t initial_metadata_flags;
-
-  /* output argument where to store the pick()ed connected subchannel, or NULL
-   * upon error. */
-  grpc_connected_subchannel** target;
-
-  /* to be invoked once the pick() has completed (regardless of success) */
-  grpc_closure* on_complete;
-};
-}  // namespace
-
 typedef struct round_robin_lb_policy {
   /** base policy: must be first */
   grpc_lb_policy base;
@@ -77,7 +52,7 @@ typedef struct round_robin_lb_policy {
   /** are we shutting down? */
   bool shutdown;
   /** List of picks that are waiting on connectivity */
-  pending_pick* pending_picks;
+  grpc_lb_policy_pick_state* pending_picks;
 
   /** our connectivity state tracker */
   grpc_connectivity_state_tracker state_tracker;
@@ -169,19 +144,27 @@ static void rr_destroy(grpc_lb_policy* pol) {
   gpr_free(p);
 }
 
-static void rr_shutdown_locked(grpc_lb_policy* pol) {
+static void rr_shutdown_locked(grpc_lb_policy* pol,
+                               grpc_lb_policy* new_policy) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
   grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   if (grpc_lb_round_robin_trace.enabled()) {
     gpr_log(GPR_DEBUG, "[RR %p] Shutting down", p);
   }
   p->shutdown = true;
-  pending_pick* pp;
-  while ((pp = p->pending_picks) != nullptr) {
-    p->pending_picks = pp->next;
-    *pp->target = nullptr;
-    GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_REF(error));
-    gpr_free(pp);
+  grpc_lb_policy_pick_state* pick;
+  while ((pick = p->pending_picks) != nullptr) {
+    p->pending_picks = pick->next;
+    if (new_policy != nullptr) {
+      // Hand off to new LB policy.
+      if (grpc_lb_policy_pick_locked(new_policy, pick)) {
+        // Synchronous return; schedule callback.
+        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
+      }
+    } else {
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error));
+    }
   }
   grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
                               GRPC_ERROR_REF(error), "rr_shutdown");
@@ -201,19 +184,18 @@ static void rr_shutdown_locked(grpc_lb_policy* pol) {
 }
 
 static void rr_cancel_pick_locked(grpc_lb_policy* pol,
-                                  grpc_connected_subchannel** target,
+                                  grpc_lb_policy_pick_state* pick,
                                   grpc_error* error) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
-  pending_pick* pp = p->pending_picks;
+  grpc_lb_policy_pick_state* pp = p->pending_picks;
   p->pending_picks = nullptr;
   while (pp != nullptr) {
-    pending_pick* next = pp->next;
-    if (pp->target == target) {
-      *target = nullptr;
-      GRPC_CLOSURE_SCHED(pp->on_complete,
+    grpc_lb_policy_pick_state* next = pp->next;
+    if (pp == pick) {
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(pick->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick cancelled", &error, 1));
-      gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
       p->pending_picks = pp;
@@ -228,22 +210,21 @@ static void rr_cancel_picks_locked(grpc_lb_policy* pol,
                                    uint32_t initial_metadata_flags_eq,
                                    grpc_error* error) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
-  pending_pick* pp = p->pending_picks;
+  grpc_lb_policy_pick_state* pick = p->pending_picks;
   p->pending_picks = nullptr;
-  while (pp != nullptr) {
-    pending_pick* next = pp->next;
-    if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
+  while (pick != nullptr) {
+    grpc_lb_policy_pick_state* next = pick->next;
+    if ((pick->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      *pp->target = nullptr;
-      GRPC_CLOSURE_SCHED(pp->on_complete,
+      pick->connected_subchannel = nullptr;
+      GRPC_CLOSURE_SCHED(pick->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick cancelled", &error, 1));
-      gpr_free(pp);
     } else {
-      pp->next = p->pending_picks;
-      p->pending_picks = pp;
+      pick->next = p->pending_picks;
+      p->pending_picks = pick;
     }
-    pp = next;
+    pick = next;
   }
   GRPC_ERROR_UNREF(error);
 }
@@ -268,13 +249,10 @@ static void rr_exit_idle_locked(grpc_lb_policy* pol) {
 }
 
 static int rr_pick_locked(grpc_lb_policy* pol,
-                          const grpc_lb_policy_pick_args* pick_args,
-                          grpc_connected_subchannel** target,
-                          grpc_call_context_element* context, void** user_data,
-                          grpc_closure* on_complete) {
+                          grpc_lb_policy_pick_state* pick) {
   round_robin_lb_policy* p = (round_robin_lb_policy*)pol;
   if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", (void*)pol,
+    gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", pol,
             p->shutdown);
   }
   GPR_ASSERT(!p->shutdown);
@@ -284,18 +262,18 @@ static int rr_pick_locked(grpc_lb_policy* pol,
       /* readily available, report right away */
       grpc_lb_subchannel_data* sd =
           &p->subchannel_list->subchannels[next_ready_index];
-      *target =
+      pick->connected_subchannel =
           GRPC_CONNECTED_SUBCHANNEL_REF(sd->connected_subchannel, "rr_picked");
-      if (user_data != nullptr) {
-        *user_data = sd->user_data;
+      if (pick->user_data != nullptr) {
+        *pick->user_data = sd->user_data;
       }
       if (grpc_lb_round_robin_trace.enabled()) {
         gpr_log(
             GPR_DEBUG,
             "[RR %p] Picked target <-- Subchannel %p (connected %p) (sl %p, "
-            "index %lu)",
-            (void*)p, (void*)sd->subchannel, (void*)*target,
-            (void*)sd->subchannel_list, (unsigned long)next_ready_index);
+            "index %" PRIuPTR ")",
+            p, sd->subchannel, pick->connected_subchannel, sd->subchannel_list,
+            next_ready_index);
       }
       /* only advance the last picked pointer if the selection was used */
       update_last_ready_subchannel_index_locked(p, next_ready_index);
@@ -306,13 +284,8 @@ static int rr_pick_locked(grpc_lb_policy* pol,
   if (!p->started_picking) {
     start_picking_locked(p);
   }
-  pending_pick* pp = (pending_pick*)gpr_malloc(sizeof(*pp));
-  pp->next = p->pending_picks;
-  pp->target = target;
-  pp->on_complete = on_complete;
-  pp->initial_metadata_flags = pick_args->initial_metadata_flags;
-  pp->user_data = user_data;
-  p->pending_picks = pp;
+  pick->next = p->pending_picks;
+  p->pending_picks = pick;
   return 0;
 }
 
@@ -495,13 +468,13 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
         // picks, update the last picked pointer
         update_last_ready_subchannel_index_locked(p, next_ready_index);
       }
-      pending_pick* pp;
-      while ((pp = p->pending_picks)) {
-        p->pending_picks = pp->next;
-        *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF(
+      grpc_lb_policy_pick_state* pick;
+      while ((pick = p->pending_picks)) {
+        p->pending_picks = pick->next;
+        pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
             selected->connected_subchannel, "rr_picked");
-        if (pp->user_data != nullptr) {
-          *pp->user_data = selected->user_data;
+        if (pick->user_data != nullptr) {
+          *pick->user_data = selected->user_data;
         }
         if (grpc_lb_round_robin_trace.enabled()) {
           gpr_log(GPR_DEBUG,
@@ -510,8 +483,7 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
                   (void*)p, (void*)selected->subchannel,
                   (void*)p->subchannel_list, (unsigned long)next_ready_index);
         }
-        GRPC_CLOSURE_SCHED(pp->on_complete, GRPC_ERROR_NONE);
-        gpr_free(pp);
+        GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
       }
     }
     // Renew notification.
diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
index a3b4c8e524d..5ce1298afc4 100644
--- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
@@ -213,13 +213,13 @@ void grpc_lb_subchannel_list_unref(grpc_lb_subchannel_list* subchannel_list,
 
 void grpc_lb_subchannel_list_ref_for_connectivity_watch(
     grpc_lb_subchannel_list* subchannel_list, const char* reason) {
-  GRPC_LB_POLICY_WEAK_REF(subchannel_list->policy, reason);
+  GRPC_LB_POLICY_REF(subchannel_list->policy, reason);
   grpc_lb_subchannel_list_ref(subchannel_list, reason);
 }
 
 void grpc_lb_subchannel_list_unref_for_connectivity_watch(
     grpc_lb_subchannel_list* subchannel_list, const char* reason) {
-  GRPC_LB_POLICY_WEAK_UNREF(subchannel_list->policy, reason);
+  GRPC_LB_POLICY_UNREF(subchannel_list->policy, reason);
   grpc_lb_subchannel_list_unref(subchannel_list, reason);
 }
 

From 83d5cd602a7a4469ff6aeb72903db488b8d34d0b Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Thu, 11 Jan 2018 08:56:53 -0800
Subject: [PATCH 077/161] Don't unref null client_stats.

---
 .../ext/filters/client_channel/lb_policy/grpclb/grpclb.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index 5849ac9d2da..272b3617b25 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -340,7 +340,9 @@ static void pending_pick_set_metadata_and_context(pending_pick* pp) {
     pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].destroy =
         destroy_client_stats;
   } else {
-    grpc_grpclb_client_stats_unref(pp->client_stats);
+    if (pp->client_stats != nullptr) {
+      grpc_grpclb_client_stats_unref(pp->client_stats);
+    }
   }
 }
 
@@ -927,7 +929,9 @@ static void glb_shutdown_locked(grpc_lb_policy* pol,
     pending_pick* next = pp->next;
     if (new_policy != nullptr) {
       // Hand pick over to new policy.
-      grpc_grpclb_client_stats_unref(pp->client_stats);
+      if (pp->client_stats != nullptr) {
+        grpc_grpclb_client_stats_unref(pp->client_stats);
+      }
       pp->pick->on_complete = pp->original_on_complete;
       if (grpc_lb_policy_pick_locked(new_policy, pp->pick)) {
         // Synchronous return; schedule callback.

From f1b933ce7fe36d150d21f03ad60a88c63a92b350 Mon Sep 17 00:00:00 2001
From: Alexander Polcyn <apolcyn@google.com>
Date: Thu, 11 Jan 2018 09:47:18 -0800
Subject: [PATCH 078/161] Avoid building c-ares lib when grpc_no_ares is true

---
 bazel/grpc_build_system.bzl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
index d146ca9c2ca..9a3b7613445 100644
--- a/bazel/grpc_build_system.bzl
+++ b/bazel/grpc_build_system.bzl
@@ -31,6 +31,9 @@ def _get_external_deps(external_deps):
   for dep in external_deps:
     if dep == "nanopb":
       ret.append("//third_party/nanopb")
+    elif dep == "cares":
+      ret += select({"//:grpc_no_ares": [],
+                     "//conditions:default": ["//external:cares"],})
     else:
       ret.append("//external:" + dep)
   return ret

From 3742b724c9deac411b51dfc7b1ac6818f61550e3 Mon Sep 17 00:00:00 2001
From: Dan Zhang <danzh@google.com>
Date: Thu, 11 Jan 2018 13:02:38 -0500
Subject: [PATCH 079/161] change to int type

---
 src/core/lib/iomgr/udp_server.cc   | 6 +++---
 src/core/lib/iomgr/udp_server.h    | 2 +-
 test/core/iomgr/udp_server_test.cc | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/core/lib/iomgr/udp_server.cc b/src/core/lib/iomgr/udp_server.cc
index 2df10ee09a9..a9a3f3aba51 100644
--- a/src/core/lib/iomgr/udp_server.cc
+++ b/src/core/lib/iomgr/udp_server.cc
@@ -285,7 +285,7 @@ static int bind_socket(grpc_socket_factory* socket_factory, int sockfd,
 /* Prepare a recently-created socket for listening. */
 static int prepare_socket(grpc_socket_factory* socket_factory, int fd,
                           const grpc_resolved_address* addr,
-                          size_t rcv_buf_size, size_t snd_buf_size) {
+                          int rcv_buf_size, int snd_buf_size) {
   grpc_resolved_address sockname_temp;
   struct sockaddr* addr_ptr = (struct sockaddr*)addr->addr;
 
@@ -461,7 +461,7 @@ static void on_write(void* arg, grpc_error* error) {
 
 static int add_socket_to_server(grpc_udp_server* s, int fd,
                                 const grpc_resolved_address* addr,
-                                size_t rcv_buf_size, size_t snd_buf_size,
+                                int rcv_buf_size, int snd_buf_size,
                                 grpc_udp_server_start_cb start_cb,
                                 grpc_udp_server_read_cb read_cb,
                                 grpc_udp_server_write_cb write_cb,
@@ -507,7 +507,7 @@ static int add_socket_to_server(grpc_udp_server* s, int fd,
 
 int grpc_udp_server_add_port(grpc_udp_server* s,
                              const grpc_resolved_address* addr,
-                             size_t rcv_buf_size, size_t snd_buf_size,
+                             int rcv_buf_size, int snd_buf_size,
                              grpc_udp_server_start_cb start_cb,
                              grpc_udp_server_read_cb read_cb,
                              grpc_udp_server_write_cb write_cb,
diff --git a/src/core/lib/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h
index ec18716f396..c1aa49f15dd 100644
--- a/src/core/lib/iomgr/udp_server.h
+++ b/src/core/lib/iomgr/udp_server.h
@@ -68,7 +68,7 @@ int grpc_udp_server_get_fd(grpc_udp_server* s, unsigned port_index);
                   all of the multiple socket port matching logic in one place */
 int grpc_udp_server_add_port(grpc_udp_server* s,
                              const grpc_resolved_address* addr,
-                             size_t rcv_buf_size, size_t snd_buf_size,
+                             int rcv_buf_size, int snd_buf_size,
                              grpc_udp_server_start_cb start_cb,
                              grpc_udp_server_read_cb read_cb,
                              grpc_udp_server_write_cb write_cb,
diff --git a/test/core/iomgr/udp_server_test.cc b/test/core/iomgr/udp_server_test.cc
index 5245840ba95..09f02830138 100644
--- a/test/core/iomgr/udp_server_test.cc
+++ b/test/core/iomgr/udp_server_test.cc
@@ -51,8 +51,8 @@ static int g_number_of_bytes_read = 0;
 static int g_number_of_orphan_calls = 0;
 static int g_number_of_starts = 0;
 
-size_t rcv_buf_size = 1024;
-size_t snd_buf_size = 1024;
+int rcv_buf_size = 1024;
+int snd_buf_size = 1024;
 
 static void on_start(grpc_fd* emfd, void* user_data) { g_number_of_starts++; }
 

From e2a6ca835c5163075a11f0c1e78fb98378298eb7 Mon Sep 17 00:00:00 2001
From: Dan Zhang <danzh@google.com>
Date: Thu, 11 Jan 2018 13:06:38 -0500
Subject: [PATCH 080/161] %zd->%d

---
 src/core/lib/iomgr/udp_server.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/core/lib/iomgr/udp_server.cc b/src/core/lib/iomgr/udp_server.cc
index a9a3f3aba51..ec3b88d9118 100644
--- a/src/core/lib/iomgr/udp_server.cc
+++ b/src/core/lib/iomgr/udp_server.cc
@@ -329,13 +329,13 @@ static int prepare_socket(grpc_socket_factory* socket_factory, int fd,
   }
 
   if (grpc_set_socket_sndbuf(fd, snd_buf_size) != GRPC_ERROR_NONE) {
-    gpr_log(GPR_ERROR, "Failed to set send buffer size to %zd bytes",
+    gpr_log(GPR_ERROR, "Failed to set send buffer size to %d bytes",
             snd_buf_size);
     goto error;
   }
 
   if (grpc_set_socket_rcvbuf(fd, rcv_buf_size) != GRPC_ERROR_NONE) {
-    gpr_log(GPR_ERROR, "Failed to set receive buffer size to %zd bytes",
+    gpr_log(GPR_ERROR, "Failed to set receive buffer size to %d bytes",
             rcv_buf_size);
     goto error;
   }

From a06720bab85239e34bc42caefd6191284061a311 Mon Sep 17 00:00:00 2001
From: Dan Zhang <danzh@google.com>
Date: Thu, 11 Jan 2018 13:20:57 -0500
Subject: [PATCH 081/161] format

---
 src/core/lib/iomgr/udp_server.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/core/lib/iomgr/udp_server.cc b/src/core/lib/iomgr/udp_server.cc
index ec3b88d9118..946846a8b87 100644
--- a/src/core/lib/iomgr/udp_server.cc
+++ b/src/core/lib/iomgr/udp_server.cc
@@ -284,8 +284,8 @@ static int bind_socket(grpc_socket_factory* socket_factory, int sockfd,
 
 /* Prepare a recently-created socket for listening. */
 static int prepare_socket(grpc_socket_factory* socket_factory, int fd,
-                          const grpc_resolved_address* addr,
-                          int rcv_buf_size, int snd_buf_size) {
+                          const grpc_resolved_address* addr, int rcv_buf_size,
+                          int snd_buf_size) {
   grpc_resolved_address sockname_temp;
   struct sockaddr* addr_ptr = (struct sockaddr*)addr->addr;
 

From 294e996a8d6fb6603134d2edb3b279eea6dcd503 Mon Sep 17 00:00:00 2001
From: Vijay Pai <vpai@google.com>
Date: Thu, 11 Jan 2018 11:40:52 -0800
Subject: [PATCH 082/161] Add me to build system owners

---
 .github/CODEOWNERS | 2 +-
 bazel/OWNERS       | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 093e9396cf4..d45d5454544 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -2,6 +2,6 @@
 # Uses OWNERS files in different modules throughout the
 # repository as the source of truth for module ownership.
 /**/OWNERS @markdroth @nicolasnoble @a11r
-/bazel/** @nicolasnoble @dgquintas @a11r
+/bazel/** @nicolasnoble @dgquintas @a11r @vjpai
 /src/core/ext/filters/client_channel/** @markdroth @dgquintas @a11r
 /tools/run_tests/performance/** @ncteisen @matt-kwong @ctiller
diff --git a/bazel/OWNERS b/bazel/OWNERS
index f6ad849be30..613ba36be87 100644
--- a/bazel/OWNERS
+++ b/bazel/OWNERS
@@ -2,4 +2,5 @@ set noparent
 @nicolasnoble
 @dgquintas
 @a11r
+@vjpai
 

From 07fb6b521836e27d684361238faddc9fb1b5002a Mon Sep 17 00:00:00 2001
From: Kun Zhang <zhangkun@google.com>
Date: Thu, 11 Jan 2018 14:27:25 -0800
Subject: [PATCH 083/161] Add Java 1.9.0 release to client_matrix.py

---
 tools/interop_matrix/client_matrix.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py
index 71d3a79023a..7dbd07a2582 100644
--- a/tools/interop_matrix/client_matrix.py
+++ b/tools/interop_matrix/client_matrix.py
@@ -132,6 +132,9 @@ LANG_RELEASE_MATRIX = {
         {
             'v1.8.0': None
         },
+        {
+            'v1.9.0': None
+        },
     ],
     'python': [
         {

From 951f84aea00a1f8a65cf160d7d8f342c30593000 Mon Sep 17 00:00:00 2001
From: "David G. Quintas" <dgq@google.com>
Date: Thu, 11 Jan 2018 15:32:34 -0800
Subject: [PATCH 084/161] Revert "Set error status correctly on server side"

---
 CMakeLists.txt                                |   2 -
 Makefile                                      |   2 -
 gRPC-Core.podspec                             |   1 -
 grpc.gyp                                      |   2 -
 src/core/lib/surface/call.cc                  |   5 +-
 test/core/end2end/end2end_nosec_tests.cc      |   8 -
 test/core/end2end/end2end_tests.cc            |   8 -
 test/core/end2end/gen_build_yaml.py           |   1 -
 test/core/end2end/generate_tests.bzl          |   1 -
 test/core/end2end/tests/filter_status_code.cc | 353 --------
 .../generated/sources_and_headers.json        |   2 -
 tools/run_tests/generated/tests.json          | 839 +-----------------
 12 files changed, 27 insertions(+), 1197 deletions(-)
 delete mode 100644 test/core/end2end/tests/filter_status_code.cc

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 78ccfb21329..deca1b3f752 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4609,7 +4609,6 @@ add_library(end2end_tests
   test/core/end2end/tests/filter_call_init_fails.cc
   test/core/end2end/tests/filter_causes_close.cc
   test/core/end2end/tests/filter_latency.cc
-  test/core/end2end/tests/filter_status_code.cc
   test/core/end2end/tests/graceful_server_shutdown.cc
   test/core/end2end/tests/high_initial_seqno.cc
   test/core/end2end/tests/hpack_size.cc
@@ -4711,7 +4710,6 @@ add_library(end2end_nosec_tests
   test/core/end2end/tests/filter_call_init_fails.cc
   test/core/end2end/tests/filter_causes_close.cc
   test/core/end2end/tests/filter_latency.cc
-  test/core/end2end/tests/filter_status_code.cc
   test/core/end2end/tests/graceful_server_shutdown.cc
   test/core/end2end/tests/high_initial_seqno.cc
   test/core/end2end/tests/hpack_size.cc
diff --git a/Makefile b/Makefile
index f50163efdce..499aca56f65 100644
--- a/Makefile
+++ b/Makefile
@@ -8552,7 +8552,6 @@ LIBEND2END_TESTS_SRC = \
     test/core/end2end/tests/filter_call_init_fails.cc \
     test/core/end2end/tests/filter_causes_close.cc \
     test/core/end2end/tests/filter_latency.cc \
-    test/core/end2end/tests/filter_status_code.cc \
     test/core/end2end/tests/graceful_server_shutdown.cc \
     test/core/end2end/tests/high_initial_seqno.cc \
     test/core/end2end/tests/hpack_size.cc \
@@ -8651,7 +8650,6 @@ LIBEND2END_NOSEC_TESTS_SRC = \
     test/core/end2end/tests/filter_call_init_fails.cc \
     test/core/end2end/tests/filter_causes_close.cc \
     test/core/end2end/tests/filter_latency.cc \
-    test/core/end2end/tests/filter_status_code.cc \
     test/core/end2end/tests/graceful_server_shutdown.cc \
     test/core/end2end/tests/high_initial_seqno.cc \
     test/core/end2end/tests/hpack_size.cc \
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index 358fad3d983..fcabd62c85a 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -1039,7 +1039,6 @@ Pod::Spec.new do |s|
                       'test/core/end2end/tests/filter_call_init_fails.cc',
                       'test/core/end2end/tests/filter_causes_close.cc',
                       'test/core/end2end/tests/filter_latency.cc',
-                      'test/core/end2end/tests/filter_status_code.cc',
                       'test/core/end2end/tests/graceful_server_shutdown.cc',
                       'test/core/end2end/tests/high_initial_seqno.cc',
                       'test/core/end2end/tests/hpack_size.cc',
diff --git a/grpc.gyp b/grpc.gyp
index 281fbfa8a6e..06da3a758fb 100644
--- a/grpc.gyp
+++ b/grpc.gyp
@@ -2381,7 +2381,6 @@
         'test/core/end2end/tests/filter_call_init_fails.cc',
         'test/core/end2end/tests/filter_causes_close.cc',
         'test/core/end2end/tests/filter_latency.cc',
-        'test/core/end2end/tests/filter_status_code.cc',
         'test/core/end2end/tests/graceful_server_shutdown.cc',
         'test/core/end2end/tests/high_initial_seqno.cc',
         'test/core/end2end/tests/hpack_size.cc',
@@ -2454,7 +2453,6 @@
         'test/core/end2end/tests/filter_call_init_fails.cc',
         'test/core/end2end/tests/filter_causes_close.cc',
         'test/core/end2end/tests/filter_latency.cc',
-        'test/core/end2end/tests/filter_status_code.cc',
         'test/core/end2end/tests/graceful_server_shutdown.cc',
         'test/core/end2end/tests/high_initial_seqno.cc',
         'test/core/end2end/tests/hpack_size.cc',
diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc
index d677576c145..a457aaa7a2f 100644
--- a/src/core/lib/surface/call.cc
+++ b/src/core/lib/surface/call.cc
@@ -1851,9 +1851,8 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
         {
           grpc_error* override_error = GRPC_ERROR_NONE;
           if (op->data.send_status_from_server.status != GRPC_STATUS_OK) {
-            override_error =
-                error_from_status(op->data.send_status_from_server.status,
-                                  "Returned non-ok status");
+            override_error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                "Error from server send status");
           }
           if (op->data.send_status_from_server.status_details != nullptr) {
             call->send_extra_metadata[1].md = grpc_mdelem_from_slices(
diff --git a/test/core/end2end/end2end_nosec_tests.cc b/test/core/end2end/end2end_nosec_tests.cc
index 6318550ad86..3236feea56e 100644
--- a/test/core/end2end/end2end_nosec_tests.cc
+++ b/test/core/end2end/end2end_nosec_tests.cc
@@ -68,8 +68,6 @@ extern void filter_causes_close(grpc_end2end_test_config config);
 extern void filter_causes_close_pre_init(void);
 extern void filter_latency(grpc_end2end_test_config config);
 extern void filter_latency_pre_init(void);
-extern void filter_status_code(grpc_end2end_test_config config);
-extern void filter_status_code_pre_init(void);
 extern void graceful_server_shutdown(grpc_end2end_test_config config);
 extern void graceful_server_shutdown_pre_init(void);
 extern void high_initial_seqno(grpc_end2end_test_config config);
@@ -172,7 +170,6 @@ void grpc_end2end_tests_pre_init(void) {
   filter_call_init_fails_pre_init();
   filter_causes_close_pre_init();
   filter_latency_pre_init();
-  filter_status_code_pre_init();
   graceful_server_shutdown_pre_init();
   high_initial_seqno_pre_init();
   hpack_size_pre_init();
@@ -240,7 +237,6 @@ void grpc_end2end_tests(int argc, char **argv,
     filter_call_init_fails(config);
     filter_causes_close(config);
     filter_latency(config);
-    filter_status_code(config);
     graceful_server_shutdown(config);
     high_initial_seqno(config);
     hpack_size(config);
@@ -360,10 +356,6 @@ void grpc_end2end_tests(int argc, char **argv,
       filter_latency(config);
       continue;
     }
-    if (0 == strcmp("filter_status_code", argv[i])) {
-      filter_status_code(config);
-      continue;
-    }
     if (0 == strcmp("graceful_server_shutdown", argv[i])) {
       graceful_server_shutdown(config);
       continue;
diff --git a/test/core/end2end/end2end_tests.cc b/test/core/end2end/end2end_tests.cc
index 9d8dfd67238..ca9443b642d 100644
--- a/test/core/end2end/end2end_tests.cc
+++ b/test/core/end2end/end2end_tests.cc
@@ -70,8 +70,6 @@ extern void filter_causes_close(grpc_end2end_test_config config);
 extern void filter_causes_close_pre_init(void);
 extern void filter_latency(grpc_end2end_test_config config);
 extern void filter_latency_pre_init(void);
-extern void filter_status_code(grpc_end2end_test_config config);
-extern void filter_status_code_pre_init(void);
 extern void graceful_server_shutdown(grpc_end2end_test_config config);
 extern void graceful_server_shutdown_pre_init(void);
 extern void high_initial_seqno(grpc_end2end_test_config config);
@@ -175,7 +173,6 @@ void grpc_end2end_tests_pre_init(void) {
   filter_call_init_fails_pre_init();
   filter_causes_close_pre_init();
   filter_latency_pre_init();
-  filter_status_code_pre_init();
   graceful_server_shutdown_pre_init();
   high_initial_seqno_pre_init();
   hpack_size_pre_init();
@@ -244,7 +241,6 @@ void grpc_end2end_tests(int argc, char **argv,
     filter_call_init_fails(config);
     filter_causes_close(config);
     filter_latency(config);
-    filter_status_code(config);
     graceful_server_shutdown(config);
     high_initial_seqno(config);
     hpack_size(config);
@@ -368,10 +364,6 @@ void grpc_end2end_tests(int argc, char **argv,
       filter_latency(config);
       continue;
     }
-    if (0 == strcmp("filter_status_code", argv[i])) {
-      filter_status_code(config);
-      continue;
-    }
     if (0 == strcmp("graceful_server_shutdown", argv[i])) {
       graceful_server_shutdown(config);
       continue;
diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py
index e7cf97b2d02..7c8e7f420ae 100755
--- a/test/core/end2end/gen_build_yaml.py
+++ b/test/core/end2end/gen_build_yaml.py
@@ -101,7 +101,6 @@ END2END_TESTS = {
     'filter_causes_close': default_test_options._replace(cpu_cost=LOWCPU),
     'filter_call_init_fails': default_test_options,
     'filter_latency': default_test_options._replace(cpu_cost=LOWCPU),
-    'filter_status_code': default_test_options._replace(cpu_cost=LOWCPU),
     'graceful_server_shutdown': default_test_options._replace(cpu_cost=LOWCPU,exclude_inproc=True),
     'hpack_size': default_test_options._replace(proxyable=False,
                                                 traceable=False,
diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl
index 1d759e1ecbf..b9a42bdb885 100755
--- a/test/core/end2end/generate_tests.bzl
+++ b/test/core/end2end/generate_tests.bzl
@@ -146,7 +146,6 @@ END2END_TESTS = {
     'trailing_metadata': test_options(),
     'authority_not_supported': test_options(),
     'filter_latency': test_options(),
-    'filter_status_code': test_options(),
     'workaround_cronet_compression': test_options(),
     'write_buffering': test_options(needs_write_buffering=True),
     'write_buffering_at_end': test_options(needs_write_buffering=True),
diff --git a/test/core/end2end/tests/filter_status_code.cc b/test/core/end2end/tests/filter_status_code.cc
deleted file mode 100644
index 261ddd93ec5..00000000000
--- a/test/core/end2end/tests/filter_status_code.cc
+++ /dev/null
@@ -1,353 +0,0 @@
-/*
- *
- * Copyright 2017 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "test/core/end2end/end2end_tests.h"
-
-#include <limits.h>
-#include <stdbool.h>
-#include <stdio.h>
-#include <string.h>
-
-#include <grpc/byte_buffer.h>
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/time.h>
-#include <grpc/support/useful.h>
-
-#include "src/core/lib/channel/channel_stack_builder.h"
-#include "src/core/lib/surface/channel_init.h"
-#include "test/core/end2end/cq_verifier.h"
-
-static bool g_enable_filter = false;
-static gpr_mu g_mu;
-static bool g_client_code_recv;
-static bool g_server_code_recv;
-static gpr_cv g_client_code_cv;
-static gpr_cv g_server_code_cv;
-static grpc_status_code g_client_status_code;
-static grpc_status_code g_server_status_code;
-
-static void* tag(intptr_t t) { return (void*)t; }
-
-static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config,
-                                            const char* test_name,
-                                            grpc_channel_args* client_args,
-                                            grpc_channel_args* server_args) {
-  grpc_end2end_test_fixture f;
-  gpr_log(GPR_INFO, "Running test: %s/%s", test_name, config.name);
-  f = config.create_fixture(client_args, server_args);
-  config.init_server(&f, server_args);
-  config.init_client(&f, client_args);
-  return f;
-}
-
-static gpr_timespec n_seconds_from_now(int n) {
-  return grpc_timeout_seconds_to_deadline(n);
-}
-
-static gpr_timespec five_seconds_from_now(void) {
-  return n_seconds_from_now(5);
-}
-
-static void drain_cq(grpc_completion_queue* cq) {
-  grpc_event ev;
-  do {
-    ev = grpc_completion_queue_next(cq, five_seconds_from_now(), nullptr);
-  } while (ev.type != GRPC_QUEUE_SHUTDOWN);
-}
-
-static void shutdown_server(grpc_end2end_test_fixture* f) {
-  if (!f->server) return;
-  grpc_server_shutdown_and_notify(f->server, f->shutdown_cq, tag(1000));
-  GPR_ASSERT(grpc_completion_queue_pluck(f->shutdown_cq, tag(1000),
-                                         grpc_timeout_seconds_to_deadline(5),
-                                         nullptr)
-                 .type == GRPC_OP_COMPLETE);
-  grpc_server_destroy(f->server);
-  f->server = nullptr;
-}
-
-static void shutdown_client(grpc_end2end_test_fixture* f) {
-  if (!f->client) return;
-  grpc_channel_destroy(f->client);
-  f->client = nullptr;
-}
-
-static void end_test(grpc_end2end_test_fixture* f) {
-  shutdown_server(f);
-  shutdown_client(f);
-
-  grpc_completion_queue_shutdown(f->cq);
-  drain_cq(f->cq);
-  grpc_completion_queue_destroy(f->cq);
-  grpc_completion_queue_destroy(f->shutdown_cq);
-}
-
-// Simple request via a server filter that saves the reported status code.
-static void test_request(grpc_end2end_test_config config) {
-  grpc_call* c;
-  grpc_call* s;
-  grpc_end2end_test_fixture f =
-      begin_test(config, "filter_status_code", nullptr, nullptr);
-  cq_verifier* cqv = cq_verifier_create(f.cq);
-  grpc_op ops[6];
-  grpc_op* op;
-  grpc_metadata_array initial_metadata_recv;
-  grpc_metadata_array trailing_metadata_recv;
-  grpc_metadata_array request_metadata_recv;
-  grpc_call_details call_details;
-  grpc_status_code status;
-  grpc_call_error error;
-  grpc_slice details;
-  int was_cancelled = 2;
-
-  gpr_mu_lock(&g_mu);
-  g_client_status_code = GRPC_STATUS_OK;
-  g_server_status_code = GRPC_STATUS_OK;
-  gpr_mu_unlock(&g_mu);
-
-  gpr_timespec deadline = five_seconds_from_now();
-  c = grpc_channel_create_call(
-      f.client, nullptr, GRPC_PROPAGATE_DEFAULTS, f.cq,
-      grpc_slice_from_static_string("/foo"),
-      get_host_override_slice("foo.test.google.fr", config), deadline, nullptr);
-  GPR_ASSERT(c);
-
-  grpc_metadata_array_init(&initial_metadata_recv);
-  grpc_metadata_array_init(&trailing_metadata_recv);
-  grpc_metadata_array_init(&request_metadata_recv);
-  grpc_call_details_init(&call_details);
-
-  memset(ops, 0, sizeof(ops));
-  op = ops;
-  op->op = GRPC_OP_SEND_INITIAL_METADATA;
-  op->data.send_initial_metadata.count = 0;
-  op->data.send_initial_metadata.metadata = nullptr;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  op->op = GRPC_OP_RECV_INITIAL_METADATA;
-  op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
-  op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
-  op->data.recv_status_on_client.status = &status;
-  op->data.recv_status_on_client.status_details = &details;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), nullptr);
-  GPR_ASSERT(GRPC_CALL_OK == error);
-
-  error =
-      grpc_server_request_call(f.server, &s, &call_details,
-                               &request_metadata_recv, f.cq, f.cq, tag(101));
-  GPR_ASSERT(GRPC_CALL_OK == error);
-
-  CQ_EXPECT_COMPLETION(cqv, tag(101), 1);
-  cq_verify(cqv);
-
-  memset(ops, 0, sizeof(ops));
-  op = ops;
-  op->op = GRPC_OP_SEND_INITIAL_METADATA;
-  op->data.send_initial_metadata.count = 0;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
-  op->data.send_status_from_server.trailing_metadata_count = 0;
-  op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED;
-  grpc_slice status_string = grpc_slice_from_static_string("xyz");
-  op->data.send_status_from_server.status_details = &status_string;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
-  op->data.recv_close_on_server.cancelled = &was_cancelled;
-  op->flags = 0;
-  op->reserved = nullptr;
-  op++;
-  error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), nullptr);
-  GPR_ASSERT(GRPC_CALL_OK == error);
-
-  CQ_EXPECT_COMPLETION(cqv, tag(102), 1);
-  CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
-  cq_verify(cqv);
-
-  GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED);
-  GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz"));
-
-  grpc_slice_unref(details);
-  grpc_metadata_array_destroy(&initial_metadata_recv);
-  grpc_metadata_array_destroy(&trailing_metadata_recv);
-  grpc_metadata_array_destroy(&request_metadata_recv);
-  grpc_call_details_destroy(&call_details);
-
-  grpc_call_unref(s);
-  grpc_call_unref(c);
-
-  cq_verifier_destroy(cqv);
-
-  end_test(&f);
-  config.tear_down_data(&f);
-
-  // Perform checks after test tear-down
-  // Guards against the case that there's outstanding channel-related work on a
-  // call prior to verification
-  // TODO(https://github.com/grpc/grpc/issues/13915) enable this for windows
-#ifndef GPR_WINDOWS
-  gpr_mu_lock(&g_mu);
-  if (!g_client_code_recv) {
-    GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu,
-                           grpc_timeout_seconds_to_deadline(3)));
-  }
-  if (!g_server_code_recv) {
-    GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu,
-                           grpc_timeout_seconds_to_deadline(3)));
-  }
-  GPR_ASSERT(g_client_status_code == GRPC_STATUS_UNIMPLEMENTED);
-  GPR_ASSERT(g_server_status_code == GRPC_STATUS_UNIMPLEMENTED);
-  gpr_mu_unlock(&g_mu);
-#endif  // GPR_WINDOWS
-}
-
-/*******************************************************************************
- * Test status_code filter
- */
-
-static grpc_error* init_call_elem(grpc_call_element* elem,
-                                  const grpc_call_element_args* args) {
-  return GRPC_ERROR_NONE;
-}
-
-static void client_destroy_call_elem(grpc_call_element* elem,
-                                     const grpc_call_final_info* final_info,
-                                     grpc_closure* ignored) {
-  gpr_mu_lock(&g_mu);
-  g_client_status_code = final_info->final_status;
-  g_client_code_recv = true;
-  gpr_cv_signal(&g_client_code_cv);
-  gpr_mu_unlock(&g_mu);
-}
-
-static void server_destroy_call_elem(grpc_call_element* elem,
-                                     const grpc_call_final_info* final_info,
-                                     grpc_closure* ignored) {
-  gpr_mu_lock(&g_mu);
-  g_server_status_code = final_info->final_status;
-  g_server_code_recv = true;
-  gpr_cv_signal(&g_server_code_cv);
-  gpr_mu_unlock(&g_mu);
-}
-
-static grpc_error* init_channel_elem(grpc_channel_element* elem,
-                                     grpc_channel_element_args* args) {
-  return GRPC_ERROR_NONE;
-}
-
-static void destroy_channel_elem(grpc_channel_element* elem) {}
-
-static const grpc_channel_filter test_client_filter = {
-    grpc_call_next_op,
-    grpc_channel_next_op,
-    0,
-    init_call_elem,
-    grpc_call_stack_ignore_set_pollset_or_pollset_set,
-    client_destroy_call_elem,
-    0,
-    init_channel_elem,
-    destroy_channel_elem,
-    grpc_channel_next_get_info,
-    "client_filter_status_code"};
-
-static const grpc_channel_filter test_server_filter = {
-    grpc_call_next_op,
-    grpc_channel_next_op,
-    0,
-    init_call_elem,
-    grpc_call_stack_ignore_set_pollset_or_pollset_set,
-    server_destroy_call_elem,
-    0,
-    init_channel_elem,
-    destroy_channel_elem,
-    grpc_channel_next_get_info,
-    "server_filter_status_code"};
-
-/*******************************************************************************
- * Registration
- */
-
-static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) {
-  grpc_channel_filter* filter = (grpc_channel_filter*)arg;
-  if (g_enable_filter) {
-    // Want to add the filter as close to the end as possible, to make
-    // sure that all of the filters work well together.  However, we
-    // can't add it at the very end, because the
-    // connected_channel/client_channel filter must be the last one.
-    // So we add it right before the last one.
-    grpc_channel_stack_builder_iterator* it =
-        grpc_channel_stack_builder_create_iterator_at_last(builder);
-    GPR_ASSERT(grpc_channel_stack_builder_move_prev(it));
-    const bool retval = grpc_channel_stack_builder_add_filter_before(
-        it, filter, nullptr, nullptr);
-    grpc_channel_stack_builder_iterator_destroy(it);
-    return retval;
-  } else {
-    return true;
-  }
-}
-
-static void init_plugin(void) {
-  gpr_mu_init(&g_mu);
-  gpr_cv_init(&g_client_code_cv);
-  gpr_cv_init(&g_server_code_cv);
-  g_client_code_recv = false;
-  g_server_code_recv = false;
-
-  grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX,
-                                   maybe_add_filter,
-                                   (void*)&test_client_filter);
-  grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX,
-                                   maybe_add_filter,
-                                   (void*)&test_client_filter);
-  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX,
-                                   maybe_add_filter,
-                                   (void*)&test_server_filter);
-}
-
-static void destroy_plugin(void) {
-  gpr_cv_destroy(&g_client_code_cv);
-  gpr_cv_destroy(&g_server_code_cv);
-  gpr_mu_destroy(&g_mu);
-}
-
-void filter_status_code(grpc_end2end_test_config config) {
-  g_enable_filter = true;
-  test_request(config);
-  g_enable_filter = false;
-}
-
-void filter_status_code_pre_init(void) {
-  grpc_register_plugin(init_plugin, destroy_plugin);
-}
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 51f0ac7ca61..7e7963916c6 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -7635,7 +7635,6 @@
       "test/core/end2end/tests/filter_call_init_fails.cc", 
       "test/core/end2end/tests/filter_causes_close.cc", 
       "test/core/end2end/tests/filter_latency.cc", 
-      "test/core/end2end/tests/filter_status_code.cc", 
       "test/core/end2end/tests/graceful_server_shutdown.cc", 
       "test/core/end2end/tests/high_initial_seqno.cc", 
       "test/core/end2end/tests/hpack_size.cc", 
@@ -7717,7 +7716,6 @@
       "test/core/end2end/tests/filter_call_init_fails.cc", 
       "test/core/end2end/tests/filter_causes_close.cc", 
       "test/core/end2end/tests/filter_latency.cc", 
-      "test/core/end2end/tests/filter_status_code.cc", 
       "test/core/end2end/tests/graceful_server_shutdown.cc", 
       "test/core/end2end/tests/high_initial_seqno.cc", 
       "test/core/end2end/tests/hpack_size.cc", 
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index 6b83cecd41d..04345bfb863 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -6821,29 +6821,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_census_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -8182,29 +8159,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_compress_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -9500,28 +9454,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_fakesec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -10728,29 +10660,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_fd_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -12018,29 +11927,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_full_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -13297,25 +13183,6 @@
       "linux"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "linux"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_full+pipe_test", 
-    "platforms": [
-      "linux"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -14500,29 +14367,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_full+trace_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -15815,29 +15659,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_full+workarounds_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -17194,30 +17015,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_http_proxy_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -18616,29 +18413,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_load_reporting_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -19995,30 +19769,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_oauth2_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -21341,7 +21091,7 @@
   }, 
   {
     "args": [
-      "filter_status_code"
+      "graceful_server_shutdown"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21365,7 +21115,7 @@
   }, 
   {
     "args": [
-      "graceful_server_shutdown"
+      "high_initial_seqno"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21389,14 +21139,14 @@
   }, 
   {
     "args": [
-      "high_initial_seqno"
+      "idempotent_request"
     ], 
     "ci_platforms": [
       "windows", 
       "linux", 
       "posix"
     ], 
-    "cpu_cost": 0.1, 
+    "cpu_cost": 1.0, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -21413,7 +21163,7 @@
   }, 
   {
     "args": [
-      "idempotent_request"
+      "invoke_large_request"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21437,7 +21187,7 @@
   }, 
   {
     "args": [
-      "invoke_large_request"
+      "large_metadata"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21461,7 +21211,7 @@
   }, 
   {
     "args": [
-      "large_metadata"
+      "load_reporting_hook"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21485,14 +21235,14 @@
   }, 
   {
     "args": [
-      "load_reporting_hook"
+      "max_connection_age"
     ], 
     "ci_platforms": [
       "windows", 
       "linux", 
       "posix"
     ], 
-    "cpu_cost": 1.0, 
+    "cpu_cost": 0.1, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -21509,31 +21259,7 @@
   }, 
   {
     "args": [
-      "max_connection_age"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_proxy_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
-  {
-    "args": [
-      "max_message_length"
+      "max_message_length"
     ], 
     "ci_platforms": [
       "windows", 
@@ -22467,30 +22193,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_sockpair_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -23715,30 +23417,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_sockpair+trace_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -24923,32 +24601,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [
-      "msan"
-    ], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_sockpair_1byte_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -26295,29 +25947,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_ssl_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -27602,30 +27231,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_ssl_proxy_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -28783,29 +28388,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_uds_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -30002,29 +29584,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "inproc_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "high_initial_seqno"
@@ -31062,29 +30621,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_census_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -32400,29 +31936,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_compress_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -33623,7 +33136,7 @@
   }, 
   {
     "args": [
-      "filter_status_code"
+      "graceful_server_shutdown"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33646,7 +33159,7 @@
   }, 
   {
     "args": [
-      "graceful_server_shutdown"
+      "high_initial_seqno"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33669,7 +33182,7 @@
   }, 
   {
     "args": [
-      "high_initial_seqno"
+      "hpack_size"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33692,14 +33205,14 @@
   }, 
   {
     "args": [
-      "hpack_size"
+      "idempotent_request"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 0.1, 
+    "cpu_cost": 1.0, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33715,7 +33228,7 @@
   }, 
   {
     "args": [
-      "idempotent_request"
+      "invoke_large_request"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33738,14 +33251,14 @@
   }, 
   {
     "args": [
-      "invoke_large_request"
+      "keepalive_timeout"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 1.0, 
+    "cpu_cost": 0.1, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33761,14 +33274,14 @@
   }, 
   {
     "args": [
-      "keepalive_timeout"
+      "large_metadata"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 0.1, 
+    "cpu_cost": 1.0, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33784,7 +33297,7 @@
   }, 
   {
     "args": [
-      "large_metadata"
+      "load_reporting_hook"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33807,14 +33320,14 @@
   }, 
   {
     "args": [
-      "load_reporting_hook"
+      "max_concurrent_streams"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 1.0, 
+    "cpu_cost": 0.1, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33830,7 +33343,7 @@
   }, 
   {
     "args": [
-      "max_concurrent_streams"
+      "max_connection_age"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33853,30 +33366,7 @@
   }, 
   {
     "args": [
-      "max_connection_age"
-    ], 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_fd_nosec_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
-  {
-    "args": [
-      "max_message_length"
+      "max_message_length"
     ], 
     "ci_platforms": [
       "linux", 
@@ -34888,29 +34378,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_full_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -36148,25 +35615,6 @@
       "linux"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "linux"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_full+pipe_nosec_test", 
-    "platforms": [
-      "linux"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -37328,29 +36776,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_full+trace_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -38620,29 +38045,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_full+workarounds_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -39975,30 +39377,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_http_proxy_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -41374,29 +40752,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_load_reporting_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -42657,30 +42012,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_proxy_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -43761,30 +43092,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_sockpair_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -44985,30 +44292,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_sockpair+trace_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -46167,32 +45450,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [
-      "msan"
-    ], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_sockpair_1byte_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -47491,29 +46748,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [
-      "uv"
-    ], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_uds_nosec_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -48687,29 +47921,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [
-      "filter_status_code"
-    ], 
-    "ci_platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 0.1, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "inproc_nosec_test", 
-    "platforms": [
-      "windows", 
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [
       "high_initial_seqno"

From 8f32e729bf14616d80158ecf43465a68dbf0f890 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Thu, 11 Jan 2018 18:26:50 -0800
Subject: [PATCH 085/161] Fix build_ruby_environment_macos.sh to pass
 shellcheck

---
 tools/distrib/build_ruby_environment_macos.sh | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tools/distrib/build_ruby_environment_macos.sh b/tools/distrib/build_ruby_environment_macos.sh
index af367402554..4a388a0e5c3 100644
--- a/tools/distrib/build_ruby_environment_macos.sh
+++ b/tools/distrib/build_ruby_environment_macos.sh
@@ -17,12 +17,12 @@ set -ex
 
 rm -rf ~/.rake-compiler
 
-CROSS_RUBY=`mktemp tmpfile.XXXXXXXX`
+CROSS_RUBY=$(mktemp tmpfile.XXXXXXXX)
 
-curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.0.3/tasks/bin/cross-ruby.rake > $CROSS_RUBY
+curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.0.3/tasks/bin/cross-ruby.rake > "$CROSS_RUBY"
 
 # See https://github.com/grpc/grpc/issues/12161 for verconf.h patch details
-patch $CROSS_RUBY << EOF
+patch "$CROSS_RUBY" << EOF
 --- cross-ruby.rake 2017-09-27 16:46:00.311020325 +0200
 +++ patched 2017-09-27 16:49:46.127016895 +0200
 @@ -133,7 +133,8 @@
@@ -49,8 +49,9 @@ MAKE="make -j8"
 
 for v in 2.5.0 2.4.0 2.3.0 2.2.2 2.1.6 2.0.0-p645 ; do
   ccache -c
-  rake -f $CROSS_RUBY cross-ruby VERSION=$v HOST=x86_64-darwin11
+  rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE"
 done
 
-sed 's/x86_64-darwin-11/universal-darwin/' ~/.rake-compiler/config.yml > $CROSS_RUBY
-mv $CROSS_RUBY ~/.rake-compiler/config.yml
+sed 's/x86_64-darwin-11/universal-darwin/' ~/.rake-compiler/config.yml > "$CROSS_RUBY"
+mv "$CROSS_RUBY" ~/.rake-compiler/config.yml
+

From 7b0a3fb4f97557a50dd046eb0d7b1c79ca360b12 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Thu, 11 Jan 2018 18:27:34 -0800
Subject: [PATCH 086/161] Fix check_nanopb_output.sh to pass shellcheck

---
 tools/distrib/check_nanopb_output.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh
index 81a04cbf574..a30b73f6892 100755
--- a/tools/distrib/check_nanopb_output.sh
+++ b/tools/distrib/check_nanopb_output.sh
@@ -51,7 +51,7 @@ readonly LOAD_BALANCER_GRPC_OUTPUT_PATH='src/core/ext/filters/client_channel/lb_
   "$LOAD_BALANCER_GRPC_OUTPUT_PATH"
 
 # compare outputs to checked compiled code
-if ! diff -r $NANOPB_TMP_OUTPUT src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1; then
+if ! diff -r "$NANOPB_TMP_OUTPUT" src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1; then
   echo "Outputs differ: $NANOPB_TMP_OUTPUT vs $LOAD_BALANCER_GRPC_OUTPUT_PATH"
   exit 2
 fi

From dfa2851462809a459dd734090c221ea322064c7d Mon Sep 17 00:00:00 2001
From: David Garcia Quintas <dgq@google.com>
Date: Thu, 11 Jan 2018 18:31:13 -0800
Subject: [PATCH 087/161] PR comments, round 2

---
 .../filters/client_channel/client_channel.cc  |   8 +-
 .../ext/filters/client_channel/lb_policy.h    |   5 +-
 .../client_channel/lb_policy/grpclb/grpclb.cc |   8 +-
 .../lb_policy/pick_first/pick_first.cc        |  27 ++---
 .../lb_policy/round_robin/round_robin.cc      |  39 +++---
 .../lb_policy/subchannel_list.cc              |   5 +-
 .../lb_policy/subchannel_list.h               |   3 +-
 .../ext/filters/client_channel/subchannel.cc  | 112 ++++++++----------
 .../ext/filters/client_channel/subchannel.h   |  21 +---
 src/core/lib/debug/trace.h                    |   5 +-
 src/core/lib/support/ref_counted.h            |   8 +-
 src/core/lib/support/ref_counted_ptr.h        |   2 +
 12 files changed, 104 insertions(+), 139 deletions(-)

diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc
index 36134c6b06e..6a10a25a325 100644
--- a/src/core/ext/filters/client_channel/client_channel.cc
+++ b/src/core/ext/filters/client_channel/client_channel.cc
@@ -1031,7 +1031,7 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
 static void pick_done_locked(grpc_call_element* elem, grpc_error* error) {
   call_data* calld = (call_data*)elem->call_data;
   channel_data* chand = (channel_data*)elem->channel_data;
-  if (calld->pick.connected_subchannel == nullptr) {
+  if (!calld->pick.connected_subchannel) {
     // Failed to create subchannel.
     GRPC_ERROR_UNREF(calld->error);
     calld->error = error == GRPC_ERROR_NONE
@@ -1283,7 +1283,7 @@ static void start_pick_locked(void* arg, grpc_error* ignored) {
   grpc_call_element* elem = (grpc_call_element*)arg;
   call_data* calld = (call_data*)elem->call_data;
   channel_data* chand = (channel_data*)elem->channel_data;
-  GPR_ASSERT(calld->pick.connected_subchannel == nullptr);
+  GPR_ASSERT(!calld->pick.connected_subchannel);
   if (chand->lb_policy != nullptr) {
     // We already have an LB policy, so ask it for a pick.
     if (pick_callback_start_locked(elem)) {
@@ -1462,8 +1462,8 @@ static void cc_destroy_call_elem(grpc_call_element* elem,
                                "client_channel_destroy_call");
   }
   GPR_ASSERT(calld->waiting_for_pick_batches_count == 0);
-  if (calld->pick.connected_subchannel != nullptr) {
-    GRPC_CONNECTED_SUBCHANNEL_UNREF(calld->pick.connected_subchannel, "picked");
+  if (calld->pick.connected_subchannel) {
+    calld->pick.connected_subchannel.reset();
   }
   for (size_t i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
     if (calld->pick.subchannel_call_context[i].value != nullptr) {
diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h
index 414cdd15796..0cc0cb59c3e 100644
--- a/src/core/ext/filters/client_channel/lb_policy.h
+++ b/src/core/ext/filters/client_channel/lb_policy.h
@@ -21,6 +21,7 @@
 
 #include "src/core/ext/filters/client_channel/subchannel.h"
 #include "src/core/lib/iomgr/polling_entity.h"
+#include "src/core/lib/support/ref_counted_ptr.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
 /** A load balancing policy: specified by a vtable and a struct (which
@@ -54,9 +55,9 @@ typedef struct grpc_lb_policy_pick_state {
   grpc_linked_mdelem lb_token_mdelem_storage;
   /// Closure to run when pick is complete, if not completed synchronously.
   grpc_closure* on_complete;
-  /// Will be set to the selected subchannel, or NULL on failure or when
+  /// Will be set to the selected subchannel, or nullptr on failure or when
   /// the LB policy decides to drop the call.
-  grpc_core::ConnectedSubchannel* connected_subchannel;
+  grpc_core::RefCountedPtr<grpc_core::ConnectedSubchannel> connected_subchannel;
   /// Will be populated with context to pass to the subchannel call, if needed.
   grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
   /// Upon success, \a *user_data will be set to whatever opaque information
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index 5849ac9d2da..39467272b36 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -322,7 +322,7 @@ static void pending_pick_set_metadata_and_context(pending_pick* pp) {
   /* if connected_subchannel is nullptr, no pick has been made by the RR
    * policy (e.g., all addresses failed to connect). There won't be any
    * user_data/token available */
-  if (pp->pick->connected_subchannel != nullptr) {
+  if (pp->pick->connected_subchannel.get() != nullptr) {
     if (!GRPC_MDISNULL(pp->lb_token)) {
       initial_metadata_add_lb_token(pp->pick->initial_metadata,
                                     &pp->pick->lb_token_mdelem_storage,
@@ -935,7 +935,8 @@ static void glb_shutdown_locked(grpc_lb_policy* pol,
       }
       gpr_free(pp);
     } else {
-      pp->pick->connected_subchannel = nullptr;
+      pp->pick->connected_subchannel =
+          grpc_core::MakeRefCounted<grpc_core::ConnectedSubchannel>(nullptr);
       GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_REF(error));
     }
     pp = next;
@@ -972,7 +973,8 @@ static void glb_cancel_pick_locked(grpc_lb_policy* pol,
   while (pp != nullptr) {
     pending_pick* next = pp->next;
     if (pp->pick == pick) {
-      pick->connected_subchannel = nullptr;
+      pick->connected_subchannel =
+          grpc_core::MakeRefCounted<grpc_core::ConnectedSubchannel>(nullptr);
       GRPC_CLOSURE_SCHED(&pp->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
index 58dcbffb2d8..c91a305f336 100644
--- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
@@ -81,7 +81,7 @@ static void pf_shutdown_locked(grpc_lb_policy* pol,
         GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
       }
     } else {
-      pick->connected_subchannel = nullptr;
+      pick->connected_subchannel.reset();
       GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error));
     }
   }
@@ -111,7 +111,7 @@ static void pf_cancel_pick_locked(grpc_lb_policy* pol,
   while (pp != nullptr) {
     grpc_lb_policy_pick_state* next = pp->next;
     if (pp == pick) {
-      pick->connected_subchannel = nullptr;
+      pick->connected_subchannel.reset();
       GRPC_CLOSURE_SCHED(pick->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
@@ -176,8 +176,7 @@ static int pf_pick_locked(grpc_lb_policy* pol,
   pick_first_lb_policy* p = (pick_first_lb_policy*)pol;
   // If we have a selected subchannel already, return synchronously.
   if (p->selected != nullptr) {
-    pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
-        p->selected->connected_subchannel, "picked");
+    pick->connected_subchannel = p->selected->connected_subchannel;
     return 1;
   }
   // No subchannel selected yet, so handle asynchronously.
@@ -295,9 +294,8 @@ static void pf_update_locked(grpc_lb_policy* policy,
                   p, p->selected->subchannel, i,
                   subchannel_list->num_subchannels);
         }
-        if (p->selected->connected_subchannel != nullptr) {
-          sd->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
-              p->selected->connected_subchannel, "pf_update_includes_selected");
+        if (p->selected->connected_subchannel) {
+          sd->connected_subchannel = p->selected->connected_subchannel;
         }
         p->selected = sd;
         if (p->subchannel_list != nullptr) {
@@ -425,7 +423,6 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
             sd->subchannel_list, "pf_selected_shutdown");
         grpc_lb_subchannel_data_unref_subchannel(
             sd, "pf_selected_shutdown");  // Unrefs connected subchannel
-
       } else {
         grpc_connectivity_state_set(&p->state_tracker,
                                     sd->curr_connectivity_state,
@@ -449,15 +446,8 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
     case GRPC_CHANNEL_READY: {
       // Case 2.  Promote p->latest_pending_subchannel_list to
       // p->subchannel_list.
-      grpc_core::ConnectedSubchannel* con =
+      sd->connected_subchannel =
           grpc_subchannel_get_connected_subchannel(sd->subchannel);
-      if (con == nullptr) {
-        // The subchannel may have become disconnected by the time this callback
-        // is invoked. Simply ignore and resubscribe: ulterior connectivity
-        // states must be in the pipeline and will eventually be invoked.
-        grpc_lb_subchannel_data_start_connectivity_watch(sd);
-        break;
-      }
       if (sd->subchannel_list == p->latest_pending_subchannel_list) {
         GPR_ASSERT(p->subchannel_list != nullptr);
         grpc_lb_subchannel_list_shutdown_and_unref(p->subchannel_list,
@@ -469,7 +459,7 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
       grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_READY,
                                   GRPC_ERROR_NONE, "connecting_ready");
       sd->connected_subchannel =
-          GRPC_CONNECTED_SUBCHANNEL_REF(con, "connected");
+          grpc_subchannel_get_connected_subchannel(sd->subchannel);
       p->selected = sd;
       if (grpc_lb_pick_first_trace.enabled()) {
         gpr_log(GPR_INFO, "Pick First %p selected subchannel %p", (void*)p,
@@ -481,8 +471,7 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
       grpc_lb_policy_pick_state* pick;
       while ((pick = p->pending_picks)) {
         p->pending_picks = pick->next;
-        pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
-            p->selected->connected_subchannel, "picked");
+        pick->connected_subchannel = p->selected->connected_subchannel;
         if (grpc_lb_pick_first_trace.enabled()) {
           gpr_log(GPR_INFO,
                   "Servicing pending pick with selected subchannel %p",
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
index 8d3fd637572..91fee2d51aa 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
@@ -36,6 +36,7 @@
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
+#include "src/core/lib/support/ref_counted_ptr.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/static_metadata.h"
 
@@ -127,7 +128,7 @@ static void update_last_ready_subchannel_index_locked(round_robin_lb_policy* p,
             (void*)p, (unsigned long)last_ready_index,
             (void*)p->subchannel_list->subchannels[last_ready_index].subchannel,
             (void*)p->subchannel_list->subchannels[last_ready_index]
-                .connected_subchannel);
+                .connected_subchannel.get());
   }
 }
 
@@ -162,7 +163,7 @@ static void rr_shutdown_locked(grpc_lb_policy* pol,
         GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
       }
     } else {
-      pick->connected_subchannel = nullptr;
+      pick->connected_subchannel.reset();
       GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error));
     }
   }
@@ -192,7 +193,7 @@ static void rr_cancel_pick_locked(grpc_lb_policy* pol,
   while (pp != nullptr) {
     grpc_lb_policy_pick_state* next = pp->next;
     if (pp == pick) {
-      pick->connected_subchannel = nullptr;
+      pick->connected_subchannel.reset();
       GRPC_CLOSURE_SCHED(pick->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick cancelled", &error, 1));
@@ -216,7 +217,7 @@ static void rr_cancel_picks_locked(grpc_lb_policy* pol,
     grpc_lb_policy_pick_state* next = pick->next;
     if ((pick->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      pick->connected_subchannel = nullptr;
+      pick->connected_subchannel.reset();
       GRPC_CLOSURE_SCHED(pick->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick cancelled", &error, 1));
@@ -262,8 +263,7 @@ static int rr_pick_locked(grpc_lb_policy* pol,
       /* readily available, report right away */
       grpc_lb_subchannel_data* sd =
           &p->subchannel_list->subchannels[next_ready_index];
-      pick->connected_subchannel =
-          GRPC_CONNECTED_SUBCHANNEL_REF(sd->connected_subchannel, "rr_picked");
+      pick->connected_subchannel = sd->connected_subchannel;
       if (pick->user_data != nullptr) {
         *pick->user_data = sd->user_data;
       }
@@ -272,8 +272,8 @@ static int rr_pick_locked(grpc_lb_policy* pol,
             GPR_DEBUG,
             "[RR %p] Picked target <-- Subchannel %p (connected %p) (sl %p, "
             "index %" PRIuPTR ")",
-            p, sd->subchannel, pick->connected_subchannel, sd->subchannel_list,
-            next_ready_index);
+            p, sd->subchannel, pick->connected_subchannel.get(),
+            sd->subchannel_list, next_ready_index);
       }
       /* only advance the last picked pointer if the selection was used */
       update_last_ready_subchannel_index_locked(p, next_ready_index);
@@ -373,7 +373,6 @@ static void update_lb_connectivity_status_locked(grpc_lb_subchannel_data* sd,
 
 static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
   grpc_lb_subchannel_data* sd = (grpc_lb_subchannel_data*)arg;
-  GPR_ASSERT(sd->curr_connectivity_state != GRPC_CHANNEL_SHUTDOWN);
   round_robin_lb_policy* p =
       (round_robin_lb_policy*)sd->subchannel_list->policy;
   if (grpc_lb_round_robin_trace.enabled()) {
@@ -408,6 +407,7 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
   // either the current or latest pending subchannel lists.
   GPR_ASSERT(sd->subchannel_list == p->subchannel_list ||
              sd->subchannel_list == p->latest_pending_subchannel_list);
+  GPR_ASSERT(sd->pending_connectivity_state_unsafe != GRPC_CHANNEL_SHUTDOWN);
   // Now that we're inside the combiner, copy the pending connectivity
   // state (which was set by the connectivity state watcher) to
   // curr_connectivity_state, which is what we use inside of the combiner.
@@ -419,18 +419,13 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
   // subchannel, if any.
   switch (sd->curr_connectivity_state) {
     case GRPC_CHANNEL_TRANSIENT_FAILURE: {
-      if (sd->connected_subchannel != nullptr) {
-        GRPC_CONNECTED_SUBCHANNEL_UNREF(
-            sd->connected_subchannel, "connected_subchannel_transient_failure");
-        sd->connected_subchannel = nullptr;
-      }
+      sd->connected_subchannel.reset();
       break;
     }
     case GRPC_CHANNEL_READY: {
-      if (sd->connected_subchannel == nullptr) {
-        sd->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
-            grpc_subchannel_get_connected_subchannel(sd->subchannel),
-            "connected");
+      if (!sd->connected_subchannel) {
+        sd->connected_subchannel =
+            grpc_subchannel_get_connected_subchannel(sd->subchannel);
       }
       if (sd->subchannel_list != p->subchannel_list) {
         // promote sd->subchannel_list to p->subchannel_list.
@@ -473,8 +468,7 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
       grpc_lb_policy_pick_state* pick;
       while ((pick = p->pending_picks)) {
         p->pending_picks = pick->next;
-        pick->connected_subchannel = GRPC_CONNECTED_SUBCHANNEL_REF(
-            selected->connected_subchannel, "rr_picked");
+        pick->connected_subchannel = selected->connected_subchannel;
         if (pick->user_data != nullptr) {
           *pick->user_data = selected->user_data;
         }
@@ -519,10 +513,9 @@ static void rr_ping_one_locked(grpc_lb_policy* pol, grpc_closure* on_initiate,
   if (next_ready_index < p->subchannel_list->num_subchannels) {
     grpc_lb_subchannel_data* selected =
         &p->subchannel_list->subchannels[next_ready_index];
-    grpc_core::ConnectedSubchannel* target = GRPC_CONNECTED_SUBCHANNEL_REF(
-        selected->connected_subchannel, "rr_ping");
+    grpc_core::RefCountedPtr<grpc_core::ConnectedSubchannel> target =
+        selected->connected_subchannel;
     target->Ping(on_initiate, on_ack);
-    GRPC_CONNECTED_SUBCHANNEL_UNREF(target, "rr_ping");
   } else {
     GRPC_CLOSURE_SCHED(on_initiate, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                                         "Round Robin not connected"));
diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
index 5ce1298afc4..fa2ffcc7966 100644
--- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.cc
@@ -42,10 +42,7 @@ void grpc_lb_subchannel_data_unref_subchannel(grpc_lb_subchannel_data* sd,
     }
     GRPC_SUBCHANNEL_UNREF(sd->subchannel, reason);
     sd->subchannel = nullptr;
-    if (sd->connected_subchannel != nullptr) {
-      GRPC_CONNECTED_SUBCHANNEL_UNREF(sd->connected_subchannel, reason);
-      sd->connected_subchannel = nullptr;
-    }
+    sd->connected_subchannel.reset();
     if (sd->user_data != nullptr) {
       GPR_ASSERT(sd->user_data_vtable != nullptr);
       sd->user_data_vtable->destroy(sd->user_data);
diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
index e4db3ef464b..f146c724e07 100644
--- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
+++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
@@ -22,6 +22,7 @@
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/ext/filters/client_channel/subchannel.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/support/ref_counted_ptr.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
 // TODO(roth): This code is intended to be shared between pick_first and
@@ -43,7 +44,7 @@ typedef struct {
   grpc_lb_subchannel_list* subchannel_list;
   /** subchannel itself */
   grpc_subchannel* subchannel;
-  grpc_core::ConnectedSubchannel* connected_subchannel;
+  grpc_core::RefCountedPtr<grpc_core::ConnectedSubchannel> connected_subchannel;
   /** Is a connectivity notification pending? */
   bool connectivity_notification_pending;
   /** notification that connectivity has changed on subchannel */
diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc
index 37c43d78ba4..fe0bb486e87 100644
--- a/src/core/ext/filters/client_channel/subchannel.cc
+++ b/src/core/ext/filters/client_channel/subchannel.cc
@@ -108,13 +108,13 @@ struct grpc_subchannel {
       being setup */
   grpc_pollset_set* pollset_set;
 
-  /** active connection, or null; of type grpc_core::ConnectedSubchannel
-   */
-  gpr_atm connected_subchannel;
-
   /** mutex protecting remaining elements */
   gpr_mu mu;
 
+  /** active connection, or null; of type grpc_core::ConnectedSubchannel
+   */
+  grpc_core::RefCountedPtr<grpc_core::ConnectedSubchannel> connected_subchannel;
+
   /** have we seen a disconnection? */
   bool disconnected;
   /** are we connecting */
@@ -169,17 +169,6 @@ static void connection_destroy(void* arg, grpc_error* error) {
   gpr_free(stk);
 }
 
-grpc_core::ConnectedSubchannel* grpc_connected_subchannel_ref(
-    grpc_core::ConnectedSubchannel* c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-  c->Ref(DEBUG_LOCATION, REF_REASON);
-  return c;
-}
-
-void grpc_connected_subchannel_unref(
-    grpc_core::ConnectedSubchannel* c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
-  c->Unref(DEBUG_LOCATION, REF_REASON);
-}
-
 /*
  * grpc_subchannel implementation
  */
@@ -250,11 +239,7 @@ static void disconnect(grpc_subchannel* c) {
   c->disconnected = true;
   grpc_connector_shutdown(c->connector, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                                             "Subchannel disconnected"));
-  grpc_core::ConnectedSubchannel* con = GET_CONNECTED_SUBCHANNEL(c, no_barrier);
-  if (con != nullptr) {
-    GRPC_CONNECTED_SUBCHANNEL_UNREF(con, "disconnect");
-    gpr_atm_no_barrier_store(&c->connected_subchannel, (gpr_atm)0xdeadbeef);
-  }
+  c->connected_subchannel.reset();
   gpr_mu_unlock(&c->mu);
 }
 
@@ -458,7 +443,7 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) {
     return;
   }
 
-  if (GET_CONNECTED_SUBCHANNEL(c, no_barrier) != nullptr) {
+  if (c->connected_subchannel) {
     /* Already connected: don't restart */
     return;
   }
@@ -536,38 +521,37 @@ static void on_connected_subchannel_connectivity_changed(void* p,
 
   gpr_mu_lock(mu);
 
-  auto* con = GET_CONNECTED_SUBCHANNEL(c, no_barrier);
-  /* if we failed just leave this closure */
-  if (connected_subchannel_watcher->connectivity_state >=
-      GRPC_CHANNEL_TRANSIENT_FAILURE) {
-    if (!c->disconnected && con != nullptr) {
-      GRPC_CONNECTED_SUBCHANNEL_UNREF(con, "transient_failure");
-      gpr_atm_no_barrier_store(&c->connected_subchannel, (gpr_atm) nullptr);
-      grpc_connectivity_state_set(&c->state_tracker,
-                                  GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                  GRPC_ERROR_REF(error), "reflect_child");
-      c->backoff_begun = false;
-      c->backoff->Reset();
-      if (grpc_trace_stream_refcount.enabled()) {
-        gpr_log(GPR_INFO,
-                "Connected subchannel %p of subchannel %p has gone into %s. "
-                "Attempting to reconnect.",
-                con, c,
-                grpc_connectivity_state_name(
-                    connected_subchannel_watcher->connectivity_state));
+  switch (connected_subchannel_watcher->connectivity_state) {
+    case GRPC_CHANNEL_TRANSIENT_FAILURE:
+    case GRPC_CHANNEL_SHUTDOWN: {
+      if (!c->disconnected && c->connected_subchannel) {
+        if (grpc_trace_stream_refcount.enabled()) {
+          gpr_log(GPR_INFO,
+                  "Connected subchannel %p of subchannel %p has gone into %s. "
+                  "Attempting to reconnect.",
+                  c->connected_subchannel.get(), c,
+                  grpc_connectivity_state_name(
+                      connected_subchannel_watcher->connectivity_state));
+        }
+        c->connected_subchannel.reset();
+        grpc_connectivity_state_set(&c->state_tracker,
+                                    GRPC_CHANNEL_TRANSIENT_FAILURE,
+                                    GRPC_ERROR_REF(error), "reflect_child");
+        c->backoff_begun = false;
+        c->backoff->Reset();
+        maybe_start_connecting_locked(c);
+      } else {
+        connected_subchannel_watcher->connectivity_state =
+            GRPC_CHANNEL_SHUTDOWN;
       }
-      maybe_start_connecting_locked(c);
-    } else {
-      connected_subchannel_watcher->connectivity_state = GRPC_CHANNEL_SHUTDOWN;
+      break;
     }
-  } else {
-    grpc_connectivity_state_set(
-        &c->state_tracker, connected_subchannel_watcher->connectivity_state,
-        GRPC_ERROR_REF(error), "reflect_child");
-    if (connected_subchannel_watcher->connectivity_state <
-        GRPC_CHANNEL_TRANSIENT_FAILURE) {
+    default: {
+      grpc_connectivity_state_set(
+          &c->state_tracker, connected_subchannel_watcher->connectivity_state,
+          GRPC_ERROR_REF(error), "reflect_child");
       GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
-      con->NotifyOnStateChange(
+      c->connected_subchannel->NotifyOnStateChange(
           nullptr, &connected_subchannel_watcher->connectivity_state,
           &connected_subchannel_watcher->closure);
       connected_subchannel_watcher = nullptr;
@@ -619,22 +603,19 @@ static bool publish_transport_locked(grpc_subchannel* c) {
   }
 
   /* publish */
-  /* TODO(ctiller): this full barrier seems to clear up a TSAN failure.
-                    I'd have expected the rel_cas below to be enough, but
-                    seemingly it's not.
-                    Re-evaluate if we really need this. */
-  grpc_core::ConnectedSubchannel* con =
-      grpc_core::New<grpc_core::ConnectedSubchannel>(stk);
+  c->connected_subchannel.reset(
+      grpc_core::New<grpc_core::ConnectedSubchannel>(stk));
   gpr_atm_full_barrier();
-  GPR_ASSERT(gpr_atm_rel_cas(&c->connected_subchannel, 0, (gpr_atm)con));
+  gpr_log(GPR_INFO, "New connected subchannel at %p for subchannel %p",
+          c->connected_subchannel.get(), c);
 
   /* setup subchannel watching connected subchannel for changes; subchannel
      ref for connecting is donated to the state watcher */
   GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
   GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting");
-  con->NotifyOnStateChange(c->pollset_set,
-                           &connected_subchannel_watcher->connectivity_state,
-                           &connected_subchannel_watcher->closure);
+  c->connected_subchannel->NotifyOnStateChange(
+      c->pollset_set, &connected_subchannel_watcher->connectivity_state,
+      &connected_subchannel_watcher->closure);
 
   /* signal completion */
   grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_READY,
@@ -684,7 +665,7 @@ static void subchannel_call_destroy(void* call, grpc_error* error) {
   grpc_core::ConnectedSubchannel* connection = c->connection;
   grpc_call_stack_destroy(SUBCHANNEL_CALL_TO_CALL_STACK(c), nullptr,
                           c->schedule_closure_after_destroy);
-  GRPC_CONNECTED_SUBCHANNEL_UNREF(connection, "subchannel_call");
+  connection->Unref(DEBUG_LOCATION, "subchannel_call");
   GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0);
 }
 
@@ -715,9 +696,12 @@ void grpc_subchannel_call_process_op(grpc_subchannel_call* call,
   GPR_TIMER_END("grpc_subchannel_call_process_op", 0);
 }
 
-grpc_core::ConnectedSubchannel* grpc_subchannel_get_connected_subchannel(
-    grpc_subchannel* c) {
-  return GET_CONNECTED_SUBCHANNEL(c, acq);
+grpc_core::RefCountedPtr<grpc_core::ConnectedSubchannel>
+grpc_subchannel_get_connected_subchannel(grpc_subchannel* c) {
+  gpr_mu_lock(&c->mu);
+  auto copy = c->connected_subchannel;
+  gpr_mu_unlock(&c->mu);
+  return copy;
 }
 
 const grpc_subchannel_key* grpc_subchannel_get_key(
diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h
index 61fa97a21aa..3bcc5c2432e 100644
--- a/src/core/ext/filters/client_channel/subchannel.h
+++ b/src/core/ext/filters/client_channel/subchannel.h
@@ -24,6 +24,7 @@
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/support/arena.h"
 #include "src/core/lib/support/ref_counted.h"
+#include "src/core/lib/support/ref_counted_ptr.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/metadata.h"
 
@@ -48,10 +49,6 @@ typedef struct grpc_subchannel_key grpc_subchannel_key;
   grpc_subchannel_weak_ref((p), __FILE__, __LINE__, (r))
 #define GRPC_SUBCHANNEL_WEAK_UNREF(p, r) \
   grpc_subchannel_weak_unref((p), __FILE__, __LINE__, (r))
-#define GRPC_CONNECTED_SUBCHANNEL_REF(p, r) \
-  grpc_connected_subchannel_ref((p), __FILE__, __LINE__, (r))
-#define GRPC_CONNECTED_SUBCHANNEL_UNREF(p, r) \
-  grpc_connected_subchannel_unref((p), __FILE__, __LINE__, (r))
 #define GRPC_SUBCHANNEL_CALL_REF(p, r) \
   grpc_subchannel_call_ref((p), __FILE__, __LINE__, (r))
 #define GRPC_SUBCHANNEL_CALL_UNREF(p, r) \
@@ -65,9 +62,6 @@ typedef struct grpc_subchannel_key grpc_subchannel_key;
 #define GRPC_SUBCHANNEL_UNREF(p, r) grpc_subchannel_unref((p))
 #define GRPC_SUBCHANNEL_WEAK_REF(p, r) grpc_subchannel_weak_ref((p))
 #define GRPC_SUBCHANNEL_WEAK_UNREF(p, r) grpc_subchannel_weak_unref((p))
-#define GRPC_CONNECTED_SUBCHANNEL_REF(p, r) grpc_connected_subchannel_ref((p))
-#define GRPC_CONNECTED_SUBCHANNEL_UNREF(p, r) \
-  grpc_connected_subchannel_unref((p))
 #define GRPC_SUBCHANNEL_CALL_REF(p, r) grpc_subchannel_call_ref((p))
 #define GRPC_SUBCHANNEL_CALL_UNREF(p, r) grpc_subchannel_call_unref((p))
 #define GRPC_SUBCHANNEL_REF_EXTRA_ARGS
@@ -111,10 +105,6 @@ grpc_subchannel* grpc_subchannel_weak_ref(
     grpc_subchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_weak_unref(
     grpc_subchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-grpc_core::ConnectedSubchannel* grpc_connected_subchannel_ref(
-    grpc_core::ConnectedSubchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-void grpc_connected_subchannel_unref(
-    grpc_core::ConnectedSubchannel* channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_call_ref(
     grpc_subchannel_call* call GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_call_unref(
@@ -130,10 +120,11 @@ void grpc_subchannel_notify_on_state_change(
     grpc_subchannel* channel, grpc_pollset_set* interested_parties,
     grpc_connectivity_state* state, grpc_closure* notify);
 
-/** retrieve the grpc_core::ConnectedSubchannel - or NULL if called before
-    the subchannel becomes connected */
-grpc_core::ConnectedSubchannel* grpc_subchannel_get_connected_subchannel(
-    grpc_subchannel* subchannel);
+/** retrieve the grpc_core::ConnectedSubchannel - or nullptr if not connected
+ * (which may happen before it initially connects or during transient failures)
+ * */
+grpc_core::RefCountedPtr<grpc_core::ConnectedSubchannel>
+grpc_subchannel_get_connected_subchannel(grpc_subchannel* c);
 
 /** return the subchannel index key for \a subchannel */
 const grpc_subchannel_key* grpc_subchannel_get_key(
diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h
index 6ee1b49f747..69ddd802222 100644
--- a/src/core/lib/debug/trace.h
+++ b/src/core/lib/debug/trace.h
@@ -88,10 +88,9 @@ class TraceFlag {
 #ifndef NDEBUG
 typedef TraceFlag DebugOnlyTraceFlag;
 #else
-class DebugOnlyTraceFlag : public TraceFlag {
+class DebugOnlyTraceFlag {
  public:
-  DebugOnlyTraceFlag(bool default_enabled, const char* name)
-      : TraceFlag(default_enabled, name) {}
+  DebugOnlyTraceFlag(bool default_enabled, const char* name) {}
   bool enabled() { return false; }
 
  private:
diff --git a/src/core/lib/support/ref_counted.h b/src/core/lib/support/ref_counted.h
index 8fdc3458d16..eaa293a86c1 100644
--- a/src/core/lib/support/ref_counted.h
+++ b/src/core/lib/support/ref_counted.h
@@ -106,13 +106,19 @@ class RefCountedWithTracing {
   template <typename T>
   friend void Delete(T*);
 
-  RefCountedWithTracing() : RefCountedWithTracing(nullptr) {}
+  RefCountedWithTracing()
+      : RefCountedWithTracing(static_cast<TraceFlag*>(nullptr)) {}
 
   explicit RefCountedWithTracing(TraceFlag* trace_flag)
       : trace_flag_(trace_flag) {
     gpr_ref_init(&refs_, 1);
   }
 
+#ifdef NDEBUG
+  explicit RefCountedWithTracing(DebugOnlyTraceFlag* trace_flag)
+      : RefCountedWithTracing() {}
+#endif
+
   virtual ~RefCountedWithTracing() {}
 
  private:
diff --git a/src/core/lib/support/ref_counted_ptr.h b/src/core/lib/support/ref_counted_ptr.h
index dc2385e369c..dad213003d6 100644
--- a/src/core/lib/support/ref_counted_ptr.h
+++ b/src/core/lib/support/ref_counted_ptr.h
@@ -76,6 +76,8 @@ class RefCountedPtr {
   T& operator*() const { return *value_; }
   T* operator->() const { return value_; }
 
+  explicit operator bool() const { return get() != nullptr; }
+
  private:
   T* value_ = nullptr;
 };

From b353297b4394a6c5fadd04ce778bed480d6d5782 Mon Sep 17 00:00:00 2001
From: Ken Payson <kpayson@google.com>
Date: Thu, 11 Jan 2018 20:25:30 -0800
Subject: [PATCH 088/161] Revert "Revert "Set error status correctly on server
 side""

This reverts commit 951f84aea00a1f8a65cf160d7d8f342c30593000.
---
 CMakeLists.txt                                |   2 +
 Makefile                                      |   2 +
 gRPC-Core.podspec                             |   1 +
 grpc.gyp                                      |   2 +
 src/core/lib/surface/call.cc                  |   5 +-
 test/core/end2end/end2end_nosec_tests.cc      |   8 +
 test/core/end2end/end2end_tests.cc            |   8 +
 test/core/end2end/gen_build_yaml.py           |   1 +
 test/core/end2end/generate_tests.bzl          |   1 +
 test/core/end2end/tests/filter_status_code.cc | 353 ++++++++
 .../generated/sources_and_headers.json        |   2 +
 tools/run_tests/generated/tests.json          | 839 +++++++++++++++++-
 12 files changed, 1197 insertions(+), 27 deletions(-)
 create mode 100644 test/core/end2end/tests/filter_status_code.cc

diff --git a/CMakeLists.txt b/CMakeLists.txt
index deca1b3f752..78ccfb21329 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4609,6 +4609,7 @@ add_library(end2end_tests
   test/core/end2end/tests/filter_call_init_fails.cc
   test/core/end2end/tests/filter_causes_close.cc
   test/core/end2end/tests/filter_latency.cc
+  test/core/end2end/tests/filter_status_code.cc
   test/core/end2end/tests/graceful_server_shutdown.cc
   test/core/end2end/tests/high_initial_seqno.cc
   test/core/end2end/tests/hpack_size.cc
@@ -4710,6 +4711,7 @@ add_library(end2end_nosec_tests
   test/core/end2end/tests/filter_call_init_fails.cc
   test/core/end2end/tests/filter_causes_close.cc
   test/core/end2end/tests/filter_latency.cc
+  test/core/end2end/tests/filter_status_code.cc
   test/core/end2end/tests/graceful_server_shutdown.cc
   test/core/end2end/tests/high_initial_seqno.cc
   test/core/end2end/tests/hpack_size.cc
diff --git a/Makefile b/Makefile
index 499aca56f65..f50163efdce 100644
--- a/Makefile
+++ b/Makefile
@@ -8552,6 +8552,7 @@ LIBEND2END_TESTS_SRC = \
     test/core/end2end/tests/filter_call_init_fails.cc \
     test/core/end2end/tests/filter_causes_close.cc \
     test/core/end2end/tests/filter_latency.cc \
+    test/core/end2end/tests/filter_status_code.cc \
     test/core/end2end/tests/graceful_server_shutdown.cc \
     test/core/end2end/tests/high_initial_seqno.cc \
     test/core/end2end/tests/hpack_size.cc \
@@ -8650,6 +8651,7 @@ LIBEND2END_NOSEC_TESTS_SRC = \
     test/core/end2end/tests/filter_call_init_fails.cc \
     test/core/end2end/tests/filter_causes_close.cc \
     test/core/end2end/tests/filter_latency.cc \
+    test/core/end2end/tests/filter_status_code.cc \
     test/core/end2end/tests/graceful_server_shutdown.cc \
     test/core/end2end/tests/high_initial_seqno.cc \
     test/core/end2end/tests/hpack_size.cc \
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index fcabd62c85a..358fad3d983 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -1039,6 +1039,7 @@ Pod::Spec.new do |s|
                       'test/core/end2end/tests/filter_call_init_fails.cc',
                       'test/core/end2end/tests/filter_causes_close.cc',
                       'test/core/end2end/tests/filter_latency.cc',
+                      'test/core/end2end/tests/filter_status_code.cc',
                       'test/core/end2end/tests/graceful_server_shutdown.cc',
                       'test/core/end2end/tests/high_initial_seqno.cc',
                       'test/core/end2end/tests/hpack_size.cc',
diff --git a/grpc.gyp b/grpc.gyp
index 06da3a758fb..281fbfa8a6e 100644
--- a/grpc.gyp
+++ b/grpc.gyp
@@ -2381,6 +2381,7 @@
         'test/core/end2end/tests/filter_call_init_fails.cc',
         'test/core/end2end/tests/filter_causes_close.cc',
         'test/core/end2end/tests/filter_latency.cc',
+        'test/core/end2end/tests/filter_status_code.cc',
         'test/core/end2end/tests/graceful_server_shutdown.cc',
         'test/core/end2end/tests/high_initial_seqno.cc',
         'test/core/end2end/tests/hpack_size.cc',
@@ -2453,6 +2454,7 @@
         'test/core/end2end/tests/filter_call_init_fails.cc',
         'test/core/end2end/tests/filter_causes_close.cc',
         'test/core/end2end/tests/filter_latency.cc',
+        'test/core/end2end/tests/filter_status_code.cc',
         'test/core/end2end/tests/graceful_server_shutdown.cc',
         'test/core/end2end/tests/high_initial_seqno.cc',
         'test/core/end2end/tests/hpack_size.cc',
diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc
index a457aaa7a2f..d677576c145 100644
--- a/src/core/lib/surface/call.cc
+++ b/src/core/lib/surface/call.cc
@@ -1851,8 +1851,9 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
         {
           grpc_error* override_error = GRPC_ERROR_NONE;
           if (op->data.send_status_from_server.status != GRPC_STATUS_OK) {
-            override_error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                "Error from server send status");
+            override_error =
+                error_from_status(op->data.send_status_from_server.status,
+                                  "Returned non-ok status");
           }
           if (op->data.send_status_from_server.status_details != nullptr) {
             call->send_extra_metadata[1].md = grpc_mdelem_from_slices(
diff --git a/test/core/end2end/end2end_nosec_tests.cc b/test/core/end2end/end2end_nosec_tests.cc
index 3236feea56e..6318550ad86 100644
--- a/test/core/end2end/end2end_nosec_tests.cc
+++ b/test/core/end2end/end2end_nosec_tests.cc
@@ -68,6 +68,8 @@ extern void filter_causes_close(grpc_end2end_test_config config);
 extern void filter_causes_close_pre_init(void);
 extern void filter_latency(grpc_end2end_test_config config);
 extern void filter_latency_pre_init(void);
+extern void filter_status_code(grpc_end2end_test_config config);
+extern void filter_status_code_pre_init(void);
 extern void graceful_server_shutdown(grpc_end2end_test_config config);
 extern void graceful_server_shutdown_pre_init(void);
 extern void high_initial_seqno(grpc_end2end_test_config config);
@@ -170,6 +172,7 @@ void grpc_end2end_tests_pre_init(void) {
   filter_call_init_fails_pre_init();
   filter_causes_close_pre_init();
   filter_latency_pre_init();
+  filter_status_code_pre_init();
   graceful_server_shutdown_pre_init();
   high_initial_seqno_pre_init();
   hpack_size_pre_init();
@@ -237,6 +240,7 @@ void grpc_end2end_tests(int argc, char **argv,
     filter_call_init_fails(config);
     filter_causes_close(config);
     filter_latency(config);
+    filter_status_code(config);
     graceful_server_shutdown(config);
     high_initial_seqno(config);
     hpack_size(config);
@@ -356,6 +360,10 @@ void grpc_end2end_tests(int argc, char **argv,
       filter_latency(config);
       continue;
     }
+    if (0 == strcmp("filter_status_code", argv[i])) {
+      filter_status_code(config);
+      continue;
+    }
     if (0 == strcmp("graceful_server_shutdown", argv[i])) {
       graceful_server_shutdown(config);
       continue;
diff --git a/test/core/end2end/end2end_tests.cc b/test/core/end2end/end2end_tests.cc
index ca9443b642d..9d8dfd67238 100644
--- a/test/core/end2end/end2end_tests.cc
+++ b/test/core/end2end/end2end_tests.cc
@@ -70,6 +70,8 @@ extern void filter_causes_close(grpc_end2end_test_config config);
 extern void filter_causes_close_pre_init(void);
 extern void filter_latency(grpc_end2end_test_config config);
 extern void filter_latency_pre_init(void);
+extern void filter_status_code(grpc_end2end_test_config config);
+extern void filter_status_code_pre_init(void);
 extern void graceful_server_shutdown(grpc_end2end_test_config config);
 extern void graceful_server_shutdown_pre_init(void);
 extern void high_initial_seqno(grpc_end2end_test_config config);
@@ -173,6 +175,7 @@ void grpc_end2end_tests_pre_init(void) {
   filter_call_init_fails_pre_init();
   filter_causes_close_pre_init();
   filter_latency_pre_init();
+  filter_status_code_pre_init();
   graceful_server_shutdown_pre_init();
   high_initial_seqno_pre_init();
   hpack_size_pre_init();
@@ -241,6 +244,7 @@ void grpc_end2end_tests(int argc, char **argv,
     filter_call_init_fails(config);
     filter_causes_close(config);
     filter_latency(config);
+    filter_status_code(config);
     graceful_server_shutdown(config);
     high_initial_seqno(config);
     hpack_size(config);
@@ -364,6 +368,10 @@ void grpc_end2end_tests(int argc, char **argv,
       filter_latency(config);
       continue;
     }
+    if (0 == strcmp("filter_status_code", argv[i])) {
+      filter_status_code(config);
+      continue;
+    }
     if (0 == strcmp("graceful_server_shutdown", argv[i])) {
       graceful_server_shutdown(config);
       continue;
diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py
index 7c8e7f420ae..e7cf97b2d02 100755
--- a/test/core/end2end/gen_build_yaml.py
+++ b/test/core/end2end/gen_build_yaml.py
@@ -101,6 +101,7 @@ END2END_TESTS = {
     'filter_causes_close': default_test_options._replace(cpu_cost=LOWCPU),
     'filter_call_init_fails': default_test_options,
     'filter_latency': default_test_options._replace(cpu_cost=LOWCPU),
+    'filter_status_code': default_test_options._replace(cpu_cost=LOWCPU),
     'graceful_server_shutdown': default_test_options._replace(cpu_cost=LOWCPU,exclude_inproc=True),
     'hpack_size': default_test_options._replace(proxyable=False,
                                                 traceable=False,
diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl
index b9a42bdb885..1d759e1ecbf 100755
--- a/test/core/end2end/generate_tests.bzl
+++ b/test/core/end2end/generate_tests.bzl
@@ -146,6 +146,7 @@ END2END_TESTS = {
     'trailing_metadata': test_options(),
     'authority_not_supported': test_options(),
     'filter_latency': test_options(),
+    'filter_status_code': test_options(),
     'workaround_cronet_compression': test_options(),
     'write_buffering': test_options(needs_write_buffering=True),
     'write_buffering_at_end': test_options(needs_write_buffering=True),
diff --git a/test/core/end2end/tests/filter_status_code.cc b/test/core/end2end/tests/filter_status_code.cc
new file mode 100644
index 00000000000..261ddd93ec5
--- /dev/null
+++ b/test/core/end2end/tests/filter_status_code.cc
@@ -0,0 +1,353 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "test/core/end2end/end2end_tests.h"
+
+#include <limits.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <grpc/byte_buffer.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/time.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/surface/channel_init.h"
+#include "test/core/end2end/cq_verifier.h"
+
+static bool g_enable_filter = false;
+static gpr_mu g_mu;
+static bool g_client_code_recv;
+static bool g_server_code_recv;
+static gpr_cv g_client_code_cv;
+static gpr_cv g_server_code_cv;
+static grpc_status_code g_client_status_code;
+static grpc_status_code g_server_status_code;
+
+static void* tag(intptr_t t) { return (void*)t; }
+
+static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config,
+                                            const char* test_name,
+                                            grpc_channel_args* client_args,
+                                            grpc_channel_args* server_args) {
+  grpc_end2end_test_fixture f;
+  gpr_log(GPR_INFO, "Running test: %s/%s", test_name, config.name);
+  f = config.create_fixture(client_args, server_args);
+  config.init_server(&f, server_args);
+  config.init_client(&f, client_args);
+  return f;
+}
+
+static gpr_timespec n_seconds_from_now(int n) {
+  return grpc_timeout_seconds_to_deadline(n);
+}
+
+static gpr_timespec five_seconds_from_now(void) {
+  return n_seconds_from_now(5);
+}
+
+static void drain_cq(grpc_completion_queue* cq) {
+  grpc_event ev;
+  do {
+    ev = grpc_completion_queue_next(cq, five_seconds_from_now(), nullptr);
+  } while (ev.type != GRPC_QUEUE_SHUTDOWN);
+}
+
+static void shutdown_server(grpc_end2end_test_fixture* f) {
+  if (!f->server) return;
+  grpc_server_shutdown_and_notify(f->server, f->shutdown_cq, tag(1000));
+  GPR_ASSERT(grpc_completion_queue_pluck(f->shutdown_cq, tag(1000),
+                                         grpc_timeout_seconds_to_deadline(5),
+                                         nullptr)
+                 .type == GRPC_OP_COMPLETE);
+  grpc_server_destroy(f->server);
+  f->server = nullptr;
+}
+
+static void shutdown_client(grpc_end2end_test_fixture* f) {
+  if (!f->client) return;
+  grpc_channel_destroy(f->client);
+  f->client = nullptr;
+}
+
+static void end_test(grpc_end2end_test_fixture* f) {
+  shutdown_server(f);
+  shutdown_client(f);
+
+  grpc_completion_queue_shutdown(f->cq);
+  drain_cq(f->cq);
+  grpc_completion_queue_destroy(f->cq);
+  grpc_completion_queue_destroy(f->shutdown_cq);
+}
+
+// Simple request via a server filter that saves the reported status code.
+static void test_request(grpc_end2end_test_config config) {
+  grpc_call* c;
+  grpc_call* s;
+  grpc_end2end_test_fixture f =
+      begin_test(config, "filter_status_code", nullptr, nullptr);
+  cq_verifier* cqv = cq_verifier_create(f.cq);
+  grpc_op ops[6];
+  grpc_op* op;
+  grpc_metadata_array initial_metadata_recv;
+  grpc_metadata_array trailing_metadata_recv;
+  grpc_metadata_array request_metadata_recv;
+  grpc_call_details call_details;
+  grpc_status_code status;
+  grpc_call_error error;
+  grpc_slice details;
+  int was_cancelled = 2;
+
+  gpr_mu_lock(&g_mu);
+  g_client_status_code = GRPC_STATUS_OK;
+  g_server_status_code = GRPC_STATUS_OK;
+  gpr_mu_unlock(&g_mu);
+
+  gpr_timespec deadline = five_seconds_from_now();
+  c = grpc_channel_create_call(
+      f.client, nullptr, GRPC_PROPAGATE_DEFAULTS, f.cq,
+      grpc_slice_from_static_string("/foo"),
+      get_host_override_slice("foo.test.google.fr", config), deadline, nullptr);
+  GPR_ASSERT(c);
+
+  grpc_metadata_array_init(&initial_metadata_recv);
+  grpc_metadata_array_init(&trailing_metadata_recv);
+  grpc_metadata_array_init(&request_metadata_recv);
+  grpc_call_details_init(&call_details);
+
+  memset(ops, 0, sizeof(ops));
+  op = ops;
+  op->op = GRPC_OP_SEND_INITIAL_METADATA;
+  op->data.send_initial_metadata.count = 0;
+  op->data.send_initial_metadata.metadata = nullptr;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  op->op = GRPC_OP_RECV_INITIAL_METADATA;
+  op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
+  op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
+  op->data.recv_status_on_client.status = &status;
+  op->data.recv_status_on_client.status_details = &details;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), nullptr);
+  GPR_ASSERT(GRPC_CALL_OK == error);
+
+  error =
+      grpc_server_request_call(f.server, &s, &call_details,
+                               &request_metadata_recv, f.cq, f.cq, tag(101));
+  GPR_ASSERT(GRPC_CALL_OK == error);
+
+  CQ_EXPECT_COMPLETION(cqv, tag(101), 1);
+  cq_verify(cqv);
+
+  memset(ops, 0, sizeof(ops));
+  op = ops;
+  op->op = GRPC_OP_SEND_INITIAL_METADATA;
+  op->data.send_initial_metadata.count = 0;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
+  op->data.send_status_from_server.trailing_metadata_count = 0;
+  op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED;
+  grpc_slice status_string = grpc_slice_from_static_string("xyz");
+  op->data.send_status_from_server.status_details = &status_string;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
+  op->data.recv_close_on_server.cancelled = &was_cancelled;
+  op->flags = 0;
+  op->reserved = nullptr;
+  op++;
+  error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), nullptr);
+  GPR_ASSERT(GRPC_CALL_OK == error);
+
+  CQ_EXPECT_COMPLETION(cqv, tag(102), 1);
+  CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
+  cq_verify(cqv);
+
+  GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED);
+  GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz"));
+
+  grpc_slice_unref(details);
+  grpc_metadata_array_destroy(&initial_metadata_recv);
+  grpc_metadata_array_destroy(&trailing_metadata_recv);
+  grpc_metadata_array_destroy(&request_metadata_recv);
+  grpc_call_details_destroy(&call_details);
+
+  grpc_call_unref(s);
+  grpc_call_unref(c);
+
+  cq_verifier_destroy(cqv);
+
+  end_test(&f);
+  config.tear_down_data(&f);
+
+  // Perform checks after test tear-down
+  // Guards against the case that there's outstanding channel-related work on a
+  // call prior to verification
+  // TODO(https://github.com/grpc/grpc/issues/13915) enable this for windows
+#ifndef GPR_WINDOWS
+  gpr_mu_lock(&g_mu);
+  if (!g_client_code_recv) {
+    GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu,
+                           grpc_timeout_seconds_to_deadline(3)));
+  }
+  if (!g_server_code_recv) {
+    GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu,
+                           grpc_timeout_seconds_to_deadline(3)));
+  }
+  GPR_ASSERT(g_client_status_code == GRPC_STATUS_UNIMPLEMENTED);
+  GPR_ASSERT(g_server_status_code == GRPC_STATUS_UNIMPLEMENTED);
+  gpr_mu_unlock(&g_mu);
+#endif  // GPR_WINDOWS
+}
+
+/*******************************************************************************
+ * Test status_code filter
+ */
+
+static grpc_error* init_call_elem(grpc_call_element* elem,
+                                  const grpc_call_element_args* args) {
+  return GRPC_ERROR_NONE;
+}
+
+static void client_destroy_call_elem(grpc_call_element* elem,
+                                     const grpc_call_final_info* final_info,
+                                     grpc_closure* ignored) {
+  gpr_mu_lock(&g_mu);
+  g_client_status_code = final_info->final_status;
+  g_client_code_recv = true;
+  gpr_cv_signal(&g_client_code_cv);
+  gpr_mu_unlock(&g_mu);
+}
+
+static void server_destroy_call_elem(grpc_call_element* elem,
+                                     const grpc_call_final_info* final_info,
+                                     grpc_closure* ignored) {
+  gpr_mu_lock(&g_mu);
+  g_server_status_code = final_info->final_status;
+  g_server_code_recv = true;
+  gpr_cv_signal(&g_server_code_cv);
+  gpr_mu_unlock(&g_mu);
+}
+
+static grpc_error* init_channel_elem(grpc_channel_element* elem,
+                                     grpc_channel_element_args* args) {
+  return GRPC_ERROR_NONE;
+}
+
+static void destroy_channel_elem(grpc_channel_element* elem) {}
+
+static const grpc_channel_filter test_client_filter = {
+    grpc_call_next_op,
+    grpc_channel_next_op,
+    0,
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
+    client_destroy_call_elem,
+    0,
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_channel_next_get_info,
+    "client_filter_status_code"};
+
+static const grpc_channel_filter test_server_filter = {
+    grpc_call_next_op,
+    grpc_channel_next_op,
+    0,
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
+    server_destroy_call_elem,
+    0,
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_channel_next_get_info,
+    "server_filter_status_code"};
+
+/*******************************************************************************
+ * Registration
+ */
+
+static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) {
+  grpc_channel_filter* filter = (grpc_channel_filter*)arg;
+  if (g_enable_filter) {
+    // Want to add the filter as close to the end as possible, to make
+    // sure that all of the filters work well together.  However, we
+    // can't add it at the very end, because the
+    // connected_channel/client_channel filter must be the last one.
+    // So we add it right before the last one.
+    grpc_channel_stack_builder_iterator* it =
+        grpc_channel_stack_builder_create_iterator_at_last(builder);
+    GPR_ASSERT(grpc_channel_stack_builder_move_prev(it));
+    const bool retval = grpc_channel_stack_builder_add_filter_before(
+        it, filter, nullptr, nullptr);
+    grpc_channel_stack_builder_iterator_destroy(it);
+    return retval;
+  } else {
+    return true;
+  }
+}
+
+static void init_plugin(void) {
+  gpr_mu_init(&g_mu);
+  gpr_cv_init(&g_client_code_cv);
+  gpr_cv_init(&g_server_code_cv);
+  g_client_code_recv = false;
+  g_server_code_recv = false;
+
+  grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX,
+                                   maybe_add_filter,
+                                   (void*)&test_client_filter);
+  grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX,
+                                   maybe_add_filter,
+                                   (void*)&test_client_filter);
+  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX,
+                                   maybe_add_filter,
+                                   (void*)&test_server_filter);
+}
+
+static void destroy_plugin(void) {
+  gpr_cv_destroy(&g_client_code_cv);
+  gpr_cv_destroy(&g_server_code_cv);
+  gpr_mu_destroy(&g_mu);
+}
+
+void filter_status_code(grpc_end2end_test_config config) {
+  g_enable_filter = true;
+  test_request(config);
+  g_enable_filter = false;
+}
+
+void filter_status_code_pre_init(void) {
+  grpc_register_plugin(init_plugin, destroy_plugin);
+}
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 7e7963916c6..51f0ac7ca61 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -7635,6 +7635,7 @@
       "test/core/end2end/tests/filter_call_init_fails.cc", 
       "test/core/end2end/tests/filter_causes_close.cc", 
       "test/core/end2end/tests/filter_latency.cc", 
+      "test/core/end2end/tests/filter_status_code.cc", 
       "test/core/end2end/tests/graceful_server_shutdown.cc", 
       "test/core/end2end/tests/high_initial_seqno.cc", 
       "test/core/end2end/tests/hpack_size.cc", 
@@ -7716,6 +7717,7 @@
       "test/core/end2end/tests/filter_call_init_fails.cc", 
       "test/core/end2end/tests/filter_causes_close.cc", 
       "test/core/end2end/tests/filter_latency.cc", 
+      "test/core/end2end/tests/filter_status_code.cc", 
       "test/core/end2end/tests/graceful_server_shutdown.cc", 
       "test/core/end2end/tests/high_initial_seqno.cc", 
       "test/core/end2end/tests/hpack_size.cc", 
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index 04345bfb863..6b83cecd41d 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -6821,6 +6821,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_census_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -8159,6 +8182,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_compress_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -9454,6 +9500,28 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_fakesec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -10660,6 +10728,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_fd_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -11927,6 +12018,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -13183,6 +13297,25 @@
       "linux"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+pipe_test", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -14367,6 +14500,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+trace_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -15659,6 +15815,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+workarounds_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -17015,6 +17194,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_http_proxy_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -18413,6 +18616,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_load_reporting_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -19769,6 +19995,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_oauth2_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -21091,7 +21341,7 @@
   }, 
   {
     "args": [
-      "graceful_server_shutdown"
+      "filter_status_code"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21115,7 +21365,7 @@
   }, 
   {
     "args": [
-      "high_initial_seqno"
+      "graceful_server_shutdown"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21139,14 +21389,14 @@
   }, 
   {
     "args": [
-      "idempotent_request"
+      "high_initial_seqno"
     ], 
     "ci_platforms": [
       "windows", 
       "linux", 
       "posix"
     ], 
-    "cpu_cost": 1.0, 
+    "cpu_cost": 0.1, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -21163,7 +21413,7 @@
   }, 
   {
     "args": [
-      "invoke_large_request"
+      "idempotent_request"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21187,7 +21437,7 @@
   }, 
   {
     "args": [
-      "large_metadata"
+      "invoke_large_request"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21211,7 +21461,7 @@
   }, 
   {
     "args": [
-      "load_reporting_hook"
+      "large_metadata"
     ], 
     "ci_platforms": [
       "windows", 
@@ -21235,14 +21485,14 @@
   }, 
   {
     "args": [
-      "max_connection_age"
+      "load_reporting_hook"
     ], 
     "ci_platforms": [
       "windows", 
       "linux", 
       "posix"
     ], 
-    "cpu_cost": 0.1, 
+    "cpu_cost": 1.0, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -21259,7 +21509,31 @@
   }, 
   {
     "args": [
-      "max_message_length"
+      "max_connection_age"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_proxy_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
+  {
+    "args": [
+      "max_message_length"
     ], 
     "ci_platforms": [
       "windows", 
@@ -22193,6 +22467,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -23417,6 +23715,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair+trace_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -24601,6 +24923,32 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [
+      "msan"
+    ], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair_1byte_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -25947,6 +26295,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_ssl_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -27231,6 +27602,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_ssl_proxy_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -28388,6 +28783,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_uds_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -29584,6 +30002,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "inproc_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "high_initial_seqno"
@@ -30621,6 +31062,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_census_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -31936,6 +32400,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_compress_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -33136,7 +33623,7 @@
   }, 
   {
     "args": [
-      "graceful_server_shutdown"
+      "filter_status_code"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33159,7 +33646,7 @@
   }, 
   {
     "args": [
-      "high_initial_seqno"
+      "graceful_server_shutdown"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33182,7 +33669,7 @@
   }, 
   {
     "args": [
-      "hpack_size"
+      "high_initial_seqno"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33205,14 +33692,14 @@
   }, 
   {
     "args": [
-      "idempotent_request"
+      "hpack_size"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 1.0, 
+    "cpu_cost": 0.1, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33228,7 +33715,7 @@
   }, 
   {
     "args": [
-      "invoke_large_request"
+      "idempotent_request"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33251,14 +33738,14 @@
   }, 
   {
     "args": [
-      "keepalive_timeout"
+      "invoke_large_request"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 0.1, 
+    "cpu_cost": 1.0, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33274,14 +33761,14 @@
   }, 
   {
     "args": [
-      "large_metadata"
+      "keepalive_timeout"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 1.0, 
+    "cpu_cost": 0.1, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33297,7 +33784,7 @@
   }, 
   {
     "args": [
-      "load_reporting_hook"
+      "large_metadata"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33320,14 +33807,14 @@
   }, 
   {
     "args": [
-      "max_concurrent_streams"
+      "load_reporting_hook"
     ], 
     "ci_platforms": [
       "linux", 
       "mac", 
       "posix"
     ], 
-    "cpu_cost": 0.1, 
+    "cpu_cost": 1.0, 
     "exclude_configs": [], 
     "exclude_iomgrs": [
       "uv"
@@ -33343,7 +33830,7 @@
   }, 
   {
     "args": [
-      "max_connection_age"
+      "max_concurrent_streams"
     ], 
     "ci_platforms": [
       "linux", 
@@ -33366,7 +33853,30 @@
   }, 
   {
     "args": [
-      "max_message_length"
+      "max_connection_age"
+    ], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_fd_nosec_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
+  {
+    "args": [
+      "max_message_length"
     ], 
     "ci_platforms": [
       "linux", 
@@ -34378,6 +34888,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -35615,6 +36148,25 @@
       "linux"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+pipe_nosec_test", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -36776,6 +37328,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+trace_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -38045,6 +38620,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_full+workarounds_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -39377,6 +39975,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_http_proxy_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -40752,6 +41374,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_load_reporting_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -42012,6 +42657,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_proxy_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -43092,6 +43761,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -44292,6 +44985,30 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair+trace_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -45450,6 +46167,32 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [
+      "msan"
+    ], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_sockpair_1byte_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -46748,6 +47491,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "h2_uds_nosec_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "graceful_server_shutdown"
@@ -47921,6 +48687,29 @@
       "posix"
     ]
   }, 
+  {
+    "args": [
+      "filter_status_code"
+    ], 
+    "ci_platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "inproc_nosec_test", 
+    "platforms": [
+      "windows", 
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "high_initial_seqno"

From 5ca71f29bc3ee804c1d28ae63cbc9a9825a66d0c Mon Sep 17 00:00:00 2001
From: Ken Payson <kpayson@google.com>
Date: Thu, 11 Jan 2018 20:24:06 -0800
Subject: [PATCH 089/161] Fix issue with filter_status_code test for proxy
 tests.

---
 test/core/end2end/tests/filter_status_code.cc | 53 ++++++++++++++-----
 1 file changed, 39 insertions(+), 14 deletions(-)

diff --git a/test/core/end2end/tests/filter_status_code.cc b/test/core/end2end/tests/filter_status_code.cc
index 261ddd93ec5..61c658b95ad 100644
--- a/test/core/end2end/tests/filter_status_code.cc
+++ b/test/core/end2end/tests/filter_status_code.cc
@@ -30,11 +30,14 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel_init.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static bool g_enable_filter = false;
 static gpr_mu g_mu;
+static grpc_call_stack* g_client_call_stack;
+static grpc_call_stack* g_server_call_stack;
 static bool g_client_code_recv;
 static bool g_server_code_recv;
 static gpr_cv g_client_code_cv;
@@ -117,6 +120,8 @@ static void test_request(grpc_end2end_test_config config) {
   int was_cancelled = 2;
 
   gpr_mu_lock(&g_mu);
+  g_client_call_stack = nullptr;
+  g_server_call_stack = nullptr;
   g_client_status_code = GRPC_STATUS_OK;
   g_server_status_code = GRPC_STATUS_OK;
   gpr_mu_unlock(&g_mu);
@@ -127,6 +132,9 @@ static void test_request(grpc_end2end_test_config config) {
       grpc_slice_from_static_string("/foo"),
       get_host_override_slice("foo.test.google.fr", config), deadline, nullptr);
   GPR_ASSERT(c);
+  gpr_mu_lock(&g_mu);
+  g_client_call_stack = grpc_call_get_call_stack(c);
+  gpr_mu_unlock(&g_mu);
 
   grpc_metadata_array_init(&initial_metadata_recv);
   grpc_metadata_array_init(&trailing_metadata_recv);
@@ -168,6 +176,10 @@ static void test_request(grpc_end2end_test_config config) {
   CQ_EXPECT_COMPLETION(cqv, tag(101), 1);
   cq_verify(cqv);
 
+  gpr_mu_lock(&g_mu);
+  g_server_call_stack = grpc_call_get_call_stack(s);
+  gpr_mu_unlock(&g_mu);
+
   memset(ops, 0, sizeof(ops));
   op = ops;
   op->op = GRPC_OP_SEND_INITIAL_METADATA;
@@ -215,49 +227,62 @@ static void test_request(grpc_end2end_test_config config) {
   // Perform checks after test tear-down
   // Guards against the case that there's outstanding channel-related work on a
   // call prior to verification
-  // TODO(https://github.com/grpc/grpc/issues/13915) enable this for windows
-#ifndef GPR_WINDOWS
   gpr_mu_lock(&g_mu);
   if (!g_client_code_recv) {
     GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu,
-                           grpc_timeout_seconds_to_deadline(3)));
+                           grpc_timeout_seconds_to_deadline(3)) == 0);
   }
   if (!g_server_code_recv) {
-    GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu,
-                           grpc_timeout_seconds_to_deadline(3)));
+    GPR_ASSERT(gpr_cv_wait(&g_server_code_cv, &g_mu,
+                           grpc_timeout_seconds_to_deadline(3)) == 0);
   }
   GPR_ASSERT(g_client_status_code == GRPC_STATUS_UNIMPLEMENTED);
   GPR_ASSERT(g_server_status_code == GRPC_STATUS_UNIMPLEMENTED);
   gpr_mu_unlock(&g_mu);
-#endif  // GPR_WINDOWS
 }
 
 /*******************************************************************************
  * Test status_code filter
  */
 
+typedef struct final_status_data {
+  grpc_call_stack* call;
+} final_status_data;
+
 static grpc_error* init_call_elem(grpc_call_element* elem,
                                   const grpc_call_element_args* args) {
+  final_status_data* data = (final_status_data*)elem->call_data;
+  data->call = args->call_stack;
   return GRPC_ERROR_NONE;
 }
 
 static void client_destroy_call_elem(grpc_call_element* elem,
                                      const grpc_call_final_info* final_info,
                                      grpc_closure* ignored) {
+  final_status_data* data = (final_status_data*)elem->call_data;
   gpr_mu_lock(&g_mu);
-  g_client_status_code = final_info->final_status;
-  g_client_code_recv = true;
-  gpr_cv_signal(&g_client_code_cv);
+  // Some fixtures, like proxies, will spawn intermidiate calls
+  // We only want the results from our explicit calls
+  if (data->call == g_client_call_stack) {
+    g_client_status_code = final_info->final_status;
+    g_client_code_recv = true;
+    gpr_cv_signal(&g_client_code_cv);
+  }
   gpr_mu_unlock(&g_mu);
 }
 
 static void server_destroy_call_elem(grpc_call_element* elem,
                                      const grpc_call_final_info* final_info,
                                      grpc_closure* ignored) {
+  final_status_data* data = (final_status_data*)elem->call_data;
   gpr_mu_lock(&g_mu);
-  g_server_status_code = final_info->final_status;
-  g_server_code_recv = true;
-  gpr_cv_signal(&g_server_code_cv);
+  // Some fixtures, like proxies, will spawn intermidiate calls
+  // We only want the results from our explicit calls
+  if (data->call == g_server_call_stack) {
+    g_server_status_code = final_info->final_status;
+    g_server_code_recv = true;
+    gpr_cv_signal(&g_server_code_cv);
+  }
   gpr_mu_unlock(&g_mu);
 }
 
@@ -271,7 +296,7 @@ static void destroy_channel_elem(grpc_channel_element* elem) {}
 static const grpc_channel_filter test_client_filter = {
     grpc_call_next_op,
     grpc_channel_next_op,
-    0,
+    sizeof(final_status_data),
     init_call_elem,
     grpc_call_stack_ignore_set_pollset_or_pollset_set,
     client_destroy_call_elem,
@@ -284,7 +309,7 @@ static const grpc_channel_filter test_client_filter = {
 static const grpc_channel_filter test_server_filter = {
     grpc_call_next_op,
     grpc_channel_next_op,
-    0,
+    sizeof(final_status_data),
     init_call_elem,
     grpc_call_stack_ignore_set_pollset_or_pollset_set,
     server_destroy_call_elem,

From 148700a8ea4b1a6d7ac82f1ad504c4e1eaa4e263 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Fri, 12 Jan 2018 09:18:21 +0100
Subject: [PATCH 090/161] windows needs fflush after fprintf

---
 test/core/fling/client.cc                             |  2 ++
 test/core/network_benchmarks/low_level_ping_pong.cc   |  4 ++++
 test/core/security/create_jwt.cc                      |  5 +++++
 .../core/security/print_google_default_creds_token.cc |  2 ++
 test/core/security/verify_jwt.cc                      |  2 ++
 test/core/support/cpu_test.cc                         |  3 +++
 test/core/support/spinlock_test.cc                    |  4 ++++
 test/core/support/sync_test.cc                        |  4 ++++
 test/core/support/time_test.cc                        | 11 +++++++++++
 test/core/util/debugger_macros.cc                     |  2 ++
 10 files changed, 39 insertions(+)

diff --git a/test/core/fling/client.cc b/test/core/fling/client.cc
index 69fb6dc7c71..28e62e0e839 100644
--- a/test/core/fling/client.cc
+++ b/test/core/fling/client.cc
@@ -186,8 +186,10 @@ int main(int argc, char** argv) {
   }
   if (!sc.name) {
     fprintf(stderr, "unsupported scenario '%s'. Valid are:", scenario_name);
+    fflush(stderr);
     for (i = 0; i < GPR_ARRAY_SIZE(scenarios); i++) {
       fprintf(stderr, " %s", scenarios[i].name);
+      fflush(stderr);
     }
     return 1;
   }
diff --git a/test/core/network_benchmarks/low_level_ping_pong.cc b/test/core/network_benchmarks/low_level_ping_pong.cc
index 96b0745f529..fb982a10fde 100644
--- a/test/core/network_benchmarks/low_level_ping_pong.cc
+++ b/test/core/network_benchmarks/low_level_ping_pong.cc
@@ -535,6 +535,7 @@ void print_usage(char* argv0) {
   fprintf(stderr, "  tcp: fds are endpoints of a TCP connection\n");
   fprintf(stderr, "  socketpair: fds come from socketpair()\n");
   fprintf(stderr, "  pipe: fds come from pipe()\n");
+  fflush(stderr);
 }
 
 typedef struct test_strategy {
@@ -565,6 +566,7 @@ int create_socket(const char* socket_type, fd_pair* client_fds,
     create_sockets_pipe(client_fds, server_fds);
   } else {
     fprintf(stderr, "Invalid socket type %s\n", socket_type);
+    fflush(stderr);
     return -1;
   }
   return 0;
@@ -657,6 +659,7 @@ int main(int argc, char** argv) {
   }
   if (msg_size <= 0) {
     fprintf(stderr, "msg_size must be > 0\n");
+    fflush(stderr);
     print_usage(argv[0]);
     return -1;
   }
@@ -668,6 +671,7 @@ int main(int argc, char** argv) {
   }
   if (strategy == nullptr) {
     fprintf(stderr, "Invalid read strategy %s\n", read_strategy);
+    fflush(stderr);
     return -1;
   }
 
diff --git a/test/core/security/create_jwt.cc b/test/core/security/create_jwt.cc
index 867a8ba5752..56ae9c891c5 100644
--- a/test/core/security/create_jwt.cc
+++ b/test/core/security/create_jwt.cc
@@ -39,6 +39,7 @@ void create_jwt(const char* json_key_file_path, const char* service_url,
   grpc_slice_unref(json_key_data);
   if (!grpc_auth_json_key_is_valid(&key)) {
     fprintf(stderr, "Could not parse json key.\n");
+    fflush(stderr);
     exit(1);
   }
   jwt = grpc_jwt_encode_and_sign(
@@ -47,6 +48,7 @@ void create_jwt(const char* json_key_file_path, const char* service_url,
   grpc_auth_json_key_destruct(&key);
   if (jwt == nullptr) {
     fprintf(stderr, "Could not create JWT.\n");
+    fflush(stderr);
     exit(1);
   }
   fprintf(stdout, "%s\n", jwt);
@@ -72,16 +74,19 @@ int main(int argc, char** argv) {
 
   if (json_key_file_path == nullptr) {
     fprintf(stderr, "Missing --json_key option.\n");
+    fflush(stderr);
     exit(1);
   }
   if (scope != nullptr) {
     if (service_url != nullptr) {
       fprintf(stderr,
               "Options --scope and --service_url are mutually exclusive.\n");
+      fflush(stderr);
       exit(1);
     }
   } else if (service_url == nullptr) {
     fprintf(stderr, "Need one of --service_url or --scope options.\n");
+    fflush(stderr);
     exit(1);
   }
 
diff --git a/test/core/security/print_google_default_creds_token.cc b/test/core/security/print_google_default_creds_token.cc
index b3742f58a89..d71116d8f6a 100644
--- a/test/core/security/print_google_default_creds_token.cc
+++ b/test/core/security/print_google_default_creds_token.cc
@@ -45,6 +45,7 @@ static void on_metadata_response(void* arg, grpc_error* error) {
   synchronizer* sync = static_cast<synchronizer*>(arg);
   if (error != GRPC_ERROR_NONE) {
     fprintf(stderr, "Fetching token failed: %s\n", grpc_error_string(error));
+    fflush(stderr);
   } else {
     char* token;
     GPR_ASSERT(sync->md_array.size == 1);
@@ -81,6 +82,7 @@ int main(int argc, char** argv) {
   creds = grpc_google_default_credentials_create();
   if (creds == nullptr) {
     fprintf(stderr, "\nCould not find default credentials.\n\n");
+    fflush(stderr);
     result = 1;
     goto end;
   }
diff --git a/test/core/security/verify_jwt.cc b/test/core/security/verify_jwt.cc
index e039970c671..5d32ce0cdbe 100644
--- a/test/core/security/verify_jwt.cc
+++ b/test/core/security/verify_jwt.cc
@@ -39,6 +39,7 @@ typedef struct {
 static void print_usage_and_exit(gpr_cmdline* cl, const char* argv0) {
   char* usage = gpr_cmdline_usage_string(cl, argv0);
   fprintf(stderr, "%s", usage);
+  fflush(stderr);
   gpr_free(usage);
   gpr_cmdline_destroy(cl);
   exit(1);
@@ -62,6 +63,7 @@ static void on_jwt_verification_done(void* user_data,
     GPR_ASSERT(claims == nullptr);
     fprintf(stderr, "Verification failed with error %s\n",
             grpc_jwt_verifier_status_to_string(status));
+    fflush(stderr);
   }
 
   gpr_mu_lock(sync->mu);
diff --git a/test/core/support/cpu_test.cc b/test/core/support/cpu_test.cc
index 334c4318e1b..87cdc0fb50b 100644
--- a/test/core/support/cpu_test.cc
+++ b/test/core/support/cpu_test.cc
@@ -119,13 +119,16 @@ static void cpu_test(void) {
   }
   gpr_mu_unlock(&ct.mu);
   fprintf(stderr, "Saw cores [");
+  fflush(stderr);
   for (i = 0; i < ct.ncores; i++) {
     if (ct.used[i]) {
       fprintf(stderr, "%d,", i);
+      fflush(stderr);
       cores_seen++;
     }
   }
   fprintf(stderr, "] (%d/%d)\n", cores_seen, ct.ncores);
+  fflush(stderr);
   gpr_free(ct.used);
 }
 
diff --git a/test/core/support/spinlock_test.cc b/test/core/support/spinlock_test.cc
index 58d5fcd42b9..ea0dbbf7c6b 100644
--- a/test/core/support/spinlock_test.cc
+++ b/test/core/support/spinlock_test.cc
@@ -95,15 +95,18 @@ static void test(const char* name, void (*body)(void* m), int timeout_s,
   gpr_timespec deadline = gpr_time_add(
       start, gpr_time_from_micros((int64_t)timeout_s * 1000000, GPR_TIMESPAN));
   fprintf(stderr, "%s:", name);
+  fflush(stderr);
   while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0) {
     if (iterations < INT64_MAX / 2) iterations <<= 1;
     fprintf(stderr, " %ld", (long)iterations);
+    fflush(stderr);
     m = test_new(10, iterations, incr_step);
     test_create_threads(m, body);
     test_wait(m);
     if (m->counter != m->thread_count * m->iterations * m->incr_step) {
       fprintf(stderr, "counter %ld  threads %d  iterations %ld\n",
               (long)m->counter, m->thread_count, (long)m->iterations);
+      fflush(stderr);
       GPR_ASSERT(0);
     }
     test_destroy(m);
@@ -111,6 +114,7 @@ static void test(const char* name, void (*body)(void* m), int timeout_s,
   time_taken = gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), start);
   fprintf(stderr, " done %lld.%09d s\n", (long long)time_taken.tv_sec,
           (int)time_taken.tv_nsec);
+  fflush(stderr);
 }
 
 /* Increment m->counter on each iteration; then mark thread as done.  */
diff --git a/test/core/support/sync_test.cc b/test/core/support/sync_test.cc
index fb7ec447541..04b9e94cbc3 100644
--- a/test/core/support/sync_test.cc
+++ b/test/core/support/sync_test.cc
@@ -238,9 +238,11 @@ static void test(const char* name, void (*body)(void* m),
   gpr_timespec deadline = gpr_time_add(
       start, gpr_time_from_micros((int64_t)timeout_s * 1000000, GPR_TIMESPAN));
   fprintf(stderr, "%s:", name);
+  fflush(stderr);
   while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0) {
     iterations <<= 1;
     fprintf(stderr, " %ld", (long)iterations);
+    fflush(stderr);
     m = test_new(10, iterations, incr_step);
     if (extra != nullptr) {
       gpr_thd_id id;
@@ -252,6 +254,7 @@ static void test(const char* name, void (*body)(void* m),
     if (m->counter != m->threads * m->iterations * m->incr_step) {
       fprintf(stderr, "counter %ld  threads %d  iterations %ld\n",
               (long)m->counter, m->threads, (long)m->iterations);
+      fflush(stderr);
       GPR_ASSERT(0);
     }
     test_destroy(m);
@@ -259,6 +262,7 @@ static void test(const char* name, void (*body)(void* m),
   time_taken = gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), start);
   fprintf(stderr, " done %lld.%09d s\n", (long long)time_taken.tv_sec,
           (int)time_taken.tv_nsec);
+  fflush(stderr);
 }
 
 /* Increment m->counter on each iteration; then mark thread as done.  */
diff --git a/test/core/support/time_test.cc b/test/core/support/time_test.cc
index 608169274f7..b2b4dce58e0 100644
--- a/test/core/support/time_test.cc
+++ b/test/core/support/time_test.cc
@@ -66,21 +66,28 @@ static void test_values(void) {
 
   x = gpr_inf_future(GPR_CLOCK_REALTIME);
   fprintf(stderr, "far future ");
+  fflush(stderr);
   i_to_s(x.tv_sec, 16, 16, &to_fp, stderr);
   fprintf(stderr, "\n");
   GPR_ASSERT(x.tv_sec == INT64_MAX);
   fprintf(stderr, "far future ");
+  fflush(stderr);
   ts_to_s(x, &to_fp, stderr);
   fprintf(stderr, "\n");
+  fflush(stderr);
 
   x = gpr_inf_past(GPR_CLOCK_REALTIME);
   fprintf(stderr, "far past   ");
+  fflush(stderr);
   i_to_s(x.tv_sec, 16, 16, &to_fp, stderr);
   fprintf(stderr, "\n");
+  fflush(stderr);
   GPR_ASSERT(x.tv_sec == INT64_MIN);
   fprintf(stderr, "far past   ");
+  fflush(stderr);
   ts_to_s(x, &to_fp, stderr);
   fprintf(stderr, "\n");
+  fflush(stderr);
 
   for (i = 1; i != 1000 * 1000 * 1000; i *= 10) {
     x = gpr_time_from_micros(i, GPR_TIMESPAN);
@@ -135,15 +142,19 @@ static void test_add_sub(void) {
         if (gpr_time_cmp(gpr_time_from_micros(sum * k, GPR_TIMESPAN), sumt) !=
             0) {
           fprintf(stderr, "i %d  j %d  sum %d    sumt ", i, j, sum);
+          fflush(stderr);
           ts_to_s(sumt, &to_fp, stderr);
           fprintf(stderr, "\n");
+          fflush(stderr);
           GPR_ASSERT(0);
         }
         if (gpr_time_cmp(gpr_time_from_micros(diff * k, GPR_TIMESPAN), difft) !=
             0) {
           fprintf(stderr, "i %d  j %d  diff %d    diff ", i, j, diff);
+          fflush(stderr);
           ts_to_s(sumt, &to_fp, stderr);
           fprintf(stderr, "\n");
+          fflush(stderr);
           GPR_ASSERT(0);
         }
       }
diff --git a/test/core/util/debugger_macros.cc b/test/core/util/debugger_macros.cc
index f1e4ffd3af9..bb96fc70540 100644
--- a/test/core/util/debugger_macros.cc
+++ b/test/core/util/debugger_macros.cc
@@ -39,6 +39,7 @@ grpc_stream* grpc_transport_stream_from_call(grpc_call* call) {
       grpc_subchannel_call* scc = grpc_client_channel_get_subchannel_call(el);
       if (scc == nullptr) {
         fprintf(stderr, "No subchannel-call");
+        fflush(stderr);
         return nullptr;
       }
       cs = grpc_subchannel_call_get_call_stack(scc);
@@ -46,6 +47,7 @@ grpc_stream* grpc_transport_stream_from_call(grpc_call* call) {
       return grpc_connected_channel_get_stream(el);
     } else {
       fprintf(stderr, "Unrecognized filter: %s", el->filter->name);
+      fflush(stderr);
       return nullptr;
     }
   }

From c9ec2c0888271491eaf425721a72736392f85945 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Fri, 12 Jan 2018 10:16:22 +0100
Subject: [PATCH 091/161] Revert "Stop using std::thread in C++ library since
 it can trigger exceptions"

---
 include/grpc++/impl/codegen/byte_buffer.h     |   4 -
 .../grpc++/impl/codegen/completion_queue.h    |   6 +-
 .../grpc++/impl/codegen/method_handler_impl.h |  17 +-
 include/grpc++/impl/codegen/server_context.h  |   6 +-
 include/grpc++/server.h                       |  21 +--
 include/grpc++/server_builder.h               |  19 ---
 src/cpp/client/secure_credentials.cc          |  14 +-
 src/cpp/server/create_default_thread_pool.cc  |   2 +-
 src/cpp/server/dynamic_thread_pool.cc         |  54 ++----
 src/cpp/server/dynamic_thread_pool.h          |  20 +--
 src/cpp/server/secure_server_credentials.cc   |   7 +-
 src/cpp/server/server_builder.cc              |   7 +-
 src/cpp/server/server_cc.cc                   |  49 ++----
 src/cpp/server/thread_pool_interface.h        |   4 +-
 src/cpp/thread_manager/thread_manager.cc      |  54 ++----
 src/cpp/thread_manager/thread_manager.h       |  29 +---
 test/cpp/end2end/thread_stress_test.cc        | 157 +++++++-----------
 test/cpp/thread_manager/BUILD                 |  31 ----
 .../cpp/thread_manager/thread_manager_test.cc |   8 +-
 19 files changed, 133 insertions(+), 376 deletions(-)
 delete mode 100644 test/cpp/thread_manager/BUILD

diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h
index 9c0246e6174..fe73ce7a83d 100644
--- a/include/grpc++/impl/codegen/byte_buffer.h
+++ b/include/grpc++/impl/codegen/byte_buffer.h
@@ -41,8 +41,6 @@ template <class ServiceType, class RequestType, class ResponseType>
 class RpcMethodHandler;
 template <class ServiceType, class RequestType, class ResponseType>
 class ServerStreamingHandler;
-template <StatusCode code>
-class ErrorMethodHandler;
 template <class R>
 class DeserializeFuncType;
 }  // namespace internal
@@ -109,8 +107,6 @@ class ByteBuffer final {
   friend class internal::RpcMethodHandler;
   template <class ServiceType, class RequestType, class ResponseType>
   friend class internal::ServerStreamingHandler;
-  template <StatusCode code>
-  friend class internal::ErrorMethodHandler;
   template <class R>
   friend class internal::DeserializeFuncType;
 
diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h
index 452eac6646b..b8a78625782 100644
--- a/include/grpc++/impl/codegen/completion_queue.h
+++ b/include/grpc++/impl/codegen/completion_queue.h
@@ -78,8 +78,7 @@ template <class ServiceType, class RequestType, class ResponseType>
 class ServerStreamingHandler;
 template <class ServiceType, class RequestType, class ResponseType>
 class BidiStreamingHandler;
-template <StatusCode code>
-class ErrorMethodHandler;
+class UnknownMethodHandler;
 template <class Streamer, bool WriteNeeded>
 class TemplatedBidiStreamingHandler;
 template <class InputMessage, class OutputMessage>
@@ -222,8 +221,7 @@ class CompletionQueue : private GrpcLibraryCodegen {
   friend class ::grpc::internal::ServerStreamingHandler;
   template <class Streamer, bool WriteNeeded>
   friend class ::grpc::internal::TemplatedBidiStreamingHandler;
-  template <StatusCode code>
-  friend class ::grpc::internal::ErrorMethodHandler;
+  friend class ::grpc::internal::UnknownMethodHandler;
   friend class ::grpc::Server;
   friend class ::grpc::ServerContext;
   friend class ::grpc::ServerInterface;
diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h
index 93b7826e8fc..c0af4ca1307 100644
--- a/include/grpc++/impl/codegen/method_handler_impl.h
+++ b/include/grpc++/impl/codegen/method_handler_impl.h
@@ -242,14 +242,12 @@ class SplitServerStreamingHandler
             ServerSplitStreamer<RequestType, ResponseType>, false>(func) {}
 };
 
-/// General method handler class for errors that prevent real method use
-/// e.g., handle unknown method by returning UNIMPLEMENTED error.
-template <StatusCode code>
-class ErrorMethodHandler : public MethodHandler {
+/// Handle unknown method by returning UNIMPLEMENTED error.
+class UnknownMethodHandler : public MethodHandler {
  public:
   template <class T>
   static void FillOps(ServerContext* context, T* ops) {
-    Status status(code, "");
+    Status status(StatusCode::UNIMPLEMENTED, "");
     if (!context->sent_initial_metadata_) {
       ops->SendInitialMetadata(context->initial_metadata_,
                                context->initial_metadata_flags());
@@ -266,18 +264,9 @@ class ErrorMethodHandler : public MethodHandler {
     FillOps(param.server_context, &ops);
     param.call->PerformOps(&ops);
     param.call->cq()->Pluck(&ops);
-    // We also have to destroy any request payload in the handler parameter
-    ByteBuffer* payload = param.request.bbuf_ptr();
-    if (payload != nullptr) {
-      payload->Clear();
-    }
   }
 };
 
-typedef ErrorMethodHandler<StatusCode::UNIMPLEMENTED> UnknownMethodHandler;
-typedef ErrorMethodHandler<StatusCode::RESOURCE_EXHAUSTED>
-    ResourceExhaustedHandler;
-
 }  // namespace internal
 }  // namespace grpc
 
diff --git a/include/grpc++/impl/codegen/server_context.h b/include/grpc++/impl/codegen/server_context.h
index 9f20335a2a2..a2d6967bf84 100644
--- a/include/grpc++/impl/codegen/server_context.h
+++ b/include/grpc++/impl/codegen/server_context.h
@@ -63,8 +63,7 @@ template <class ServiceType, class RequestType, class ResponseType>
 class ServerStreamingHandler;
 template <class ServiceType, class RequestType, class ResponseType>
 class BidiStreamingHandler;
-template <StatusCode code>
-class ErrorMethodHandler;
+class UnknownMethodHandler;
 template <class Streamer, bool WriteNeeded>
 class TemplatedBidiStreamingHandler;
 class Call;
@@ -256,8 +255,7 @@ class ServerContext {
   friend class ::grpc::internal::ServerStreamingHandler;
   template <class Streamer, bool WriteNeeded>
   friend class ::grpc::internal::TemplatedBidiStreamingHandler;
-  template <StatusCode code>
-  friend class ::grpc::internal::ErrorMethodHandler;
+  friend class ::grpc::internal::UnknownMethodHandler;
   friend class ::grpc::ClientContext;
 
   /// Prevent copying.
diff --git a/include/grpc++/server.h b/include/grpc++/server.h
index cf590185d1a..01c4a60d216 100644
--- a/include/grpc++/server.h
+++ b/include/grpc++/server.h
@@ -35,7 +35,6 @@
 #include <grpc++/support/config.h>
 #include <grpc++/support/status.h>
 #include <grpc/compression.h>
-#include <grpc/support/thd.h>
 
 struct grpc_server;
 
@@ -139,20 +138,10 @@ class Server final : public ServerInterface, private GrpcLibraryCodegen {
   ///
   /// \param sync_cq_timeout_msec The timeout to use when calling AsyncNext() on
   /// server completion queues passed via sync_server_cqs param.
-  ///
-  /// \param thread_creator The thread creation function for the sync
-  /// server. Typically gpr_thd_new
-  ///
-  /// \param thread_joiner The thread joining function for the sync
-  /// server. Typically gpr_thd_join
   Server(int max_message_size, ChannelArguments* args,
          std::shared_ptr<std::vector<std::unique_ptr<ServerCompletionQueue>>>
              sync_server_cqs,
-         int min_pollers, int max_pollers, int sync_cq_timeout_msec,
-         std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
-                           const gpr_thd_options*)>
-             thread_creator,
-         std::function<void(gpr_thd_id)> thread_joiner);
+         int min_pollers, int max_pollers, int sync_cq_timeout_msec);
 
   /// Register a service. This call does not take ownership of the service.
   /// The service must exist for the lifetime of the Server instance.
@@ -231,14 +220,6 @@ class Server final : public ServerInterface, private GrpcLibraryCodegen {
 
   std::unique_ptr<HealthCheckServiceInterface> health_check_service_;
   bool health_check_service_disabled_;
-
-  std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
-                    const gpr_thd_options*)>
-      thread_creator_;
-  std::function<void(gpr_thd_id)> thread_joiner_;
-
-  // A special handler for resource exhausted in sync case
-  std::unique_ptr<internal::MethodHandler> resource_exhausted_handler_;
 };
 
 }  // namespace grpc
diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h
index 25bbacbbc7a..e2bae4b41fa 100644
--- a/include/grpc++/server_builder.h
+++ b/include/grpc++/server_builder.h
@@ -20,7 +20,6 @@
 #define GRPCXX_SERVER_BUILDER_H
 
 #include <climits>
-#include <functional>
 #include <map>
 #include <memory>
 #include <vector>
@@ -31,7 +30,6 @@
 #include <grpc++/support/config.h>
 #include <grpc/compression.h>
 #include <grpc/support/cpu.h>
-#include <grpc/support/thd.h>
 #include <grpc/support/useful.h>
 #include <grpc/support/workaround_list.h>
 
@@ -49,7 +47,6 @@ class Service;
 
 namespace testing {
 class ServerBuilderPluginTest;
-class ServerBuilderThreadCreatorOverrideTest;
 }  // namespace testing
 
 /// A builder class for the creation and startup of \a grpc::Server instances.
@@ -216,17 +213,6 @@ class ServerBuilder {
 
  private:
   friend class ::grpc::testing::ServerBuilderPluginTest;
-  friend class ::grpc::testing::ServerBuilderThreadCreatorOverrideTest;
-
-  ServerBuilder& SetThreadFunctions(
-      std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
-                        const gpr_thd_options*)>
-          thread_creator,
-      std::function<void(gpr_thd_id)> thread_joiner) {
-    thread_creator_ = thread_creator;
-    thread_joiner_ = thread_joiner;
-    return *this;
-  }
 
   struct Port {
     grpc::string addr;
@@ -286,11 +272,6 @@ class ServerBuilder {
     grpc_compression_algorithm algorithm;
   } maybe_default_compression_algorithm_;
   uint32_t enabled_compression_algorithms_bitset_;
-
-  std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
-                    const gpr_thd_options*)>
-      thread_creator_;
-  std::function<void(gpr_thd_id)> thread_joiner_;
 };
 
 }  // namespace grpc
diff --git a/src/cpp/client/secure_credentials.cc b/src/cpp/client/secure_credentials.cc
index 94519d817bf..4fb128d98b1 100644
--- a/src/cpp/client/secure_credentials.cc
+++ b/src/cpp/client/secure_credentials.cc
@@ -189,16 +189,10 @@ int MetadataCredentialsPluginWrapper::GetMetadata(
   }
   if (w->plugin_->IsBlocking()) {
     // Asynchronous return.
-    if (w->thread_pool_->Add(std::bind(
-            &MetadataCredentialsPluginWrapper::InvokePlugin, w, context, cb,
-            user_data, nullptr, nullptr, nullptr, nullptr))) {
-      return 0;
-    } else {
-      *num_creds_md = 0;
-      *status = GRPC_STATUS_RESOURCE_EXHAUSTED;
-      *error_details = nullptr;
-      return true;
-    }
+    w->thread_pool_->Add(
+        std::bind(&MetadataCredentialsPluginWrapper::InvokePlugin, w, context,
+                  cb, user_data, nullptr, nullptr, nullptr, nullptr));
+    return 0;
   } else {
     // Synchronous return.
     w->InvokePlugin(context, cb, user_data, creds_md, num_creds_md, status,
diff --git a/src/cpp/server/create_default_thread_pool.cc b/src/cpp/server/create_default_thread_pool.cc
index 2d2abbe9d1c..8ca3e32c2fb 100644
--- a/src/cpp/server/create_default_thread_pool.cc
+++ b/src/cpp/server/create_default_thread_pool.cc
@@ -28,7 +28,7 @@ namespace {
 ThreadPoolInterface* CreateDefaultThreadPoolImpl() {
   int cores = gpr_cpu_num_cores();
   if (!cores) cores = 4;
-  return new DynamicThreadPool(cores, gpr_thd_new, gpr_thd_join);
+  return new DynamicThreadPool(cores);
 }
 
 CreateThreadPoolFunc g_ctp_impl = CreateDefaultThreadPoolImpl;
diff --git a/src/cpp/server/dynamic_thread_pool.cc b/src/cpp/server/dynamic_thread_pool.cc
index d0e62313f68..81c78fe739b 100644
--- a/src/cpp/server/dynamic_thread_pool.cc
+++ b/src/cpp/server/dynamic_thread_pool.cc
@@ -19,32 +19,19 @@
 #include "src/cpp/server/dynamic_thread_pool.h"
 
 #include <mutex>
+#include <thread>
 
 #include <grpc/support/log.h>
-#include <grpc/support/thd.h>
 
 namespace grpc {
 
-DynamicThreadPool::DynamicThread::DynamicThread(DynamicThreadPool* pool,
-                                                bool* valid)
-    : pool_(pool) {
-  gpr_thd_options opt = gpr_thd_options_default();
-  gpr_thd_options_set_joinable(&opt);
-
-  std::lock_guard<std::mutex> l(dt_mu_);
-  valid_ = *valid = pool->thread_creator_(
-      &thd_, "dynamic thread",
-      [](void* th) {
-        reinterpret_cast<DynamicThreadPool::DynamicThread*>(th)->ThreadFunc();
-      },
-      this, &opt);
-}
-
+DynamicThreadPool::DynamicThread::DynamicThread(DynamicThreadPool* pool)
+    : pool_(pool),
+      thd_(new std::thread(&DynamicThreadPool::DynamicThread::ThreadFunc,
+                           this)) {}
 DynamicThreadPool::DynamicThread::~DynamicThread() {
-  std::lock_guard<std::mutex> l(dt_mu_);
-  if (valid_) {
-    pool_->thread_joiner_(thd_);
-  }
+  thd_->join();
+  thd_.reset();
 }
 
 void DynamicThreadPool::DynamicThread::ThreadFunc() {
@@ -86,26 +73,15 @@ void DynamicThreadPool::ThreadFunc() {
   }
 }
 
-DynamicThreadPool::DynamicThreadPool(
-    int reserve_threads,
-    std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
-                      const gpr_thd_options*)>
-        thread_creator,
-    std::function<void(gpr_thd_id)> thread_joiner)
+DynamicThreadPool::DynamicThreadPool(int reserve_threads)
     : shutdown_(false),
       reserve_threads_(reserve_threads),
       nthreads_(0),
-      threads_waiting_(0),
-      thread_creator_(thread_creator),
-      thread_joiner_(thread_joiner) {
+      threads_waiting_(0) {
   for (int i = 0; i < reserve_threads_; i++) {
     std::lock_guard<std::mutex> lock(mu_);
     nthreads_++;
-    bool valid;
-    auto* th = new DynamicThread(this, &valid);
-    if (!valid) {
-      delete th;
-    }
+    new DynamicThread(this);
   }
 }
 
@@ -125,7 +101,7 @@ DynamicThreadPool::~DynamicThreadPool() {
   ReapThreads(&dead_threads_);
 }
 
-bool DynamicThreadPool::Add(const std::function<void()>& callback) {
+void DynamicThreadPool::Add(const std::function<void()>& callback) {
   std::lock_guard<std::mutex> lock(mu_);
   // Add works to the callbacks list
   callbacks_.push(callback);
@@ -133,12 +109,7 @@ bool DynamicThreadPool::Add(const std::function<void()>& callback) {
   if (threads_waiting_ == 0) {
     // Kick off a new thread
     nthreads_++;
-    bool valid;
-    auto* th = new DynamicThread(this, &valid);
-    if (!valid) {
-      delete th;
-      return false;
-    }
+    new DynamicThread(this);
   } else {
     cv_.notify_one();
   }
@@ -146,7 +117,6 @@ bool DynamicThreadPool::Add(const std::function<void()>& callback) {
   if (!dead_threads_.empty()) {
     ReapThreads(&dead_threads_);
   }
-  return true;
 }
 
 }  // namespace grpc
diff --git a/src/cpp/server/dynamic_thread_pool.h b/src/cpp/server/dynamic_thread_pool.h
index 75d31cd9083..9237c6e5cac 100644
--- a/src/cpp/server/dynamic_thread_pool.h
+++ b/src/cpp/server/dynamic_thread_pool.h
@@ -24,9 +24,9 @@
 #include <memory>
 #include <mutex>
 #include <queue>
+#include <thread>
 
 #include <grpc++/support/config.h>
-#include <grpc/support/thd.h>
 
 #include "src/cpp/server/thread_pool_interface.h"
 
@@ -34,26 +34,20 @@ namespace grpc {
 
 class DynamicThreadPool final : public ThreadPoolInterface {
  public:
-  DynamicThreadPool(int reserve_threads,
-                    std::function<int(gpr_thd_id*, const char*, void (*)(void*),
-                                      void*, const gpr_thd_options*)>
-                        thread_creator,
-                    std::function<void(gpr_thd_id)> thread_joiner);
+  explicit DynamicThreadPool(int reserve_threads);
   ~DynamicThreadPool();
 
-  bool Add(const std::function<void()>& callback) override;
+  void Add(const std::function<void()>& callback) override;
 
  private:
   class DynamicThread {
    public:
-    DynamicThread(DynamicThreadPool* pool, bool* valid);
+    DynamicThread(DynamicThreadPool* pool);
     ~DynamicThread();
 
    private:
     DynamicThreadPool* pool_;
-    std::mutex dt_mu_;
-    gpr_thd_id thd_;
-    bool valid_;
+    std::unique_ptr<std::thread> thd_;
     void ThreadFunc();
   };
   std::mutex mu_;
@@ -65,10 +59,6 @@ class DynamicThreadPool final : public ThreadPoolInterface {
   int nthreads_;
   int threads_waiting_;
   std::list<DynamicThread*> dead_threads_;
-  std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
-                    const gpr_thd_options*)>
-      thread_creator_;
-  std::function<void(gpr_thd_id)> thread_joiner_;
 
   void ThreadFunc();
   static void ReapThreads(std::list<DynamicThread*>* tlist);
diff --git a/src/cpp/server/secure_server_credentials.cc b/src/cpp/server/secure_server_credentials.cc
index fa08a6200f1..0fbe4ccd18b 100644
--- a/src/cpp/server/secure_server_credentials.cc
+++ b/src/cpp/server/secure_server_credentials.cc
@@ -43,14 +43,9 @@ void AuthMetadataProcessorAyncWrapper::Process(
     return;
   }
   if (w->processor_->IsBlocking()) {
-    bool added = w->thread_pool_->Add(
+    w->thread_pool_->Add(
         std::bind(&AuthMetadataProcessorAyncWrapper::InvokeProcessor, w,
                   context, md, num_md, cb, user_data));
-    if (!added) {
-      // no thread available, so fail with temporary resource unavailability
-      cb(user_data, nullptr, 0, nullptr, 0, GRPC_STATUS_UNAVAILABLE, nullptr);
-      return;
-    }
   } else {
     // invoke directly.
     w->InvokeProcessor(context, md, num_md, cb, user_data);
diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc
index d91ee7f4e3a..200e477822c 100644
--- a/src/cpp/server/server_builder.cc
+++ b/src/cpp/server/server_builder.cc
@@ -23,7 +23,6 @@
 #include <grpc++/server.h>
 #include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
-#include <grpc/support/thd.h>
 #include <grpc/support/useful.h>
 
 #include "src/cpp/server/thread_pool_interface.h"
@@ -44,9 +43,7 @@ ServerBuilder::ServerBuilder()
       max_send_message_size_(-1),
       sync_server_settings_(SyncServerSettings()),
       resource_quota_(nullptr),
-      generic_service_(nullptr),
-      thread_creator_(gpr_thd_new),
-      thread_joiner_(gpr_thd_join) {
+      generic_service_(nullptr) {
   gpr_once_init(&once_init_plugin_list, do_plugin_list_init);
   for (auto it = g_plugin_factory_list->begin();
        it != g_plugin_factory_list->end(); it++) {
@@ -265,7 +262,7 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
   std::unique_ptr<Server> server(new Server(
       max_receive_message_size_, &args, sync_server_cqs,
       sync_server_settings_.min_pollers, sync_server_settings_.max_pollers,
-      sync_server_settings_.cq_timeout_msec, thread_creator_, thread_joiner_));
+      sync_server_settings_.cq_timeout_msec));
 
   if (has_sync_methods) {
     // This is a Sync server
diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc
index 02a663d660e..4f8f4e06fcc 100644
--- a/src/cpp/server/server_cc.cc
+++ b/src/cpp/server/server_cc.cc
@@ -36,7 +36,6 @@
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/thd.h>
 
 #include "src/core/ext/transport/inproc/inproc_transport.h"
 #include "src/core/lib/profiling/timers.h"
@@ -196,10 +195,8 @@ class Server::SyncRequest final : public internal::CompletionQueueTag {
           call_(mrd->call_, server, &cq_, server->max_receive_message_size()),
           ctx_(mrd->deadline_, &mrd->request_metadata_),
           has_request_payload_(mrd->has_request_payload_),
-          request_payload_(has_request_payload_ ? mrd->request_payload_
-                                                : nullptr),
-          method_(mrd->method_),
-          server_(server) {
+          request_payload_(mrd->request_payload_),
+          method_(mrd->method_) {
       ctx_.set_call(mrd->call_);
       ctx_.cq_ = &cq_;
       GPR_ASSERT(mrd->in_flight_);
@@ -213,13 +210,10 @@ class Server::SyncRequest final : public internal::CompletionQueueTag {
       }
     }
 
-    void Run(std::shared_ptr<GlobalCallbacks> global_callbacks,
-             bool resources) {
+    void Run(std::shared_ptr<GlobalCallbacks> global_callbacks) {
       ctx_.BeginCompletionOp(&call_);
       global_callbacks->PreSynchronousRequest(&ctx_);
-      auto* handler = resources ? method_->handler()
-                                : server_->resource_exhausted_handler_.get();
-      handler->RunHandler(internal::MethodHandler::HandlerParameter(
+      method_->handler()->RunHandler(internal::MethodHandler::HandlerParameter(
           &call_, &ctx_, request_payload_));
       global_callbacks->PostSynchronousRequest(&ctx_);
       request_payload_ = nullptr;
@@ -241,7 +235,6 @@ class Server::SyncRequest final : public internal::CompletionQueueTag {
     const bool has_request_payload_;
     grpc_byte_buffer* request_payload_;
     internal::RpcServiceMethod* const method_;
-    Server* server_;
   };
 
  private:
@@ -262,15 +255,11 @@ class Server::SyncRequest final : public internal::CompletionQueueTag {
 // appropriate RPC handlers
 class Server::SyncRequestThreadManager : public ThreadManager {
  public:
-  SyncRequestThreadManager(
-      Server* server, CompletionQueue* server_cq,
-      std::shared_ptr<GlobalCallbacks> global_callbacks, int min_pollers,
-      int max_pollers, int cq_timeout_msec,
-      std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
-                        const gpr_thd_options*)>
-          thread_creator,
-      std::function<void(gpr_thd_id)> thread_joiner)
-      : ThreadManager(min_pollers, max_pollers, thread_creator, thread_joiner),
+  SyncRequestThreadManager(Server* server, CompletionQueue* server_cq,
+                           std::shared_ptr<GlobalCallbacks> global_callbacks,
+                           int min_pollers, int max_pollers,
+                           int cq_timeout_msec)
+      : ThreadManager(min_pollers, max_pollers),
         server_(server),
         server_cq_(server_cq),
         cq_timeout_msec_(cq_timeout_msec),
@@ -296,7 +285,7 @@ class Server::SyncRequestThreadManager : public ThreadManager {
     GPR_UNREACHABLE_CODE(return TIMEOUT);
   }
 
-  void DoWork(void* tag, bool ok, bool resources) override {
+  void DoWork(void* tag, bool ok) override {
     SyncRequest* sync_req = static_cast<SyncRequest*>(tag);
 
     if (!sync_req) {
@@ -316,7 +305,7 @@ class Server::SyncRequestThreadManager : public ThreadManager {
       }
 
       GPR_TIMER_SCOPE("cd.Run()", 0);
-      cd.Run(global_callbacks_, resources);
+      cd.Run(global_callbacks_);
     }
     // TODO (sreek) If ok is false here (which it isn't in case of
     // grpc_request_registered_call), we should still re-queue the request
@@ -378,11 +367,7 @@ Server::Server(
     int max_receive_message_size, ChannelArguments* args,
     std::shared_ptr<std::vector<std::unique_ptr<ServerCompletionQueue>>>
         sync_server_cqs,
-    int min_pollers, int max_pollers, int sync_cq_timeout_msec,
-    std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
-                      const gpr_thd_options*)>
-        thread_creator,
-    std::function<void(gpr_thd_id)> thread_joiner)
+    int min_pollers, int max_pollers, int sync_cq_timeout_msec)
     : max_receive_message_size_(max_receive_message_size),
       sync_server_cqs_(sync_server_cqs),
       started_(false),
@@ -391,9 +376,7 @@ Server::Server(
       has_generic_service_(false),
       server_(nullptr),
       server_initializer_(new ServerInitializer(this)),
-      health_check_service_disabled_(false),
-      thread_creator_(thread_creator),
-      thread_joiner_(thread_joiner) {
+      health_check_service_disabled_(false) {
   g_gli_initializer.summon();
   gpr_once_init(&g_once_init_callbacks, InitGlobalCallbacks);
   global_callbacks_ = g_callbacks;
@@ -403,7 +386,7 @@ Server::Server(
        it++) {
     sync_req_mgrs_.emplace_back(new SyncRequestThreadManager(
         this, (*it).get(), global_callbacks_, min_pollers, max_pollers,
-        sync_cq_timeout_msec, thread_creator_, thread_joiner_));
+        sync_cq_timeout_msec));
   }
 
   grpc_channel_args channel_args;
@@ -566,10 +549,6 @@ void Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) {
     }
   }
 
-  if (!sync_server_cqs_->empty()) {
-    resource_exhausted_handler_.reset(new internal::ResourceExhaustedHandler);
-  }
-
   for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) {
     (*it)->Start();
   }
diff --git a/src/cpp/server/thread_pool_interface.h b/src/cpp/server/thread_pool_interface.h
index 656e6673f12..028842a776f 100644
--- a/src/cpp/server/thread_pool_interface.h
+++ b/src/cpp/server/thread_pool_interface.h
@@ -29,9 +29,7 @@ class ThreadPoolInterface {
   virtual ~ThreadPoolInterface() {}
 
   // Schedule the given callback for execution.
-  // Return true on success, false on failure
-  virtual bool Add(const std::function<void()>& callback)
-      GRPC_MUST_USE_RESULT = 0;
+  virtual void Add(const std::function<void()>& callback) = 0;
 };
 
 // Allows different codebases to use their own thread pool impls
diff --git a/src/cpp/thread_manager/thread_manager.cc b/src/cpp/thread_manager/thread_manager.cc
index 107c60f4ebe..23264f1b5bc 100644
--- a/src/cpp/thread_manager/thread_manager.cc
+++ b/src/cpp/thread_manager/thread_manager.cc
@@ -20,26 +20,18 @@
 
 #include <climits>
 #include <mutex>
+#include <thread>
 
 #include <grpc/support/log.h>
-#include <grpc/support/thd.h>
 
 namespace grpc {
 
-ThreadManager::WorkerThread::WorkerThread(ThreadManager* thd_mgr, bool* valid)
+ThreadManager::WorkerThread::WorkerThread(ThreadManager* thd_mgr)
     : thd_mgr_(thd_mgr) {
-  gpr_thd_options opt = gpr_thd_options_default();
-  gpr_thd_options_set_joinable(&opt);
-
   // Make thread creation exclusive with respect to its join happening in
   // ~WorkerThread().
   std::lock_guard<std::mutex> lock(wt_mu_);
-  *valid = valid_ = thd_mgr->thread_creator_(
-      &thd_, "worker thread",
-      [](void* th) {
-        reinterpret_cast<ThreadManager::WorkerThread*>(th)->Run();
-      },
-      this, &opt);
+  thd_ = std::thread(&ThreadManager::WorkerThread::Run, this);
 }
 
 void ThreadManager::WorkerThread::Run() {
@@ -50,24 +42,15 @@ void ThreadManager::WorkerThread::Run() {
 ThreadManager::WorkerThread::~WorkerThread() {
   // Don't join until the thread is fully constructed.
   std::lock_guard<std::mutex> lock(wt_mu_);
-  if (valid_) {
-    thd_mgr_->thread_joiner_(thd_);
-  }
+  thd_.join();
 }
 
-ThreadManager::ThreadManager(
-    int min_pollers, int max_pollers,
-    std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
-                      const gpr_thd_options*)>
-        thread_creator,
-    std::function<void(gpr_thd_id)> thread_joiner)
+ThreadManager::ThreadManager(int min_pollers, int max_pollers)
     : shutdown_(false),
       num_pollers_(0),
       min_pollers_(min_pollers),
       max_pollers_(max_pollers == -1 ? INT_MAX : max_pollers),
-      num_threads_(0),
-      thread_creator_(thread_creator),
-      thread_joiner_(thread_joiner) {}
+      num_threads_(0) {}
 
 ThreadManager::~ThreadManager() {
   {
@@ -128,9 +111,7 @@ void ThreadManager::Initialize() {
 
   for (int i = 0; i < min_pollers_; i++) {
     // Create a new thread (which ends up calling the MainWorkLoop() function
-    bool valid;
-    new WorkerThread(this, &valid);
-    GPR_ASSERT(valid);  // we need to have at least this minimum
+    new WorkerThread(this);
   }
 }
 
@@ -157,27 +138,18 @@ void ThreadManager::MainWorkLoop() {
       case WORK_FOUND:
         // If we got work and there are now insufficient pollers, start a new
         // one
-        bool resources;
         if (!shutdown_ && num_pollers_ < min_pollers_) {
-          bool valid;
+          num_pollers_++;
+          num_threads_++;
           // Drop lock before spawning thread to avoid contention
           lock.unlock();
-          auto* th = new WorkerThread(this, &valid);
-          lock.lock();
-          if (valid) {
-            num_pollers_++;
-            num_threads_++;
-          } else {
-            delete th;
-          }
-          resources = (num_pollers_ > 0);
+          new WorkerThread(this);
         } else {
-          resources = true;
+          // Drop lock for consistency with above branch
+          lock.unlock();
         }
-        // Drop lock before any application work
-        lock.unlock();
         // Lock is always released at this point - do the application work
-        DoWork(tag, ok, resources);
+        DoWork(tag, ok);
         // Take the lock again to check post conditions
         lock.lock();
         // If we're shutdown, we should finish at this point.
diff --git a/src/cpp/thread_manager/thread_manager.h b/src/cpp/thread_manager/thread_manager.h
index c1783baa607..a206e0bd8a5 100644
--- a/src/cpp/thread_manager/thread_manager.h
+++ b/src/cpp/thread_manager/thread_manager.h
@@ -20,23 +20,18 @@
 #define GRPC_INTERNAL_CPP_THREAD_MANAGER_H
 
 #include <condition_variable>
-#include <functional>
 #include <list>
 #include <memory>
 #include <mutex>
+#include <thread>
 
 #include <grpc++/support/config.h>
-#include <grpc/support/thd.h>
 
 namespace grpc {
 
 class ThreadManager {
  public:
-  ThreadManager(int min_pollers, int max_pollers,
-                std::function<int(gpr_thd_id*, const char*, void (*)(void*),
-                                  void*, const gpr_thd_options*)>
-                    thread_creator,
-                std::function<void(gpr_thd_id)> thread_joiner);
+  explicit ThreadManager(int min_pollers, int max_pollers);
   virtual ~ThreadManager();
 
   // Initializes and Starts the Rpc Manager threads
@@ -55,8 +50,6 @@ class ThreadManager {
   //  - ThreadManager does not interpret the values of 'tag' and 'ok'
   //  - ThreadManager WILL call DoWork() and pass '*tag' and 'ok' as input to
   //    DoWork()
-  //  - ThreadManager will also pass DoWork a bool saying if there are actually
-  //    resources to do the work
   //
   // If the return value is SHUTDOWN:,
   //  - ThreadManager WILL NOT call DoWork() and terminates the thead
@@ -76,7 +69,7 @@ class ThreadManager {
   // The implementation of DoWork() should also do any setup needed to ensure
   // that the next call to PollForWork() (not necessarily by the current thread)
   // actually finds some work
-  virtual void DoWork(void* tag, bool ok, bool resources) = 0;
+  virtual void DoWork(void* tag, bool ok) = 0;
 
   // Mark the ThreadManager as shutdown and begin draining the work. This is a
   // non-blocking call and the caller should call Wait(), a blocking call which
@@ -91,15 +84,15 @@ class ThreadManager {
   virtual void Wait();
 
  private:
-  // Helper wrapper class around thread. This takes a ThreadManager object
-  // and starts a new thread to calls the Run() function.
+  // Helper wrapper class around std::thread. This takes a ThreadManager object
+  // and starts a new std::thread to calls the Run() function.
   //
   // The Run() function calls ThreadManager::MainWorkLoop() function and once
   // that completes, it marks the WorkerThread completed by calling
   // ThreadManager::MarkAsCompleted()
   class WorkerThread {
    public:
-    WorkerThread(ThreadManager* thd_mgr, bool* valid);
+    WorkerThread(ThreadManager* thd_mgr);
     ~WorkerThread();
 
    private:
@@ -109,8 +102,7 @@ class ThreadManager {
 
     ThreadManager* const thd_mgr_;
     std::mutex wt_mu_;
-    gpr_thd_id thd_;
-    bool valid_;
+    std::thread thd_;
   };
 
   // The main funtion in ThreadManager
@@ -137,13 +129,6 @@ class ThreadManager {
   // currently polling i.e num_pollers_)
   int num_threads_;
 
-  // Functions for creating/joining threads. Normally, these should
-  // be gpr_thd_new/gpr_thd_join but they are overridable
-  std::function<int(gpr_thd_id*, const char*, void (*)(void*), void*,
-                    const gpr_thd_options*)>
-      thread_creator_;
-  std::function<void(gpr_thd_id)> thread_joiner_;
-
   std::mutex list_mu_;
   std::list<WorkerThread*> completed_threads_;
 };
diff --git a/test/cpp/end2end/thread_stress_test.cc b/test/cpp/end2end/thread_stress_test.cc
index fd43c8f584c..90b2eddbbb4 100644
--- a/test/cpp/end2end/thread_stress_test.cc
+++ b/test/cpp/end2end/thread_stress_test.cc
@@ -26,7 +26,6 @@
 #include <grpc++/server_builder.h>
 #include <grpc++/server_context.h>
 #include <grpc/grpc.h>
-#include <grpc/support/atm.h>
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
 
@@ -53,13 +52,63 @@ namespace testing {
 
 class TestServiceImpl : public ::grpc::testing::EchoTestService::Service {
  public:
-  TestServiceImpl() {}
+  TestServiceImpl() : signal_client_(false) {}
 
   Status Echo(ServerContext* context, const EchoRequest* request,
               EchoResponse* response) override {
     response->set_message(request->message());
     return Status::OK;
   }
+
+  // Unimplemented is left unimplemented to test the returned error.
+
+  Status RequestStream(ServerContext* context,
+                       ServerReader<EchoRequest>* reader,
+                       EchoResponse* response) override {
+    EchoRequest request;
+    response->set_message("");
+    while (reader->Read(&request)) {
+      response->mutable_message()->append(request.message());
+    }
+    return Status::OK;
+  }
+
+  // Return 3 messages.
+  // TODO(yangg) make it generic by adding a parameter into EchoRequest
+  Status ResponseStream(ServerContext* context, const EchoRequest* request,
+                        ServerWriter<EchoResponse>* writer) override {
+    EchoResponse response;
+    response.set_message(request->message() + "0");
+    writer->Write(response);
+    response.set_message(request->message() + "1");
+    writer->Write(response);
+    response.set_message(request->message() + "2");
+    writer->Write(response);
+
+    return Status::OK;
+  }
+
+  Status BidiStream(
+      ServerContext* context,
+      ServerReaderWriter<EchoResponse, EchoRequest>* stream) override {
+    EchoRequest request;
+    EchoResponse response;
+    while (stream->Read(&request)) {
+      gpr_log(GPR_INFO, "recv msg %s", request.message().c_str());
+      response.set_message(request.message());
+      stream->Write(response);
+    }
+    return Status::OK;
+  }
+
+  bool signal_client() {
+    std::unique_lock<std::mutex> lock(mu_);
+    return signal_client_;
+  }
+
+ private:
+  bool signal_client_;
+  std::mutex mu_;
 };
 
 template <class Service>
@@ -70,15 +119,10 @@ class CommonStressTest {
   virtual void SetUp() = 0;
   virtual void TearDown() = 0;
   virtual void ResetStub() = 0;
-  virtual bool AllowExhaustion() = 0;
   grpc::testing::EchoTestService::Stub* GetStub() { return stub_.get(); }
 
  protected:
   std::unique_ptr<grpc::testing::EchoTestService::Stub> stub_;
-  // Some tests use a custom thread creator. This should be declared before the
-  // server so that it's destructor happens after the server
-  std::unique_ptr<ServerBuilderThreadCreatorOverrideTest> creator_;
-
   std::unique_ptr<Server> server_;
 
   virtual void SetUpStart(ServerBuilder* builder, Service* service) = 0;
@@ -103,7 +147,6 @@ class CommonStressTestInsecure : public CommonStressTest<Service> {
         CreateChannel(server_address_.str(), InsecureChannelCredentials());
     this->stub_ = grpc::testing::EchoTestService::NewStub(channel);
   }
-  bool AllowExhaustion() override { return false; }
 
  protected:
   void SetUpStart(ServerBuilder* builder, Service* service) override {
@@ -119,7 +162,7 @@ class CommonStressTestInsecure : public CommonStressTest<Service> {
   std::ostringstream server_address_;
 };
 
-template <class Service, bool allow_resource_exhaustion>
+template <class Service>
 class CommonStressTestInproc : public CommonStressTest<Service> {
  public:
   void ResetStub() override {
@@ -127,7 +170,6 @@ class CommonStressTestInproc : public CommonStressTest<Service> {
     std::shared_ptr<Channel> channel = this->server_->InProcessChannel(args);
     this->stub_ = grpc::testing::EchoTestService::NewStub(channel);
   }
-  bool AllowExhaustion() override { return allow_resource_exhaustion; }
 
  protected:
   void SetUpStart(ServerBuilder* builder, Service* service) override {
@@ -152,67 +194,6 @@ class CommonStressTestSyncServer : public BaseClass {
   TestServiceImpl service_;
 };
 
-class ServerBuilderThreadCreatorOverrideTest {
- public:
-  ServerBuilderThreadCreatorOverrideTest(ServerBuilder* builder, size_t limit)
-      : limit_(limit), threads_(0) {
-    builder->SetThreadFunctions(
-        [this](gpr_thd_id* id, const char* name, void (*f)(void*), void* arg,
-               const gpr_thd_options* options) -> int {
-          std::unique_lock<std::mutex> l(mu_);
-          if (threads_ < limit_) {
-            l.unlock();
-            if (gpr_thd_new(id, name, f, arg, options) != 0) {
-              l.lock();
-              threads_++;
-              return 1;
-            }
-          }
-          return 0;
-        },
-        [this](gpr_thd_id id) {
-          gpr_thd_join(id);
-          std::unique_lock<std::mutex> l(mu_);
-          threads_--;
-          if (threads_ == 0) {
-            done_.notify_one();
-          }
-        });
-  }
-  ~ServerBuilderThreadCreatorOverrideTest() {
-    // Don't allow destruction until all threads are really done and uncounted
-    std::unique_lock<std::mutex> l(mu_);
-    done_.wait(l, [this] { return (threads_ == 0); });
-  }
-
- private:
-  size_t limit_;
-  size_t threads_;
-  std::mutex mu_;
-  std::condition_variable done_;
-};
-
-template <class BaseClass>
-class CommonStressTestSyncServerLowThreadCount : public BaseClass {
- public:
-  void SetUp() override {
-    ServerBuilder builder;
-    this->SetUpStart(&builder, &service_);
-    builder.SetSyncServerOption(ServerBuilder::SyncServerOption::MIN_POLLERS,
-                                1);
-    this->creator_.reset(
-        new ServerBuilderThreadCreatorOverrideTest(&builder, 4));
-    this->SetUpEnd(&builder);
-  }
-  void TearDown() override {
-    this->TearDownStart();
-    this->TearDownEnd();
-  }
-
- private:
-  TestServiceImpl service_;
-};
-
 template <class BaseClass>
 class CommonStressTestAsyncServer : public BaseClass {
  public:
@@ -313,8 +294,7 @@ class End2endTest : public ::testing::Test {
   Common common_;
 };
 
-static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs,
-                    bool allow_exhaustion, gpr_atm* errors) {
+static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs) {
   EchoRequest request;
   EchoResponse response;
   request.set_message("Hello");
@@ -322,48 +302,33 @@ static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs,
   for (int i = 0; i < num_rpcs; ++i) {
     ClientContext context;
     Status s = stub->Echo(&context, request, &response);
-    EXPECT_TRUE(s.ok() || (allow_exhaustion &&
-                           s.error_code() == StatusCode::RESOURCE_EXHAUSTED));
+    EXPECT_EQ(response.message(), request.message());
     if (!s.ok()) {
-      if (!(allow_exhaustion &&
-            s.error_code() == StatusCode::RESOURCE_EXHAUSTED)) {
-        gpr_log(GPR_ERROR, "RPC error: %d: %s", s.error_code(),
-                s.error_message().c_str());
-      }
-      gpr_atm_no_barrier_fetch_add(errors, static_cast<gpr_atm>(1));
-    } else {
-      EXPECT_EQ(response.message(), request.message());
+      gpr_log(GPR_ERROR, "RPC error: %d: %s", s.error_code(),
+              s.error_message().c_str());
     }
+    ASSERT_TRUE(s.ok());
   }
 }
 
 typedef ::testing::Types<
     CommonStressTestSyncServer<CommonStressTestInsecure<TestServiceImpl>>,
-    CommonStressTestSyncServer<CommonStressTestInproc<TestServiceImpl, false>>,
-    CommonStressTestSyncServerLowThreadCount<
-        CommonStressTestInproc<TestServiceImpl, true>>,
+    CommonStressTestSyncServer<CommonStressTestInproc<TestServiceImpl>>,
     CommonStressTestAsyncServer<
         CommonStressTestInsecure<grpc::testing::EchoTestService::AsyncService>>,
-    CommonStressTestAsyncServer<CommonStressTestInproc<
-        grpc::testing::EchoTestService::AsyncService, false>>>
+    CommonStressTestAsyncServer<
+        CommonStressTestInproc<grpc::testing::EchoTestService::AsyncService>>>
     CommonTypes;
 TYPED_TEST_CASE(End2endTest, CommonTypes);
 TYPED_TEST(End2endTest, ThreadStress) {
   this->common_.ResetStub();
   std::vector<std::thread> threads;
-  gpr_atm errors;
-  gpr_atm_rel_store(&errors, static_cast<gpr_atm>(0));
   for (int i = 0; i < kNumThreads; ++i) {
-    threads.emplace_back(SendRpc, this->common_.GetStub(), kNumRpcs,
-                         this->common_.AllowExhaustion(), &errors);
+    threads.emplace_back(SendRpc, this->common_.GetStub(), kNumRpcs);
   }
   for (int i = 0; i < kNumThreads; ++i) {
     threads[i].join();
   }
-  uint64_t error_cnt = static_cast<uint64_t>(gpr_atm_no_barrier_load(&errors));
-  if (error_cnt != 0) {
-    gpr_log(GPR_INFO, "RPC error count: %" PRIu64, error_cnt);
-  }
 }
 
 template <class Common>
diff --git a/test/cpp/thread_manager/BUILD b/test/cpp/thread_manager/BUILD
deleted file mode 100644
index 1f0878770b5..00000000000
--- a/test/cpp/thread_manager/BUILD
+++ /dev/null
@@ -1,31 +0,0 @@
-# Copyright 2017 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-licenses(["notice"])  # Apache v2
-
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_package")
-
-grpc_package(name = "test/cpp/thread_manager")
-
-grpc_cc_test(
-    name = "thread_manager_test",
-    srcs = ["thread_manager_test.cc"],
-    deps = [
-        "//:gpr",
-        "//:grpc",
-        "//:grpc++",
-        "//test/cpp/util:test_config",
-    ],
-)
-
diff --git a/test/cpp/thread_manager/thread_manager_test.cc b/test/cpp/thread_manager/thread_manager_test.cc
index d3d31f9dd97..8282d466948 100644
--- a/test/cpp/thread_manager/thread_manager_test.cc
+++ b/test/cpp/thread_manager/thread_manager_test.cc
@@ -20,10 +20,10 @@
 #include <memory>
 #include <string>
 
+#include <gflags/gflags.h>
 #include <grpc++/grpc++.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
-#include <grpc/support/thd.h>
 
 #include "src/cpp/thread_manager/thread_manager.h"
 #include "test/cpp/util/test_config.h"
@@ -32,13 +32,13 @@ namespace grpc {
 class ThreadManagerTest final : public grpc::ThreadManager {
  public:
   ThreadManagerTest()
-      : ThreadManager(kMinPollers, kMaxPollers, gpr_thd_new, gpr_thd_join),
+      : ThreadManager(kMinPollers, kMaxPollers),
         num_do_work_(0),
         num_poll_for_work_(0),
         num_work_found_(0) {}
 
   grpc::ThreadManager::WorkStatus PollForWork(void** tag, bool* ok) override;
-  void DoWork(void* tag, bool ok, bool resources) override;
+  void DoWork(void* tag, bool ok) override;
   void PerformTest();
 
  private:
@@ -89,7 +89,7 @@ grpc::ThreadManager::WorkStatus ThreadManagerTest::PollForWork(void** tag,
   }
 }
 
-void ThreadManagerTest::DoWork(void* tag, bool ok, bool resources) {
+void ThreadManagerTest::DoWork(void* tag, bool ok) {
   gpr_atm_no_barrier_fetch_add(&num_do_work_, 1);
   SleepForMs(kDoWorkDurationMsec);  // Simulate doing work by sleeping
 }

From 86c17bfc7d5a191e3738038028c1cfd7b3c181cf Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Fri, 12 Jan 2018 13:55:51 +0100
Subject: [PATCH 092/161] start with fewer iterations

---
 test/core/support/sync_test.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/core/support/sync_test.cc b/test/core/support/sync_test.cc
index 04b9e94cbc3..768f96d093f 100644
--- a/test/core/support/sync_test.cc
+++ b/test/core/support/sync_test.cc
@@ -231,7 +231,7 @@ static void mark_thread_done(struct test* m) {
    */
 static void test(const char* name, void (*body)(void* m),
                  void (*extra)(void* m), int timeout_s, int incr_step) {
-  int64_t iterations = 1024;
+  int64_t iterations = 256;
   struct test* m;
   gpr_timespec start = gpr_now(GPR_CLOCK_REALTIME);
   gpr_timespec time_taken;
@@ -240,7 +240,6 @@ static void test(const char* name, void (*body)(void* m),
   fprintf(stderr, "%s:", name);
   fflush(stderr);
   while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0) {
-    iterations <<= 1;
     fprintf(stderr, " %ld", (long)iterations);
     fflush(stderr);
     m = test_new(10, iterations, incr_step);
@@ -258,6 +257,7 @@ static void test(const char* name, void (*body)(void* m),
       GPR_ASSERT(0);
     }
     test_destroy(m);
+    iterations <<= 1;
   }
   time_taken = gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), start);
   fprintf(stderr, " done %lld.%09d s\n", (long long)time_taken.tv_sec,

From 9df154ca32ab4bc750f58717af438cde0797e228 Mon Sep 17 00:00:00 2001
From: Alexander Polcyn <apolcyn@google.com>
Date: Fri, 12 Jan 2018 11:44:08 -0800
Subject: [PATCH 093/161] Correct usage of select in _get_external_deps

---
 bazel/grpc_build_system.bzl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
index 9a3b7613445..25605192843 100644
--- a/bazel/grpc_build_system.bzl
+++ b/bazel/grpc_build_system.bzl
@@ -30,12 +30,12 @@ def _get_external_deps(external_deps):
   ret = []
   for dep in external_deps:
     if dep == "nanopb":
-      ret.append("//third_party/nanopb")
+      ret += ["//third_party/nanopb"]
     elif dep == "cares":
       ret += select({"//:grpc_no_ares": [],
                      "//conditions:default": ["//external:cares"],})
     else:
-      ret.append("//external:" + dep)
+      ret += ["//external:" + dep]
   return ret
 
 def _maybe_update_cc_library_hdrs(hdrs):

From 94dad609783b0e67c9b4b1de079330e19cf813c2 Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Fri, 12 Jan 2018 12:29:09 -0800
Subject: [PATCH 094/161] Add equality operators to RefCountedPtr.

---
 src/core/lib/support/ref_counted_ptr.h    | 13 +++++++++++++
 test/core/support/ref_counted_ptr_test.cc | 13 +++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/src/core/lib/support/ref_counted_ptr.h b/src/core/lib/support/ref_counted_ptr.h
index dc2385e369c..83e99d8ca60 100644
--- a/src/core/lib/support/ref_counted_ptr.h
+++ b/src/core/lib/support/ref_counted_ptr.h
@@ -76,6 +76,19 @@ class RefCountedPtr {
   T& operator*() const { return *value_; }
   T* operator->() const { return value_; }
 
+  bool operator==(const RefCountedPtr& other) const {
+    return value_ == other.value_;
+  }
+  bool operator==(T* other) const {
+    return value_ == other;
+  }
+  bool operator!=(const RefCountedPtr& other) const {
+    return value_ != other.value_;
+  }
+  bool operator!=(T* other) const {
+    return value_ != other;
+  }
+
  private:
   T* value_ = nullptr;
 };
diff --git a/test/core/support/ref_counted_ptr_test.cc b/test/core/support/ref_counted_ptr_test.cc
index 1830edc4e59..ce4975d3477 100644
--- a/test/core/support/ref_counted_ptr_test.cc
+++ b/test/core/support/ref_counted_ptr_test.cc
@@ -138,6 +138,19 @@ TEST(RefCountedPtr, DerefernceOperators) {
   foo_ref.value();
 }
 
+TEST(RefCountedPtr, EqualityOperators) {
+  RefCountedPtr<Foo> foo(New<Foo>());
+  RefCountedPtr<Foo> bar = foo;
+  RefCountedPtr<Foo> empty;
+  // Test equality between RefCountedPtrs.
+  EXPECT_EQ(foo, bar);
+  EXPECT_NE(foo, empty);
+  // Test equality with bare pointers.
+  EXPECT_EQ(foo, foo.get());
+  EXPECT_EQ(empty, nullptr);
+  EXPECT_NE(foo, nullptr);
+}
+
 TEST(MakeRefCounted, NoArgs) {
   RefCountedPtr<Foo> foo = MakeRefCounted<Foo>();
   EXPECT_EQ(0, foo->value());

From be1b7f986e55756ac83b3807ee1d73929fbb3546 Mon Sep 17 00:00:00 2001
From: David Garcia Quintas <dgq@google.com>
Date: Fri, 12 Jan 2018 14:01:38 -0800
Subject: [PATCH 095/161] PR comments, round 3

---
 src/core/ext/filters/client_channel/client_channel.cc    | 6 +++---
 .../filters/client_channel/lb_policy/grpclb/grpclb.cc    | 8 +++-----
 .../client_channel/lb_policy/pick_first/pick_first.cc    | 4 +---
 .../client_channel/lb_policy/round_robin/round_robin.cc  | 2 +-
 src/core/ext/filters/client_channel/subchannel.cc        | 9 ++-------
 src/core/lib/support/ref_counted_ptr.h                   | 9 ++++++++-
 6 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc
index 6a10a25a325..bb29f65af97 100644
--- a/src/core/ext/filters/client_channel/client_channel.cc
+++ b/src/core/ext/filters/client_channel/client_channel.cc
@@ -1031,7 +1031,7 @@ static void create_subchannel_call_locked(grpc_call_element* elem,
 static void pick_done_locked(grpc_call_element* elem, grpc_error* error) {
   call_data* calld = (call_data*)elem->call_data;
   channel_data* chand = (channel_data*)elem->channel_data;
-  if (!calld->pick.connected_subchannel) {
+  if (calld->pick.connected_subchannel == nullptr) {
     // Failed to create subchannel.
     GRPC_ERROR_UNREF(calld->error);
     calld->error = error == GRPC_ERROR_NONE
@@ -1283,7 +1283,7 @@ static void start_pick_locked(void* arg, grpc_error* ignored) {
   grpc_call_element* elem = (grpc_call_element*)arg;
   call_data* calld = (call_data*)elem->call_data;
   channel_data* chand = (channel_data*)elem->channel_data;
-  GPR_ASSERT(!calld->pick.connected_subchannel);
+  GPR_ASSERT(calld->pick.connected_subchannel == nullptr);
   if (chand->lb_policy != nullptr) {
     // We already have an LB policy, so ask it for a pick.
     if (pick_callback_start_locked(elem)) {
@@ -1462,7 +1462,7 @@ static void cc_destroy_call_elem(grpc_call_element* elem,
                                "client_channel_destroy_call");
   }
   GPR_ASSERT(calld->waiting_for_pick_batches_count == 0);
-  if (calld->pick.connected_subchannel) {
+  if (calld->pick.connected_subchannel != nullptr) {
     calld->pick.connected_subchannel.reset();
   }
   for (size_t i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index 39467272b36..9da21c73924 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -322,7 +322,7 @@ static void pending_pick_set_metadata_and_context(pending_pick* pp) {
   /* if connected_subchannel is nullptr, no pick has been made by the RR
    * policy (e.g., all addresses failed to connect). There won't be any
    * user_data/token available */
-  if (pp->pick->connected_subchannel.get() != nullptr) {
+  if (pp->pick->connected_subchannel != nullptr) {
     if (!GRPC_MDISNULL(pp->lb_token)) {
       initial_metadata_add_lb_token(pp->pick->initial_metadata,
                                     &pp->pick->lb_token_mdelem_storage,
@@ -935,8 +935,7 @@ static void glb_shutdown_locked(grpc_lb_policy* pol,
       }
       gpr_free(pp);
     } else {
-      pp->pick->connected_subchannel =
-          grpc_core::MakeRefCounted<grpc_core::ConnectedSubchannel>(nullptr);
+      pp->pick->connected_subchannel.reset(nullptr);
       GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_REF(error));
     }
     pp = next;
@@ -973,8 +972,7 @@ static void glb_cancel_pick_locked(grpc_lb_policy* pol,
   while (pp != nullptr) {
     pending_pick* next = pp->next;
     if (pp->pick == pick) {
-      pick->connected_subchannel =
-          grpc_core::MakeRefCounted<grpc_core::ConnectedSubchannel>(nullptr);
+      pick->connected_subchannel.reset(nullptr);
       GRPC_CLOSURE_SCHED(&pp->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));
diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
index c91a305f336..725b78d4787 100644
--- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc
@@ -294,7 +294,7 @@ static void pf_update_locked(grpc_lb_policy* policy,
                   p, p->selected->subchannel, i,
                   subchannel_list->num_subchannels);
         }
-        if (p->selected->connected_subchannel) {
+        if (p->selected->connected_subchannel != nullptr) {
           sd->connected_subchannel = p->selected->connected_subchannel;
         }
         p->selected = sd;
@@ -458,8 +458,6 @@ static void pf_connectivity_changed_locked(void* arg, grpc_error* error) {
       // Cases 1 and 2.
       grpc_connectivity_state_set(&p->state_tracker, GRPC_CHANNEL_READY,
                                   GRPC_ERROR_NONE, "connecting_ready");
-      sd->connected_subchannel =
-          grpc_subchannel_get_connected_subchannel(sd->subchannel);
       p->selected = sd;
       if (grpc_lb_pick_first_trace.enabled()) {
         gpr_log(GPR_INFO, "Pick First %p selected subchannel %p", (void*)p,
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
index 91fee2d51aa..dca345566ad 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
@@ -423,7 +423,7 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
       break;
     }
     case GRPC_CHANNEL_READY: {
-      if (!sd->connected_subchannel) {
+      if (sd->connected_subchannel == nullptr) {
         sd->connected_subchannel =
             grpc_subchannel_get_connected_subchannel(sd->subchannel);
       }
diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc
index fe0bb486e87..5b8a14b9e7b 100644
--- a/src/core/ext/filters/client_channel/subchannel.cc
+++ b/src/core/ext/filters/client_channel/subchannel.cc
@@ -56,10 +56,6 @@
 #define GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS 120
 #define GRPC_SUBCHANNEL_RECONNECT_JITTER 0.2
 
-#define GET_CONNECTED_SUBCHANNEL(subchannel, barrier)          \
-  ((grpc_core::ConnectedSubchannel*)(gpr_atm_##barrier##_load( \
-      &(subchannel)->connected_subchannel)))
-
 namespace {
 struct state_watcher {
   grpc_closure closure;
@@ -443,7 +439,7 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) {
     return;
   }
 
-  if (c->connected_subchannel) {
+  if (c->connected_subchannel != nullptr) {
     /* Already connected: don't restart */
     return;
   }
@@ -524,7 +520,7 @@ static void on_connected_subchannel_connectivity_changed(void* p,
   switch (connected_subchannel_watcher->connectivity_state) {
     case GRPC_CHANNEL_TRANSIENT_FAILURE:
     case GRPC_CHANNEL_SHUTDOWN: {
-      if (!c->disconnected && c->connected_subchannel) {
+      if (!c->disconnected && c->connected_subchannel != nullptr) {
         if (grpc_trace_stream_refcount.enabled()) {
           gpr_log(GPR_INFO,
                   "Connected subchannel %p of subchannel %p has gone into %s. "
@@ -605,7 +601,6 @@ static bool publish_transport_locked(grpc_subchannel* c) {
   /* publish */
   c->connected_subchannel.reset(
       grpc_core::New<grpc_core::ConnectedSubchannel>(stk));
-  gpr_atm_full_barrier();
   gpr_log(GPR_INFO, "New connected subchannel at %p for subchannel %p",
           c->connected_subchannel.get(), c);
 
diff --git a/src/core/lib/support/ref_counted_ptr.h b/src/core/lib/support/ref_counted_ptr.h
index dad213003d6..8c8606ca0a1 100644
--- a/src/core/lib/support/ref_counted_ptr.h
+++ b/src/core/lib/support/ref_counted_ptr.h
@@ -76,7 +76,14 @@ class RefCountedPtr {
   T& operator*() const { return *value_; }
   T* operator->() const { return value_; }
 
-  explicit operator bool() const { return get() != nullptr; }
+  bool operator==(const RefCountedPtr& other) const {
+    return value_ == other.value_;
+  }
+  bool operator==(T* other) const { return value_ == other; }
+  bool operator!=(const RefCountedPtr& other) const {
+    return value_ != other.value_;
+  }
+  bool operator!=(T* other) const { return value_ != other; }
 
  private:
   T* value_ = nullptr;

From 269ee29e032e3edc62d5b63ce44b5135979249da Mon Sep 17 00:00:00 2001
From: David Garcia Quintas <dgq@google.com>
Date: Fri, 12 Jan 2018 14:18:37 -0800
Subject: [PATCH 096/161] PR comments, round 4

---
 .../ext/filters/client_channel/lb_policy/grpclb/grpclb.cc     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index 9da21c73924..d71cb2fefa7 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -935,7 +935,7 @@ static void glb_shutdown_locked(grpc_lb_policy* pol,
       }
       gpr_free(pp);
     } else {
-      pp->pick->connected_subchannel.reset(nullptr);
+      pp->pick->connected_subchannel.reset();
       GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_REF(error));
     }
     pp = next;
@@ -972,7 +972,7 @@ static void glb_cancel_pick_locked(grpc_lb_policy* pol,
   while (pp != nullptr) {
     pending_pick* next = pp->next;
     if (pp->pick == pick) {
-      pick->connected_subchannel.reset(nullptr);
+      pick->connected_subchannel.reset();
       GRPC_CLOSURE_SCHED(&pp->on_complete,
                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                              "Pick Cancelled", &error, 1));

From 2c1b6d253503ebf2f3ad1f2452c8dfa2de9410d7 Mon Sep 17 00:00:00 2001
From: Craig Tiller <ctiller@google.com>
Date: Fri, 12 Jan 2018 15:13:09 -0800
Subject: [PATCH 097/161] Remove ctiller projects

---
 summerofcode/ideas.md | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/summerofcode/ideas.md b/summerofcode/ideas.md
index d87cf1b8fa3..49991f0c759 100644
--- a/summerofcode/ideas.md
+++ b/summerofcode/ideas.md
@@ -19,15 +19,8 @@ gRPC C Core:
 
 1. Port gRPC to  one of the major BSD platforms ([FreeBSD](https://freebsd.org), [NetBSD](https://netbsd.org), and [OpenBSD](https://openbsd.org)) and create packages for them. Add [kqueue](https://www.freebsd.org/cgi/man.cgi?query=kqueue) support in the process.
  * **Required skills:** C programming language, BSD operating system.
- * **Likely mentors:** [Craig Tiller](https://github.com/ctiller),
- [Nicolas Noble](https://github.com/nicolasnoble),
+ * **Likely mentors:** [Nicolas Noble](https://github.com/nicolasnoble),
  [Vijay Pai](https://github.com/vjpai).
-1. Fix gRPC C-core's URI parser. The current parser does not qualify as a standard parser according to [RFC3986]( https://tools.ietf.org/html/rfc3986). Write test suites to verify this and make changes necessary to make the URI parser compliant.
- * **Required skills:** C programming language, HTTP standard compliance.
- * **Likely mentors:** [Craig Tiller](https://github.com/ctiller).
-1. HPACK compression efficiency evaluation - Figure out how to benchmark gRPC's compression efficiency (both in terms of bytes on the wire and cpu cycles). Implement benchmarks. Potentially extend this to other full-stack gRPC implementations (Java and Go).
- * **Required skills:** C programming language, software performance benchmarking, potentially Java and Go.
- * **Likely mentors:** [Craig Tiller](https://github.com/ctiller).
 
 
 gRPC Python:
@@ -38,7 +31,7 @@ gRPC Python:
 1. Develop and test Python 3.5 Support for gRPC. Make necessary changes to port gRPC and package it for supported platforms.
  * **Required skills:** Python programming language, Python 3.5 interpreter.
  * **Likely mentors:** [Nathaniel Manista](https://github.com/nathanielmanistaatgoogle), [Masood Malekghassemi](https://github.com/soltanmm).
- 
+
 gRPC Ruby/Java:
 
 1. [jRuby](http://jruby.org) support for gRPC. Develop a jRuby wrapper for gRPC based on grpc-java and ensure that it is API compatible with the existing Ruby implementation and passes all tests.

From 145b199c4ddcd5e7b985585f3e733ae5092adbbb Mon Sep 17 00:00:00 2001
From: Evan Jones <evan.jones@bluecore.com>
Date: Wed, 10 Jan 2018 10:47:48 -0500
Subject: [PATCH 098/161] python: Context.abort should fail RPC even for
 StatusCode.OK

grpc.ServicerContext.abort is documented to always raise an exception
to terminate the RPC. The code argument "must not be StatusCode.OK."
However, if you do pass StatusCode.OK, the RPC terminates successfully
on the client side, but returns None.

_server.py: If the user accidentally passes StatusCode.OK, treat it as
    StatusCode.UNKNOWN. This is what happens if the user accidentally
    passes something that is not a StatusCode instance. Additionally
    set details to ''.

_metadata_code_details_test.py: update test to verify the behavior of
    abort with invalid codes.
---
 src/python/grpcio/grpc/_server.py             |   6 +
 .../tests/unit/_metadata_code_details_test.py | 200 ++++++++++--------
 2 files changed, 119 insertions(+), 87 deletions(-)

diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py
index 1cdb2d45b6c..a89987a86df 100644
--- a/src/python/grpcio/grpc/_server.py
+++ b/src/python/grpcio/grpc/_server.py
@@ -278,6 +278,12 @@ class _Context(grpc.ServicerContext):
             self._state.trailing_metadata = trailing_metadata
 
     def abort(self, code, details):
+        # treat OK like other invalid arguments: fail the RPC
+        if code == grpc.StatusCode.OK:
+            logging.error(
+                'abort() called with StatusCode.OK; returning UNKNOWN')
+            code = grpc.StatusCode.UNKNOWN
+            details = ''
         with self._state.condition:
             self._state.code = code
             self._state.details = _common.encode(details)
diff --git a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py
index bb6ac704970..ca10bd4dab5 100644
--- a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py
+++ b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py
@@ -50,6 +50,12 @@ _SERVER_TRAILING_METADATA = (('server-trailing-md-key',
 _NON_OK_CODE = grpc.StatusCode.NOT_FOUND
 _DETAILS = 'Test details!'
 
+# calling abort should always fail an RPC, even for "invalid" codes
+_ABORT_CODES = (_NON_OK_CODE, 3, grpc.StatusCode.OK)
+_EXPECTED_CLIENT_CODES = (_NON_OK_CODE, grpc.StatusCode.UNKNOWN,
+                          grpc.StatusCode.UNKNOWN)
+_EXPECTED_DETAILS = (_DETAILS, _DETAILS, '')
+
 
 class _Servicer(object):
 
@@ -302,99 +308,119 @@ class MetadataCodeDetailsTest(unittest.TestCase):
         self.assertEqual(_DETAILS, response_iterator_call.details())
 
     def testAbortedUnaryUnary(self):
-        self._servicer.set_code(_NON_OK_CODE)
-        self._servicer.set_details(_DETAILS)
-        self._servicer.set_abort_call()
-
-        with self.assertRaises(grpc.RpcError) as exception_context:
-            self._unary_unary.with_call(object(), metadata=_CLIENT_METADATA)
-
-        self.assertTrue(
-            test_common.metadata_transmitted(
-                _CLIENT_METADATA, self._servicer.received_client_metadata()))
-        self.assertTrue(
-            test_common.metadata_transmitted(
-                _SERVER_INITIAL_METADATA,
-                exception_context.exception.initial_metadata()))
-        self.assertTrue(
-            test_common.metadata_transmitted(
-                _SERVER_TRAILING_METADATA,
-                exception_context.exception.trailing_metadata()))
-        self.assertIs(_NON_OK_CODE, exception_context.exception.code())
-        self.assertEqual(_DETAILS, exception_context.exception.details())
+        test_cases = zip(_ABORT_CODES, _EXPECTED_CLIENT_CODES,
+                         _EXPECTED_DETAILS)
+        for abort_code, expected_code, expected_details in test_cases:
+            self._servicer.set_code(abort_code)
+            self._servicer.set_details(_DETAILS)
+            self._servicer.set_abort_call()
+
+            with self.assertRaises(grpc.RpcError) as exception_context:
+                self._unary_unary.with_call(object(), metadata=_CLIENT_METADATA)
+
+            self.assertTrue(
+                test_common.metadata_transmitted(
+                    _CLIENT_METADATA,
+                    self._servicer.received_client_metadata()))
+            self.assertTrue(
+                test_common.metadata_transmitted(
+                    _SERVER_INITIAL_METADATA,
+                    exception_context.exception.initial_metadata()))
+            self.assertTrue(
+                test_common.metadata_transmitted(
+                    _SERVER_TRAILING_METADATA,
+                    exception_context.exception.trailing_metadata()))
+            self.assertIs(expected_code, exception_context.exception.code())
+            self.assertEqual(expected_details,
+                             exception_context.exception.details())
 
     def testAbortedUnaryStream(self):
-        self._servicer.set_code(_NON_OK_CODE)
-        self._servicer.set_details(_DETAILS)
-        self._servicer.set_abort_call()
-
-        response_iterator_call = self._unary_stream(
-            _SERIALIZED_REQUEST, metadata=_CLIENT_METADATA)
-        received_initial_metadata = response_iterator_call.initial_metadata()
-        with self.assertRaises(grpc.RpcError):
-            self.assertEqual(len(list(response_iterator_call)), 0)
-
-        self.assertTrue(
-            test_common.metadata_transmitted(
-                _CLIENT_METADATA, self._servicer.received_client_metadata()))
-        self.assertTrue(
-            test_common.metadata_transmitted(_SERVER_INITIAL_METADATA,
-                                             received_initial_metadata))
-        self.assertTrue(
-            test_common.metadata_transmitted(
-                _SERVER_TRAILING_METADATA,
-                response_iterator_call.trailing_metadata()))
-        self.assertIs(_NON_OK_CODE, response_iterator_call.code())
-        self.assertEqual(_DETAILS, response_iterator_call.details())
+        test_cases = zip(_ABORT_CODES, _EXPECTED_CLIENT_CODES,
+                         _EXPECTED_DETAILS)
+        for abort_code, expected_code, expected_details in test_cases:
+            self._servicer.set_code(abort_code)
+            self._servicer.set_details(_DETAILS)
+            self._servicer.set_abort_call()
+
+            response_iterator_call = self._unary_stream(
+                _SERIALIZED_REQUEST, metadata=_CLIENT_METADATA)
+            received_initial_metadata = \
+                response_iterator_call.initial_metadata()
+            with self.assertRaises(grpc.RpcError):
+                self.assertEqual(len(list(response_iterator_call)), 0)
+
+            self.assertTrue(
+                test_common.metadata_transmitted(
+                    _CLIENT_METADATA,
+                    self._servicer.received_client_metadata()))
+            self.assertTrue(
+                test_common.metadata_transmitted(_SERVER_INITIAL_METADATA,
+                                                 received_initial_metadata))
+            self.assertTrue(
+                test_common.metadata_transmitted(
+                    _SERVER_TRAILING_METADATA,
+                    response_iterator_call.trailing_metadata()))
+            self.assertIs(expected_code, response_iterator_call.code())
+            self.assertEqual(expected_details, response_iterator_call.details())
 
     def testAbortedStreamUnary(self):
-        self._servicer.set_code(_NON_OK_CODE)
-        self._servicer.set_details(_DETAILS)
-        self._servicer.set_abort_call()
-
-        with self.assertRaises(grpc.RpcError) as exception_context:
-            self._stream_unary.with_call(
-                iter([_SERIALIZED_REQUEST] * test_constants.STREAM_LENGTH),
-                metadata=_CLIENT_METADATA)
-
-        self.assertTrue(
-            test_common.metadata_transmitted(
-                _CLIENT_METADATA, self._servicer.received_client_metadata()))
-        self.assertTrue(
-            test_common.metadata_transmitted(
-                _SERVER_INITIAL_METADATA,
-                exception_context.exception.initial_metadata()))
-        self.assertTrue(
-            test_common.metadata_transmitted(
-                _SERVER_TRAILING_METADATA,
-                exception_context.exception.trailing_metadata()))
-        self.assertIs(_NON_OK_CODE, exception_context.exception.code())
-        self.assertEqual(_DETAILS, exception_context.exception.details())
+        test_cases = zip(_ABORT_CODES, _EXPECTED_CLIENT_CODES,
+                         _EXPECTED_DETAILS)
+        for abort_code, expected_code, expected_details in test_cases:
+            self._servicer.set_code(abort_code)
+            self._servicer.set_details(_DETAILS)
+            self._servicer.set_abort_call()
+
+            with self.assertRaises(grpc.RpcError) as exception_context:
+                self._stream_unary.with_call(
+                    iter([_SERIALIZED_REQUEST] * test_constants.STREAM_LENGTH),
+                    metadata=_CLIENT_METADATA)
+
+            self.assertTrue(
+                test_common.metadata_transmitted(
+                    _CLIENT_METADATA,
+                    self._servicer.received_client_metadata()))
+            self.assertTrue(
+                test_common.metadata_transmitted(
+                    _SERVER_INITIAL_METADATA,
+                    exception_context.exception.initial_metadata()))
+            self.assertTrue(
+                test_common.metadata_transmitted(
+                    _SERVER_TRAILING_METADATA,
+                    exception_context.exception.trailing_metadata()))
+            self.assertIs(expected_code, exception_context.exception.code())
+            self.assertEqual(expected_details,
+                             exception_context.exception.details())
 
     def testAbortedStreamStream(self):
-        self._servicer.set_code(_NON_OK_CODE)
-        self._servicer.set_details(_DETAILS)
-        self._servicer.set_abort_call()
-
-        response_iterator_call = self._stream_stream(
-            iter([object()] * test_constants.STREAM_LENGTH),
-            metadata=_CLIENT_METADATA)
-        received_initial_metadata = response_iterator_call.initial_metadata()
-        with self.assertRaises(grpc.RpcError):
-            self.assertEqual(len(list(response_iterator_call)), 0)
-
-        self.assertTrue(
-            test_common.metadata_transmitted(
-                _CLIENT_METADATA, self._servicer.received_client_metadata()))
-        self.assertTrue(
-            test_common.metadata_transmitted(_SERVER_INITIAL_METADATA,
-                                             received_initial_metadata))
-        self.assertTrue(
-            test_common.metadata_transmitted(
-                _SERVER_TRAILING_METADATA,
-                response_iterator_call.trailing_metadata()))
-        self.assertIs(_NON_OK_CODE, response_iterator_call.code())
-        self.assertEqual(_DETAILS, response_iterator_call.details())
+        test_cases = zip(_ABORT_CODES, _EXPECTED_CLIENT_CODES,
+                         _EXPECTED_DETAILS)
+        for abort_code, expected_code, expected_details in test_cases:
+            self._servicer.set_code(abort_code)
+            self._servicer.set_details(_DETAILS)
+            self._servicer.set_abort_call()
+
+            response_iterator_call = self._stream_stream(
+                iter([object()] * test_constants.STREAM_LENGTH),
+                metadata=_CLIENT_METADATA)
+            received_initial_metadata = \
+                response_iterator_call.initial_metadata()
+            with self.assertRaises(grpc.RpcError):
+                self.assertEqual(len(list(response_iterator_call)), 0)
+
+            self.assertTrue(
+                test_common.metadata_transmitted(
+                    _CLIENT_METADATA,
+                    self._servicer.received_client_metadata()))
+            self.assertTrue(
+                test_common.metadata_transmitted(_SERVER_INITIAL_METADATA,
+                                                 received_initial_metadata))
+            self.assertTrue(
+                test_common.metadata_transmitted(
+                    _SERVER_TRAILING_METADATA,
+                    response_iterator_call.trailing_metadata()))
+            self.assertIs(expected_code, response_iterator_call.code())
+            self.assertEqual(expected_details, response_iterator_call.details())
 
     def testCustomCodeUnaryUnary(self):
         self._servicer.set_code(_NON_OK_CODE)

From 0633c9fbd258d9bff2cd70bb0a24d1e6025419e0 Mon Sep 17 00:00:00 2001
From: Adele Zhou <adelez@google.com>
Date: Fri, 12 Jan 2018 15:25:27 -0800
Subject: [PATCH 099/161] Add cloud gateways to interop tests.

---
 tools/internal_ci/linux/grpc_interop_toprod.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/internal_ci/linux/grpc_interop_toprod.cfg b/tools/internal_ci/linux/grpc_interop_toprod.cfg
index 8d025c4f60d..ff7a98f44e6 100644
--- a/tools/internal_ci/linux/grpc_interop_toprod.cfg
+++ b/tools/internal_ci/linux/grpc_interop_toprod.cfg
@@ -26,5 +26,5 @@ action {
 
 env_vars {
   key: "RUN_TESTS_FLAGS"
-  value: "-l all --cloud_to_prod --cloud_to_prod_auth --prod_servers default gateway_v4 --use_docker --internal_ci -t -j 12 --bq_result_table interop_results"
+  value: "-l all --cloud_to_prod --cloud_to_prod_auth --prod_servers default gateway_v4 cloud_gateway cloud_gateway_v4 --use_docker --internal_ci -t -j 12 --bq_result_table interop_results"
 }

From 4bba3187cbbeec368ecd1625359ab51834f4bf45 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Tue, 16 Jan 2018 15:56:41 +0100
Subject: [PATCH 100/161] better log message when auth interceptor throws

---
 .../Internal/NativeMetadataCredentialsPlugin.cs       | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs b/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs
index a8cb3571811..4d695e88505 100644
--- a/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs
+++ b/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs
@@ -27,7 +27,8 @@ namespace Grpc.Core.Internal
 
     internal class NativeMetadataCredentialsPlugin
     {
-        const string GetMetadataExceptionMsg = "Exception occured in metadata credentials plugin.";
+        const string GetMetadataExceptionStatusMsg = "Exception occurred in metadata credentials plugin.";
+        const string GetMetadataExceptionLogMsg = GetMetadataExceptionStatusMsg + " This is likely not a problem with gRPC itself. Please verify that the code supplying the metadata (usually an authentication token) works correctly.";
         static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<NativeMetadataCredentialsPlugin>();
         static readonly NativeMethods Native = NativeMethods.Get();
 
@@ -67,8 +68,8 @@ namespace Grpc.Core.Internal
             }
             catch (Exception e)
             {
-                Native.grpcsharp_metadata_credentials_notify_from_plugin(callbackPtr, userDataPtr, MetadataArraySafeHandle.Create(Metadata.Empty), StatusCode.Unknown, GetMetadataExceptionMsg);
-                Logger.Error(e, GetMetadataExceptionMsg);
+                Native.grpcsharp_metadata_credentials_notify_from_plugin(callbackPtr, userDataPtr, MetadataArraySafeHandle.Create(Metadata.Empty), StatusCode.Unknown, GetMetadataExceptionStatusMsg);
+                Logger.Error(e, GetMetadataExceptionLogMsg);
             }
         }
 
@@ -86,8 +87,8 @@ namespace Grpc.Core.Internal
             }
             catch (Exception e)
             {
-                Native.grpcsharp_metadata_credentials_notify_from_plugin(callbackPtr, userDataPtr, MetadataArraySafeHandle.Create(Metadata.Empty), StatusCode.Unknown, GetMetadataExceptionMsg);
-                Logger.Error(e, GetMetadataExceptionMsg);
+                Native.grpcsharp_metadata_credentials_notify_from_plugin(callbackPtr, userDataPtr, MetadataArraySafeHandle.Create(Metadata.Empty), StatusCode.Unknown, GetMetadataExceptionStatusMsg);
+                Logger.Error(e, GetMetadataExceptionLogMsg);
             }
         }
     }

From a41e997a8785243ee3c1e91c5dcfb73d698ac48d Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Tue, 2 Jan 2018 16:51:13 +0100
Subject: [PATCH 101/161] cmake output to gitignore

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index 2b35c5fb9a3..882828ef799 100644
--- a/.gitignore
+++ b/.gitignore
@@ -130,3 +130,5 @@ bm_diff_new/
 bm_diff_old/
 bm_*.json
 
+# cmake build files
+/cmake/build

From b6cdfca30a5f09975ee7789a66d5b248f92b5c50 Mon Sep 17 00:00:00 2001
From: Zhouyihai Ding <ddyihai@google.com>
Date: Sat, 13 Jan 2018 16:25:28 -0800
Subject: [PATCH 102/161] php channel: fix memory leak

---
 src/php/ext/grpc/channel.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/php/ext/grpc/channel.c b/src/php/ext/grpc/channel.c
index dc3acc89bb7..db59869c7f7 100644
--- a/src/php/ext/grpc/channel.c
+++ b/src/php/ext/grpc/channel.c
@@ -242,6 +242,7 @@ PHP_METHOD(Channel, __construct) {
 
   // parse the rest of the channel args array
   if (php_grpc_read_args_array(args_array, &args TSRMLS_CC) == FAILURE) {
+    efree(args.args);
     return;
   }
 
@@ -301,6 +302,7 @@ PHP_METHOD(Channel, __construct) {
       create_and_add_channel_to_persistent_list(
           channel, target, args, creds, key, key_len TSRMLS_CC);
     } else {
+      efree(args.args);
       channel->wrapper = le->channel;
     }
   }

From 69c539fd855c416fd2a2fc7406c6c854500f6a12 Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Tue, 16 Jan 2018 08:21:05 -0800
Subject: [PATCH 103/161] clang-format

---
 src/core/lib/support/ref_counted_ptr.h | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/core/lib/support/ref_counted_ptr.h b/src/core/lib/support/ref_counted_ptr.h
index 83e99d8ca60..8c8606ca0a1 100644
--- a/src/core/lib/support/ref_counted_ptr.h
+++ b/src/core/lib/support/ref_counted_ptr.h
@@ -79,15 +79,11 @@ class RefCountedPtr {
   bool operator==(const RefCountedPtr& other) const {
     return value_ == other.value_;
   }
-  bool operator==(T* other) const {
-    return value_ == other;
-  }
+  bool operator==(T* other) const { return value_ == other; }
   bool operator!=(const RefCountedPtr& other) const {
     return value_ != other.value_;
   }
-  bool operator!=(T* other) const {
-    return value_ != other;
-  }
+  bool operator!=(T* other) const { return value_ != other; }
 
  private:
   T* value_ = nullptr;

From 586f719898486303c0c990919bfba05a3e127a16 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Tue, 16 Jan 2018 17:57:17 +0100
Subject: [PATCH 104/161] cleanup cmake include directories

---
 cmake/benchmark.cmake             |  5 ++++-
 cmake/cares.cmake                 |  5 ++++-
 cmake/gflags.cmake                | 14 +++++++------
 cmake/protobuf.cmake              | 13 +++++++-----
 cmake/zlib.cmake                  | 15 ++++++-------
 templates/CMakeLists.txt.template | 35 +++++++++++++++++--------------
 6 files changed, 51 insertions(+), 36 deletions(-)

diff --git a/cmake/benchmark.cmake b/cmake/benchmark.cmake
index c6284225a34..753dc0696f6 100644
--- a/cmake/benchmark.cmake
+++ b/cmake/benchmark.cmake
@@ -20,14 +20,17 @@ if("${gRPC_BENCHMARK_PROVIDER}" STREQUAL "module")
       add_subdirectory(${BENCHMARK_ROOT_DIR} third_party/benchmark)
       if(TARGET benchmark)
           set(_gRPC_BENCHMARK_LIBRARIES benchmark)
+          set(_gRPC_BENCHMARK_INCLUDE_DIR "${BENCHMARK_ROOT_DIR}/include")
       endif()
   else()
       message(WARNING "gRPC_BENCHMARK_PROVIDER is \"module\" but BENCHMARK_ROOT_DIR is wrong")
   endif()
 elseif("${gRPC_BENCHMARK_PROVIDER}" STREQUAL "package")
-  find_package(benchmark)
+  find_package(benchmark REQUIRED)
   if(TARGET benchmark::benchmark)
     set(_gRPC_BENCHMARK_LIBRARIES benchmark::benchmark)
+    # extract the include dir from target's properties
+    get_target_property(_gRPC_BENCHMARK_INCLUDE_DIR benchmark::benchmark INTERFACE_INCLUDE_DIRECTORIES)
   endif()
   set(_gRPC_FIND_BENCHMARK "if(NOT benchmark_FOUND)\n  find_package(benchmark)\nendif()")
 endif()
diff --git a/cmake/cares.cmake b/cmake/cares.cmake
index 521cf52e77d..53d7582f6f5 100644
--- a/cmake/cares.cmake
+++ b/cmake/cares.cmake
@@ -18,11 +18,13 @@ if("${gRPC_CARES_PROVIDER}" STREQUAL "module")
   endif()
   set(CARES_SHARED OFF CACHE BOOL "disable shared library")
   set(CARES_STATIC ON CACHE BOOL "link cares statically")
-  set(CARES_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/cares/cares")
   add_subdirectory(third_party/cares/cares)
+
   if(TARGET c-ares)
     set(_gRPC_CARES_LIBRARIES c-ares)
+    set(_gRPC_CARES_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/cares/cares" "${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares")
   endif()
+
   if(gRPC_INSTALL)
     message(WARNING "gRPC_INSTALL will be forced to FALSE because gRPC_CARES_PROVIDER is \"module\"")
     set(gRPC_INSTALL FALSE)
@@ -31,6 +33,7 @@ elseif("${gRPC_CARES_PROVIDER}" STREQUAL "package")
   find_package(c-ares REQUIRED CONFIG)
   if(TARGET c-ares::cares)
     set(_gRPC_CARES_LIBRARIES c-ares::cares)
+    set(_gRPC_CARES_INCLUDE_DIR ${c-ares_INCLUDE_DIR})
   endif()
   set(_gRPC_FIND_CARES "if(NOT c-ares_FOUND)\n  find_package(c-ares CONFIG)\nendif()")
 endif()
diff --git a/cmake/gflags.cmake b/cmake/gflags.cmake
index 1864bda357e..f86a141c1d8 100644
--- a/cmake/gflags.cmake
+++ b/cmake/gflags.cmake
@@ -17,17 +17,19 @@ if("${gRPC_GFLAGS_PROVIDER}" STREQUAL "module")
     set(GFLAGS_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/gflags)
   endif()
   if(EXISTS "${GFLAGS_ROOT_DIR}/CMakeLists.txt")
-      add_subdirectory(${GFLAGS_ROOT_DIR} third_party/gflags)
-      if(TARGET gflags_static)
-          set(_gRPC_GFLAGS_LIBRARIES gflags_static)
-      endif()
+    add_subdirectory(${GFLAGS_ROOT_DIR} third_party/gflags)
+    if(TARGET gflags_static)
+      set(_gRPC_GFLAGS_LIBRARIES gflags_static)
+      set(_gRPC_GFLAGS_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include")
+    endif()
   else()
-      message(WARNING "gRPC_GFLAGS_PROVIDER is \"module\" but GFLAGS_ROOT_DIR is wrong")
+    message(WARNING "gRPC_GFLAGS_PROVIDER is \"module\" but GFLAGS_ROOT_DIR is wrong")
   endif()
 elseif("${gRPC_GFLAGS_PROVIDER}" STREQUAL "package")
-  find_package(gflags)
+  find_package(gflags REQUIRED)
   if(TARGET gflags::gflags)
     set(_gRPC_GFLAGS_LIBRARIES gflags::gflags)
+    set(_gRPC_GFLAGS_INCLUDE_DIR ${GFLAGS_INCLUDE_DIR})
   endif()
   set(_gRPC_FIND_GFLAGS "if(NOT gflags_FOUND)\n  find_package(gflags)\nendif()")
 endif()
diff --git a/cmake/protobuf.cmake b/cmake/protobuf.cmake
index e2206a23195..fccabd455a8 100644
--- a/cmake/protobuf.cmake
+++ b/cmake/protobuf.cmake
@@ -27,7 +27,7 @@ if("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "module")
   if(NOT PROTOBUF_ROOT_DIR)
     set(PROTOBUF_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf)
   endif()
-  set(PROTOBUF_WELLKNOWN_IMPORT_DIR ${PROTOBUF_ROOT_DIR}/src)
+
   if(EXISTS "${PROTOBUF_ROOT_DIR}/cmake/CMakeLists.txt")
     set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "Link static runtime libraries")
     add_subdirectory(${PROTOBUF_ROOT_DIR}/cmake third_party/protobuf)
@@ -41,6 +41,9 @@ if("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "module")
       set(_gRPC_PROTOBUF_PROTOC protoc)
       set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $<TARGET_FILE:protoc>)
     endif()
+    set(_gRPC_PROTOBUF_INCLUDE_DIR "${PROTOBUF_ROOT_DIR}")
+    # For well-known .proto files distributed with protobuf
+    set(_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR "${PROTOBUF_ROOT_DIR}/src")
   else()
       message(WARNING "gRPC_PROTOBUF_PROVIDER is \"module\" but PROTOBUF_ROOT_DIR is wrong")
   endif()
@@ -58,8 +61,11 @@ elseif("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "package")
     endif()
     if(TARGET protobuf::libprotoc)
       set(_gRPC_PROTOBUF_PROTOC_LIBRARIES protobuf::libprotoc)
+      # extract the include dir from target's properties
+      get_target_property(_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR protobuf::libprotoc INTERFACE_INCLUDE_DIRECTORIES)
     else()
       set(_gRPC_PROTOBUF_PROTOC_LIBRARIES ${PROTOBUF_PROTOC_LIBRARIES})
+      set(_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR ${PROTOBUF_INCLUDE_DIRS})
     endif()
     if(TARGET protobuf::protoc)
       set(_gRPC_PROTOBUF_PROTOC protobuf::protoc)
@@ -68,10 +74,7 @@ elseif("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "package")
       set(_gRPC_PROTOBUF_PROTOC ${PROTOBUF_PROTOC_EXECUTABLE})
       set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE ${PROTOBUF_PROTOC_EXECUTABLE})
     endif()
+    set(_gRPC_PROTOBUF_INCLUDE_DIR ${PROTOBUF_INCLUDE_DIRS})
     set(_gRPC_FIND_PROTOBUF "if(NOT Protobuf_FOUND AND NOT PROTOBUF_FOUND)\n  find_package(Protobuf ${gRPC_PROTOBUF_PACKAGE_TYPE})\nendif()")
   endif()
-  if(PROTOBUF_FOUND)
-    include_directories(${PROTOBUF_INCLUDE_DIRS})
-  endif()
-  set(PROTOBUF_WELLKNOWN_IMPORT_DIR /usr/local/include)
 endif()
diff --git a/cmake/zlib.cmake b/cmake/zlib.cmake
index 16cd9e66d5a..4a9d2f011b1 100644
--- a/cmake/zlib.cmake
+++ b/cmake/zlib.cmake
@@ -16,15 +16,15 @@ if("${gRPC_ZLIB_PROVIDER}" STREQUAL "module")
   if(NOT ZLIB_ROOT_DIR)
     set(ZLIB_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/zlib)
   endif()
-  set(ZLIB_INCLUDE_DIR "${ZLIB_ROOT_DIR}")
   if(EXISTS "${ZLIB_ROOT_DIR}/CMakeLists.txt")
-      # TODO(jtattermusch): workaround for https://github.com/madler/zlib/issues/218
-      include_directories(${ZLIB_INCLUDE_DIR})
+    # TODO(jtattermusch): workaround for https://github.com/madler/zlib/issues/218
+    include_directories("${ZLIB_ROOT_DIR}")
+    add_subdirectory(${ZLIB_ROOT_DIR} third_party/zlib)
 
-      add_subdirectory(${ZLIB_ROOT_DIR} third_party/zlib)
-      if(TARGET zlibstatic)
-          set(_gRPC_ZLIB_LIBRARIES zlibstatic)
-      endif()
+    if(TARGET zlibstatic)
+      set(_gRPC_ZLIB_LIBRARIES zlibstatic)
+      set(_gRPC_ZLIB_INCLUDE_DIR "${ZLIB_ROOT_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib")
+    endif()
   else()
       message(WARNING "gRPC_ZLIB_PROVIDER is \"module\" but ZLIB_ROOT_DIR is wrong")
   endif()
@@ -35,5 +35,6 @@ if("${gRPC_ZLIB_PROVIDER}" STREQUAL "module")
 elseif("${gRPC_ZLIB_PROVIDER}" STREQUAL "package")
   find_package(ZLIB REQUIRED)
   set(_gRPC_ZLIB_LIBRARIES ${ZLIB_LIBRARIES})
+  set(_gRPC_ZLIB_INCLUDE_DIR ${ZLIB_INCLUDE_DIRS})
   set(_gRPC_FIND_ZLIB "if(NOT ZLIB_FOUND)\n  find_package(ZLIB)\nendif()")
 endif()
diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template
index 8de0ccde82a..de0f2eb328a 100644
--- a/templates/CMakeLists.txt.template
+++ b/templates/CMakeLists.txt.template
@@ -48,7 +48,10 @@
       deps.append("${_gRPC_CARES_LIBRARIES}")
     deps.append("${_gRPC_ALLTARGETS_LIBRARIES}")
     for d in target_dict.get('deps', []):
-      deps.append(d)
+      if d == 'benchmark':
+        deps.append("${_gRPC_BENCHMARK_LIBRARIES}")
+      else:
+        deps.append(d)
     if target_dict.build == 'test' and target_dict.language == 'c++':
       deps.append("${_gRPC_GFLAGS_LIBRARIES}")
     return deps
@@ -90,6 +93,10 @@
   set(gRPC_INSTALL <%text>${gRPC_INSTALL_default}</%text> CACHE BOOL
       "Generate installation target: gRPC_ZLIB_PROVIDER, gRPC_CARES_PROVIDER, gRPC_SSL_PROVIDER and gRPC_PROTOBUF_PROVIDER must all be \"package\"")
 
+  # Providers for third-party dependencies (gRPC_*_PROVIDER properties):
+  # "module": build the dependency using sources from git submodule (under third_party)
+  # "package": use cmake's find_package functionality to locate a pre-installed dependency
+  
   set(gRPC_ZLIB_PROVIDER "module" CACHE STRING "Provider of zlib library")
   set_property(CACHE gRPC_ZLIB_PROVIDER PROPERTY STRINGS "module" "package")
 
@@ -190,7 +197,7 @@
       return()
     endif()
 
-    set(_protobuf_include_path -I . -I <%text>${PROTOBUF_WELLKNOWN_IMPORT_DIR}</%text>)
+    set(_protobuf_include_path -I . -I <%text>${_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR}</%text>)
     foreach(FIL <%text>${ARGN}</%text>)
       get_filename_component(ABS_FIL <%text>${FIL}</%text> ABSOLUTE)
       get_filename_component(FIL_WE <%text>${FIL}</%text> NAME_WE)
@@ -346,13 +353,11 @@
     PUBLIC <%text>$<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include></%text>
     PRIVATE <%text>${CMAKE_CURRENT_SOURCE_DIR}</%text>
     PRIVATE <%text>${_gRPC_SSL_INCLUDE_DIR}</%text>
-    PRIVATE <%text>${PROTOBUF_ROOT_DIR}</%text>/src
-    PRIVATE <%text>${ZLIB_INCLUDE_DIR}</%text>
-    PRIVATE <%text>${BENCHMARK}</%text>/include
-    PRIVATE <%text>${CMAKE_CURRENT_BINARY_DIR}</%text>/third_party/zlib
-    PRIVATE <%text>${CARES_INCLUDE_DIR}</%text>
-    PRIVATE <%text>${CMAKE_CURRENT_BINARY_DIR}</%text>/third_party/cares/cares
-    PRIVATE <%text>${CMAKE_CURRENT_BINARY_DIR}</%text>/third_party/gflags/include
+    PRIVATE <%text>${_gRPC_PROTOBUF_INCLUDE_DIR}</%text>
+    PRIVATE <%text>${_gRPC_ZLIB_INCLUDE_DIR}</%text>
+    PRIVATE <%text>${_gRPC_BENCHMARK_INCLUDE_DIR}</%text>
+    PRIVATE <%text>${_gRPC_CARES_INCLUDE_DIR}</%text>
+    PRIVATE <%text>${_gRPC_GFLAGS_INCLUDE_DIR}</%text>
   % if lib.build in ['test', 'private'] and lib.language == 'c++':
     PRIVATE third_party/googletest/googletest/include
     PRIVATE third_party/googletest/googletest
@@ -417,13 +422,11 @@
     PRIVATE <%text>${CMAKE_CURRENT_SOURCE_DIR}</%text>
     PRIVATE <%text>${CMAKE_CURRENT_SOURCE_DIR}</%text>/include
     PRIVATE <%text>${_gRPC_SSL_INCLUDE_DIR}</%text>
-    PRIVATE <%text>${PROTOBUF_ROOT_DIR}</%text>/src
-    PRIVATE <%text>${BENCHMARK_ROOT_DIR}</%text>/include
-    PRIVATE <%text>${ZLIB_ROOT_DIR}</%text>
-    PRIVATE <%text>${CMAKE_CURRENT_BINARY_DIR}</%text>/third_party/zlib
-    PRIVATE <%text>${CARES_INCLUDE_DIR}</%text>
-    PRIVATE <%text>${CMAKE_CURRENT_BINARY_DIR}</%text>/third_party/cares/cares
-    PRIVATE <%text>${CMAKE_CURRENT_BINARY_DIR}</%text>/third_party/gflags/include
+    PRIVATE <%text>${_gRPC_PROTOBUF_INCLUDE_DIR}</%text>
+    PRIVATE <%text>${_gRPC_ZLIB_INCLUDE_DIR}</%text>
+    PRIVATE <%text>${_gRPC_BENCHMARK_INCLUDE_DIR}</%text>
+    PRIVATE <%text>${_gRPC_CARES_INCLUDE_DIR}</%text>
+    PRIVATE <%text>${_gRPC_GFLAGS_INCLUDE_DIR}</%text>
   % if tgt.build in ['test', 'private'] and tgt.language == 'c++':
     PRIVATE third_party/googletest/googletest/include
     PRIVATE third_party/googletest/googletest

From 41e58b4b1ac7e2934299e3855a1b1c5571282690 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Tue, 16 Jan 2018 17:58:05 +0100
Subject: [PATCH 105/161] regenerate

---
 CMakeLists.txt | 3926 ++++++++++++++++++++----------------------------
 1 file changed, 1641 insertions(+), 2285 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8fd9483f23e..93954b412d7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -46,6 +46,10 @@ endif()
 set(gRPC_INSTALL ${gRPC_INSTALL_default} CACHE BOOL
     "Generate installation target: gRPC_ZLIB_PROVIDER, gRPC_CARES_PROVIDER, gRPC_SSL_PROVIDER and gRPC_PROTOBUF_PROVIDER must all be \"package\"")
 
+# Providers for third-party dependencies (gRPC_*_PROVIDER properties):
+# "module": build the dependency using sources from git submodule (under third_party)
+# "package": use cmake's find_package functionality to locate a pre-installed dependency
+
 set(gRPC_ZLIB_PROVIDER "module" CACHE STRING "Provider of zlib library")
 set_property(CACHE gRPC_ZLIB_PROVIDER PROPERTY STRINGS "module" "package")
 
@@ -145,7 +149,7 @@ function(protobuf_generate_grpc_cpp)
     return()
   endif()
 
-  set(_protobuf_include_path -I . -I ${PROTOBUF_WELLKNOWN_IMPORT_DIR})
+  set(_protobuf_include_path -I . -I ${_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR})
   foreach(FIL ${ARGN})
     get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
     get_filename_component(FIL_WE ${FIL} NAME_WE)
@@ -680,13 +684,11 @@ target_include_directories(gpr
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr
@@ -772,13 +774,11 @@ target_include_directories(gpr_test_util
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_test_util
@@ -1059,13 +1059,11 @@ target_include_directories(grpc
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc
@@ -1372,13 +1370,11 @@ target_include_directories(grpc_cronet
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_cronet
@@ -1659,13 +1655,11 @@ target_include_directories(grpc_test_util
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_test_util
@@ -1928,13 +1922,11 @@ target_include_directories(grpc_test_util_unsecure
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_test_util_unsecure
@@ -2215,13 +2207,11 @@ target_include_directories(grpc_unsecure
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_unsecure
@@ -2306,13 +2296,11 @@ target_include_directories(reconnect_server
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(reconnect_server
@@ -2348,13 +2336,11 @@ target_include_directories(test_tcp_server
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(test_tcp_server
@@ -2429,13 +2415,11 @@ target_include_directories(grpc++
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -2631,13 +2615,11 @@ target_include_directories(grpc++_core_stats
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -2913,13 +2895,11 @@ target_include_directories(grpc++_cronet
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -3114,13 +3094,11 @@ target_include_directories(grpc++_error_details
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -3179,13 +3157,11 @@ target_include_directories(grpc++_proto_reflection_desc_db
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -3240,13 +3216,11 @@ target_include_directories(grpc++_reflection
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -3298,13 +3272,11 @@ target_include_directories(grpc++_test_config
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -3376,13 +3348,11 @@ target_include_directories(grpc++_test_util
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -3516,13 +3486,11 @@ target_include_directories(grpc++_test_util_unsecure
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -3658,13 +3626,11 @@ target_include_directories(grpc++_unsecure
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -3850,13 +3816,11 @@ target_include_directories(grpc_benchmark
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -3867,7 +3831,7 @@ target_include_directories(grpc_benchmark
 target_link_libraries(grpc_benchmark
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_unsecure
   grpc_test_util_unsecure
   grpc_unsecure
@@ -3909,13 +3873,11 @@ target_include_directories(grpc_cli_libs
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -3969,13 +3931,11 @@ target_include_directories(grpc_plugin_support
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -4047,13 +4007,11 @@ target_include_directories(http2_client_main
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -4102,13 +4060,11 @@ target_include_directories(interop_client_helper
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -4172,13 +4128,11 @@ target_include_directories(interop_client_main
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -4223,13 +4177,11 @@ target_include_directories(interop_server_helper
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -4292,13 +4244,11 @@ target_include_directories(interop_server_lib
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -4343,13 +4293,11 @@ target_include_directories(interop_server_main
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -4431,13 +4379,11 @@ target_include_directories(qps
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -4478,13 +4424,11 @@ target_include_directories(grpc_csharp_ext
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_csharp_ext
@@ -4525,13 +4469,11 @@ target_include_directories(bad_client_test
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(bad_client_test
@@ -4566,13 +4508,11 @@ target_include_directories(bad_ssl_test_server
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(bad_ssl_test_server
@@ -4668,13 +4608,11 @@ target_include_directories(end2end_tests
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(end2end_tests
@@ -4770,13 +4708,11 @@ target_include_directories(end2end_nosec_tests
   PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${ZLIB_INCLUDE_DIR}
-  PRIVATE ${BENCHMARK}/include
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(end2end_nosec_tests
@@ -4801,13 +4737,11 @@ target_include_directories(alarm_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(alarm_test
@@ -4830,13 +4764,11 @@ target_include_directories(algorithm_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(algorithm_test
@@ -4859,13 +4791,11 @@ target_include_directories(alloc_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(alloc_test
@@ -4886,13 +4816,11 @@ target_include_directories(alpn_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(alpn_test
@@ -4915,13 +4843,11 @@ target_include_directories(arena_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(arena_test
@@ -4942,13 +4868,11 @@ target_include_directories(bad_server_response_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(bad_server_response_test
@@ -4972,13 +4896,11 @@ target_include_directories(bin_decoder_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(bin_decoder_test
@@ -4999,13 +4921,11 @@ target_include_directories(bin_encoder_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(bin_encoder_test
@@ -5026,13 +4946,11 @@ target_include_directories(byte_stream_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(byte_stream_test
@@ -5055,13 +4973,11 @@ target_include_directories(channel_create_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(channel_create_test
@@ -5083,13 +4999,11 @@ target_include_directories(check_epollexclusive
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(check_epollexclusive
@@ -5118,13 +5032,11 @@ target_include_directories(chttp2_hpack_encoder_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(chttp2_hpack_encoder_test
@@ -5147,13 +5059,11 @@ target_include_directories(chttp2_stream_map_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(chttp2_stream_map_test
@@ -5176,13 +5086,11 @@ target_include_directories(chttp2_varint_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(chttp2_varint_test
@@ -5205,13 +5113,11 @@ target_include_directories(combiner_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(combiner_test
@@ -5234,13 +5140,11 @@ target_include_directories(compression_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(compression_test
@@ -5263,13 +5167,11 @@ target_include_directories(concurrent_connectivity_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(concurrent_connectivity_test
@@ -5292,13 +5194,11 @@ target_include_directories(connection_refused_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(connection_refused_test
@@ -5321,13 +5221,11 @@ target_include_directories(dns_resolver_connectivity_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(dns_resolver_connectivity_test
@@ -5350,13 +5248,11 @@ target_include_directories(dns_resolver_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(dns_resolver_test
@@ -5380,13 +5276,11 @@ target_include_directories(dualstack_socket_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(dualstack_socket_test
@@ -5410,13 +5304,11 @@ target_include_directories(endpoint_pair_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(endpoint_pair_test
@@ -5439,13 +5331,11 @@ target_include_directories(error_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(error_test
@@ -5469,13 +5359,11 @@ target_include_directories(ev_epollsig_linux_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(ev_epollsig_linux_test
@@ -5499,13 +5387,11 @@ target_include_directories(fake_resolver_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(fake_resolver_test
@@ -5530,13 +5416,11 @@ target_include_directories(fake_transport_security_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(fake_transport_security_test
@@ -5560,13 +5444,11 @@ target_include_directories(fd_conservation_posix_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(fd_conservation_posix_test
@@ -5591,13 +5473,11 @@ target_include_directories(fd_posix_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(fd_posix_test
@@ -5621,13 +5501,11 @@ target_include_directories(fling_client
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(fling_client
@@ -5650,13 +5528,11 @@ target_include_directories(fling_server
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(fling_server
@@ -5680,13 +5556,11 @@ target_include_directories(fling_stream_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(fling_stream_test
@@ -5711,13 +5585,11 @@ target_include_directories(fling_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(fling_test
@@ -5742,13 +5614,11 @@ target_include_directories(goaway_server_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(goaway_server_test
@@ -5772,13 +5642,11 @@ target_include_directories(gpr_avl_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_avl_test
@@ -5799,13 +5667,11 @@ target_include_directories(gpr_cmdline_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_cmdline_test
@@ -5826,13 +5692,11 @@ target_include_directories(gpr_cpu_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_cpu_test
@@ -5853,13 +5717,11 @@ target_include_directories(gpr_env_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_env_test
@@ -5880,13 +5742,11 @@ target_include_directories(gpr_host_port_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_host_port_test
@@ -5907,13 +5767,11 @@ target_include_directories(gpr_log_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_log_test
@@ -5934,13 +5792,11 @@ target_include_directories(gpr_manual_constructor_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_manual_constructor_test
@@ -5961,13 +5817,11 @@ target_include_directories(gpr_mpscq_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_mpscq_test
@@ -5988,13 +5842,11 @@ target_include_directories(gpr_spinlock_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_spinlock_test
@@ -6015,13 +5867,11 @@ target_include_directories(gpr_string_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_string_test
@@ -6042,13 +5892,11 @@ target_include_directories(gpr_sync_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_sync_test
@@ -6069,13 +5917,11 @@ target_include_directories(gpr_thd_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_thd_test
@@ -6096,13 +5942,11 @@ target_include_directories(gpr_time_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_time_test
@@ -6123,13 +5967,11 @@ target_include_directories(gpr_tls_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_tls_test
@@ -6150,13 +5992,11 @@ target_include_directories(gpr_useful_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gpr_useful_test
@@ -6177,13 +6017,11 @@ target_include_directories(grpc_auth_context_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_auth_context_test
@@ -6206,13 +6044,11 @@ target_include_directories(grpc_b64_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_b64_test
@@ -6235,13 +6071,11 @@ target_include_directories(grpc_byte_buffer_reader_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_byte_buffer_reader_test
@@ -6264,13 +6098,11 @@ target_include_directories(grpc_channel_args_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_channel_args_test
@@ -6293,13 +6125,11 @@ target_include_directories(grpc_channel_stack_builder_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_channel_stack_builder_test
@@ -6322,13 +6152,11 @@ target_include_directories(grpc_channel_stack_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_channel_stack_test
@@ -6351,13 +6179,11 @@ target_include_directories(grpc_completion_queue_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_completion_queue_test
@@ -6380,13 +6206,11 @@ target_include_directories(grpc_completion_queue_threading_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_completion_queue_threading_test
@@ -6408,13 +6232,11 @@ target_include_directories(grpc_create_jwt
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_create_jwt
@@ -6444,13 +6266,11 @@ target_include_directories(grpc_credentials_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_credentials_test
@@ -6473,13 +6293,11 @@ target_include_directories(grpc_fetch_oauth2
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_fetch_oauth2
@@ -6502,13 +6320,11 @@ target_include_directories(grpc_invalid_channel_args_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_invalid_channel_args_test
@@ -6532,13 +6348,11 @@ target_include_directories(grpc_json_token_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_json_token_test
@@ -6562,13 +6376,11 @@ target_include_directories(grpc_jwt_verifier_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_jwt_verifier_test
@@ -6590,13 +6402,11 @@ target_include_directories(grpc_print_google_default_creds_token
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_print_google_default_creds_token
@@ -6625,13 +6435,11 @@ target_include_directories(grpc_security_connector_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_security_connector_test
@@ -6654,13 +6462,11 @@ target_include_directories(grpc_ssl_credentials_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_ssl_credentials_test
@@ -6682,13 +6488,11 @@ target_include_directories(grpc_verify_jwt
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(grpc_verify_jwt
@@ -6718,13 +6522,11 @@ target_include_directories(handshake_client
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(handshake_client
@@ -6751,13 +6553,11 @@ target_include_directories(handshake_server
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(handshake_server
@@ -6784,13 +6584,11 @@ target_include_directories(handshake_server_with_readahead_handshaker
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(handshake_server_with_readahead_handshaker
@@ -6815,13 +6613,11 @@ target_include_directories(histogram_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(histogram_test
@@ -6842,13 +6638,11 @@ target_include_directories(hpack_parser_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(hpack_parser_test
@@ -6871,13 +6665,11 @@ target_include_directories(hpack_table_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(hpack_table_test
@@ -6900,13 +6692,11 @@ target_include_directories(http_parser_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(http_parser_test
@@ -6929,13 +6719,11 @@ target_include_directories(httpcli_format_request_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(httpcli_format_request_test
@@ -6959,13 +6747,11 @@ target_include_directories(httpcli_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(httpcli_test
@@ -6990,13 +6776,11 @@ target_include_directories(httpscli_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(httpscli_test
@@ -7020,13 +6804,11 @@ target_include_directories(init_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(init_test
@@ -7049,13 +6831,11 @@ target_include_directories(invalid_call_argument_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(invalid_call_argument_test
@@ -7078,13 +6858,11 @@ target_include_directories(json_rewrite
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(json_rewrite
@@ -7105,13 +6883,11 @@ target_include_directories(json_rewrite_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(json_rewrite_test
@@ -7134,13 +6910,11 @@ target_include_directories(json_stream_error_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(json_stream_error_test
@@ -7163,13 +6937,11 @@ target_include_directories(json_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(json_test
@@ -7192,13 +6964,11 @@ target_include_directories(lame_client_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(lame_client_test
@@ -7221,13 +6991,11 @@ target_include_directories(lb_policies_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(lb_policies_test
@@ -7250,13 +7018,11 @@ target_include_directories(load_file_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(load_file_test
@@ -7279,13 +7045,11 @@ target_include_directories(memory_profile_client
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(memory_profile_client
@@ -7308,13 +7072,11 @@ target_include_directories(memory_profile_server
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(memory_profile_server
@@ -7338,13 +7100,11 @@ target_include_directories(memory_profile_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(memory_profile_test
@@ -7368,13 +7128,11 @@ target_include_directories(message_compress_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(message_compress_test
@@ -7397,13 +7155,11 @@ target_include_directories(minimal_stack_is_minimal_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(minimal_stack_is_minimal_test
@@ -7426,13 +7182,11 @@ target_include_directories(multiple_server_queues_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(multiple_server_queues_test
@@ -7455,13 +7209,11 @@ target_include_directories(murmur_hash_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(murmur_hash_test
@@ -7482,13 +7234,11 @@ target_include_directories(no_server_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(no_server_test
@@ -7511,13 +7261,11 @@ target_include_directories(num_external_connectivity_watchers_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(num_external_connectivity_watchers_test
@@ -7540,13 +7288,11 @@ target_include_directories(parse_address_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(parse_address_test
@@ -7569,13 +7315,11 @@ target_include_directories(percent_encoding_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(percent_encoding_test
@@ -7599,13 +7343,11 @@ target_include_directories(pollset_set_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(pollset_set_test
@@ -7630,13 +7372,11 @@ target_include_directories(resolve_address_posix_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(resolve_address_posix_test
@@ -7660,13 +7400,11 @@ target_include_directories(resolve_address_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(resolve_address_test
@@ -7689,13 +7427,11 @@ target_include_directories(resource_quota_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(resource_quota_test
@@ -7718,13 +7454,11 @@ target_include_directories(secure_channel_create_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(secure_channel_create_test
@@ -7747,13 +7481,11 @@ target_include_directories(secure_endpoint_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(secure_endpoint_test
@@ -7776,13 +7508,11 @@ target_include_directories(sequential_connectivity_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(sequential_connectivity_test
@@ -7805,13 +7535,11 @@ target_include_directories(server_chttp2_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(server_chttp2_test
@@ -7834,13 +7562,11 @@ target_include_directories(server_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(server_test
@@ -7863,13 +7589,11 @@ target_include_directories(slice_buffer_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(slice_buffer_test
@@ -7892,13 +7616,11 @@ target_include_directories(slice_hash_table_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(slice_hash_table_test
@@ -7921,13 +7643,11 @@ target_include_directories(slice_string_helpers_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(slice_string_helpers_test
@@ -7950,13 +7670,11 @@ target_include_directories(slice_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(slice_test
@@ -7979,13 +7697,11 @@ target_include_directories(sockaddr_resolver_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(sockaddr_resolver_test
@@ -8008,13 +7724,11 @@ target_include_directories(sockaddr_utils_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(sockaddr_utils_test
@@ -8038,13 +7752,11 @@ target_include_directories(socket_utils_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(socket_utils_test
@@ -8070,13 +7782,11 @@ target_include_directories(ssl_transport_security_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(ssl_transport_security_test
@@ -8099,13 +7809,11 @@ target_include_directories(status_conversion_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(status_conversion_test
@@ -8128,13 +7836,11 @@ target_include_directories(stream_compression_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(stream_compression_test
@@ -8157,13 +7863,11 @@ target_include_directories(stream_owned_slice_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(stream_owned_slice_test
@@ -8187,13 +7891,11 @@ target_include_directories(tcp_client_posix_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(tcp_client_posix_test
@@ -8217,13 +7919,11 @@ target_include_directories(tcp_client_uv_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(tcp_client_uv_test
@@ -8247,13 +7947,11 @@ target_include_directories(tcp_posix_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(tcp_posix_test
@@ -8278,13 +7976,11 @@ target_include_directories(tcp_server_posix_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(tcp_server_posix_test
@@ -8308,13 +8004,11 @@ target_include_directories(tcp_server_uv_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(tcp_server_uv_test
@@ -8337,13 +8031,11 @@ target_include_directories(time_averaged_stats_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(time_averaged_stats_test
@@ -8366,13 +8058,11 @@ target_include_directories(timeout_encoding_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(timeout_encoding_test
@@ -8395,13 +8085,11 @@ target_include_directories(timer_heap_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(timer_heap_test
@@ -8424,13 +8112,11 @@ target_include_directories(timer_list_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(timer_list_test
@@ -8453,13 +8139,11 @@ target_include_directories(transport_connectivity_state_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(transport_connectivity_state_test
@@ -8482,13 +8166,11 @@ target_include_directories(transport_metadata_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(transport_metadata_test
@@ -8512,13 +8194,11 @@ target_include_directories(transport_security_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(transport_security_test
@@ -8543,13 +8223,11 @@ target_include_directories(udp_server_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(udp_server_test
@@ -8573,13 +8251,11 @@ target_include_directories(uri_parser_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(uri_parser_test
@@ -8603,13 +8279,11 @@ target_include_directories(wakeup_fd_cv_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(wakeup_fd_cv_test
@@ -8635,13 +8309,11 @@ target_include_directories(alarm_cpp_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -8675,13 +8347,11 @@ target_include_directories(async_end2end_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -8715,13 +8385,11 @@ target_include_directories(auth_property_iterator_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -8755,13 +8423,11 @@ target_include_directories(backoff_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -8793,13 +8459,11 @@ target_include_directories(bdp_estimator_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -8834,13 +8498,11 @@ target_include_directories(bm_arena
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -8852,7 +8514,7 @@ target_link_libraries(bm_arena
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -8878,13 +8540,11 @@ target_include_directories(bm_call_create
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -8896,7 +8556,7 @@ target_link_libraries(bm_call_create
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -8922,13 +8582,11 @@ target_include_directories(bm_chttp2_hpack
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -8940,7 +8598,7 @@ target_link_libraries(bm_chttp2_hpack
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -8966,13 +8624,11 @@ target_include_directories(bm_chttp2_transport
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -8984,7 +8640,7 @@ target_link_libraries(bm_chttp2_transport
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -9010,13 +8666,11 @@ target_include_directories(bm_closure
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9028,7 +8682,7 @@ target_link_libraries(bm_closure
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -9054,13 +8708,11 @@ target_include_directories(bm_cq
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9072,7 +8724,7 @@ target_link_libraries(bm_cq
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -9098,13 +8750,11 @@ target_include_directories(bm_cq_multiple_threads
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9116,7 +8766,7 @@ target_link_libraries(bm_cq_multiple_threads
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -9142,13 +8792,11 @@ target_include_directories(bm_error
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9160,7 +8808,7 @@ target_link_libraries(bm_error
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -9186,13 +8834,11 @@ target_include_directories(bm_fullstack_streaming_ping_pong
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9204,7 +8850,7 @@ target_link_libraries(bm_fullstack_streaming_ping_pong
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -9230,13 +8876,11 @@ target_include_directories(bm_fullstack_streaming_pump
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9248,7 +8892,7 @@ target_link_libraries(bm_fullstack_streaming_pump
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -9274,13 +8918,11 @@ target_include_directories(bm_fullstack_trickle
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9292,7 +8934,7 @@ target_link_libraries(bm_fullstack_trickle
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -9319,13 +8961,11 @@ target_include_directories(bm_fullstack_unary_ping_pong
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9337,7 +8977,7 @@ target_link_libraries(bm_fullstack_unary_ping_pong
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -9363,13 +9003,11 @@ target_include_directories(bm_metadata
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9381,7 +9019,7 @@ target_link_libraries(bm_metadata
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -9407,13 +9045,11 @@ target_include_directories(bm_pollset
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9425,7 +9061,7 @@ target_link_libraries(bm_pollset
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_benchmark
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -9450,13 +9086,11 @@ target_include_directories(channel_arguments_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9487,13 +9121,11 @@ target_include_directories(channel_filter_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9524,13 +9156,11 @@ target_include_directories(chttp2_settings_timeout_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9562,13 +9192,11 @@ target_include_directories(cli_call_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9610,13 +9238,11 @@ target_include_directories(client_channel_stress_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9651,13 +9277,11 @@ target_include_directories(client_crash_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9692,13 +9316,11 @@ target_include_directories(client_crash_test_server
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9732,13 +9354,11 @@ target_include_directories(client_lb_end2end_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9807,13 +9427,11 @@ target_include_directories(codegen_test_full
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9881,13 +9499,11 @@ target_include_directories(codegen_test_minimal
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9918,13 +9534,11 @@ target_include_directories(credentials_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9955,13 +9569,11 @@ target_include_directories(cxx_byte_buffer_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -9994,13 +9606,11 @@ target_include_directories(cxx_slice_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10033,13 +9643,11 @@ target_include_directories(cxx_string_ref_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10069,13 +9677,11 @@ target_include_directories(cxx_time_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10108,13 +9714,11 @@ target_include_directories(end2end_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10155,13 +9759,11 @@ target_include_directories(error_details_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10191,13 +9793,11 @@ target_include_directories(exception_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10231,13 +9831,11 @@ target_include_directories(filter_end2end_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10271,13 +9869,11 @@ target_include_directories(generic_end2end_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10318,13 +9914,11 @@ target_include_directories(golden_file_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10355,13 +9949,11 @@ target_include_directories(grpc_cli
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10392,13 +9984,11 @@ target_include_directories(grpc_cpp_plugin
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -10428,13 +10018,11 @@ target_include_directories(grpc_csharp_plugin
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -10464,13 +10052,11 @@ target_include_directories(grpc_node_plugin
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -10500,13 +10086,11 @@ target_include_directories(grpc_objective_c_plugin
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -10536,13 +10120,11 @@ target_include_directories(grpc_php_plugin
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -10572,13 +10154,11 @@ target_include_directories(grpc_python_plugin
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -10608,13 +10188,11 @@ target_include_directories(grpc_ruby_plugin
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE ${_gRPC_PROTO_GENS_DIR}
 )
 
@@ -10661,13 +10239,11 @@ target_include_directories(grpc_tool_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10711,13 +10287,11 @@ target_include_directories(grpclb_api_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10756,13 +10330,11 @@ target_include_directories(grpclb_end2end_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10803,13 +10375,11 @@ target_include_directories(grpclb_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10843,13 +10413,11 @@ target_include_directories(h2_ssl_cert_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10882,13 +10450,11 @@ target_include_directories(health_service_end2end_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10922,13 +10488,11 @@ target_include_directories(http2_client
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -10963,13 +10527,11 @@ target_include_directories(hybrid_end2end_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11004,13 +10566,11 @@ target_include_directories(inproc_sync_unary_ping_pong_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11048,13 +10608,11 @@ target_include_directories(interop_client
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11092,13 +10650,11 @@ target_include_directories(interop_server
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11138,13 +10694,11 @@ target_include_directories(interop_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11179,13 +10733,11 @@ target_include_directories(json_run_localhost
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11221,13 +10773,11 @@ target_include_directories(memory_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11267,13 +10817,11 @@ target_include_directories(metrics_client
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11305,13 +10853,11 @@ target_include_directories(mock_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11345,13 +10891,11 @@ target_include_directories(noop-benchmark
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11362,7 +10906,7 @@ target_include_directories(noop-benchmark
 target_link_libraries(noop-benchmark
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
-  benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
   ${_gRPC_GFLAGS_LIBRARIES}
 )
 
@@ -11380,13 +10924,11 @@ target_include_directories(proto_server_reflection_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11422,13 +10964,11 @@ target_include_directories(proto_utils_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11459,13 +10999,11 @@ target_include_directories(qps_interarrival_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11502,13 +11040,11 @@ target_include_directories(qps_json_driver
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11546,13 +11082,11 @@ target_include_directories(qps_openloop_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11590,13 +11124,11 @@ target_include_directories(qps_worker
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11654,13 +11186,11 @@ target_include_directories(reconnect_interop_client
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11716,13 +11246,11 @@ target_include_directories(reconnect_interop_server
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11759,13 +11287,11 @@ target_include_directories(ref_counted_ptr_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11798,13 +11324,11 @@ target_include_directories(ref_counted_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11837,13 +11361,11 @@ target_include_directories(secure_auth_context_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11878,13 +11400,11 @@ target_include_directories(secure_sync_unary_ping_pong_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11922,13 +11442,11 @@ target_include_directories(server_builder_plugin_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -11976,13 +11494,11 @@ target_include_directories(server_builder_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12016,13 +11532,11 @@ target_include_directories(server_context_test_spouse_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12056,13 +11570,11 @@ target_include_directories(server_crash_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12097,13 +11609,11 @@ target_include_directories(server_crash_test_client
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12151,13 +11661,11 @@ target_include_directories(server_request_call_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12191,13 +11699,11 @@ target_include_directories(shutdown_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12231,13 +11737,11 @@ target_include_directories(stats_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12270,13 +11774,11 @@ target_include_directories(status_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12310,13 +11812,11 @@ target_include_directories(streaming_throughput_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12382,13 +11882,11 @@ target_include_directories(stress_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12423,13 +11921,11 @@ target_include_directories(thread_manager_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12461,13 +11957,11 @@ target_include_directories(thread_stress_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12501,13 +11995,11 @@ target_include_directories(transport_pid_controller_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12541,13 +12033,11 @@ target_include_directories(vector_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12581,13 +12071,11 @@ target_include_directories(writes_per_rpc_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -12620,13 +12108,11 @@ target_include_directories(public_headers_must_be_c89
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(public_headers_must_be_c89
@@ -12646,13 +12132,11 @@ target_include_directories(gen_hpack_tables
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gen_hpack_tables
@@ -12680,13 +12164,11 @@ target_include_directories(gen_legal_metadata_characters
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gen_legal_metadata_characters
@@ -12712,13 +12194,11 @@ target_include_directories(gen_percent_encoding_tables
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(gen_percent_encoding_tables
@@ -12745,13 +12225,11 @@ target_include_directories(badreq_bad_client_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(badreq_bad_client_test
@@ -12776,13 +12254,11 @@ target_include_directories(connection_prefix_bad_client_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(connection_prefix_bad_client_test
@@ -12807,13 +12283,11 @@ target_include_directories(head_of_line_blocking_bad_client_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(head_of_line_blocking_bad_client_test
@@ -12838,13 +12312,11 @@ target_include_directories(headers_bad_client_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(headers_bad_client_test
@@ -12869,13 +12341,11 @@ target_include_directories(initial_settings_frame_bad_client_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(initial_settings_frame_bad_client_test
@@ -12900,13 +12370,11 @@ target_include_directories(server_registered_method_bad_client_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(server_registered_method_bad_client_test
@@ -12931,13 +12399,11 @@ target_include_directories(simple_request_bad_client_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(simple_request_bad_client_test
@@ -12962,13 +12428,11 @@ target_include_directories(unknown_frame_bad_client_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(unknown_frame_bad_client_test
@@ -12993,13 +12457,11 @@ target_include_directories(window_overflow_bad_client_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(window_overflow_bad_client_test
@@ -13025,13 +12487,11 @@ target_include_directories(bad_ssl_cert_server
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(bad_ssl_cert_server
@@ -13057,13 +12517,11 @@ target_include_directories(bad_ssl_cert_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(bad_ssl_cert_test
@@ -13087,13 +12545,11 @@ target_include_directories(h2_census_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_census_test
@@ -13117,13 +12573,11 @@ target_include_directories(h2_compress_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_compress_test
@@ -13147,13 +12601,11 @@ target_include_directories(h2_fakesec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_fakesec_test
@@ -13178,13 +12630,11 @@ target_include_directories(h2_fd_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_fd_test
@@ -13209,13 +12659,11 @@ target_include_directories(h2_full_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_full_test
@@ -13240,13 +12688,11 @@ target_include_directories(h2_full+pipe_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_full+pipe_test
@@ -13271,13 +12717,11 @@ target_include_directories(h2_full+trace_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_full+trace_test
@@ -13301,13 +12745,11 @@ target_include_directories(h2_full+workarounds_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_full+workarounds_test
@@ -13331,13 +12773,11 @@ target_include_directories(h2_http_proxy_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_http_proxy_test
@@ -13361,13 +12801,11 @@ target_include_directories(h2_load_reporting_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_load_reporting_test
@@ -13391,13 +12829,11 @@ target_include_directories(h2_oauth2_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_oauth2_test
@@ -13421,13 +12857,11 @@ target_include_directories(h2_proxy_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_proxy_test
@@ -13451,13 +12885,11 @@ target_include_directories(h2_sockpair_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_sockpair_test
@@ -13481,13 +12913,11 @@ target_include_directories(h2_sockpair+trace_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_sockpair+trace_test
@@ -13511,13 +12941,11 @@ target_include_directories(h2_sockpair_1byte_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_sockpair_1byte_test
@@ -13541,13 +12969,11 @@ target_include_directories(h2_ssl_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_ssl_test
@@ -13571,13 +12997,11 @@ target_include_directories(h2_ssl_proxy_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_ssl_proxy_test
@@ -13602,13 +13026,11 @@ target_include_directories(h2_uds_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_uds_test
@@ -13633,13 +13055,11 @@ target_include_directories(inproc_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(inproc_test
@@ -13663,13 +13083,11 @@ target_include_directories(h2_census_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_census_nosec_test
@@ -13693,13 +13111,11 @@ target_include_directories(h2_compress_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_compress_nosec_test
@@ -13724,13 +13140,11 @@ target_include_directories(h2_fd_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_fd_nosec_test
@@ -13755,13 +13169,11 @@ target_include_directories(h2_full_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_full_nosec_test
@@ -13786,13 +13198,11 @@ target_include_directories(h2_full+pipe_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_full+pipe_nosec_test
@@ -13817,13 +13227,11 @@ target_include_directories(h2_full+trace_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_full+trace_nosec_test
@@ -13847,13 +13255,11 @@ target_include_directories(h2_full+workarounds_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_full+workarounds_nosec_test
@@ -13877,13 +13283,11 @@ target_include_directories(h2_http_proxy_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_http_proxy_nosec_test
@@ -13907,13 +13311,11 @@ target_include_directories(h2_load_reporting_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_load_reporting_nosec_test
@@ -13937,13 +13339,11 @@ target_include_directories(h2_proxy_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_proxy_nosec_test
@@ -13967,13 +13367,11 @@ target_include_directories(h2_sockpair_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_sockpair_nosec_test
@@ -13997,13 +13395,11 @@ target_include_directories(h2_sockpair+trace_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_sockpair+trace_nosec_test
@@ -14027,13 +13423,11 @@ target_include_directories(h2_sockpair_1byte_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_sockpair_1byte_nosec_test
@@ -14058,13 +13452,11 @@ target_include_directories(h2_uds_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(h2_uds_nosec_test
@@ -14089,13 +13481,11 @@ target_include_directories(inproc_nosec_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(inproc_nosec_test
@@ -14122,13 +13512,11 @@ target_include_directories(resolver_component_test_unsecure
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -14165,13 +13553,11 @@ target_include_directories(resolver_component_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -14208,13 +13594,11 @@ target_include_directories(resolver_component_tests_runner_invoker_unsecure
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -14251,13 +13635,11 @@ target_include_directories(resolver_component_tests_runner_invoker
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include
@@ -14292,13 +13674,11 @@ target_include_directories(api_fuzzer_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(api_fuzzer_one_entry
@@ -14322,13 +13702,11 @@ target_include_directories(client_fuzzer_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(client_fuzzer_one_entry
@@ -14352,13 +13730,11 @@ target_include_directories(hpack_parser_fuzzer_test_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(hpack_parser_fuzzer_test_one_entry
@@ -14382,13 +13758,11 @@ target_include_directories(http_request_fuzzer_test_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(http_request_fuzzer_test_one_entry
@@ -14412,13 +13786,11 @@ target_include_directories(http_response_fuzzer_test_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(http_response_fuzzer_test_one_entry
@@ -14442,13 +13814,11 @@ target_include_directories(json_fuzzer_test_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(json_fuzzer_test_one_entry
@@ -14472,13 +13842,11 @@ target_include_directories(nanopb_fuzzer_response_test_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(nanopb_fuzzer_response_test_one_entry
@@ -14502,13 +13870,11 @@ target_include_directories(nanopb_fuzzer_serverlist_test_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(nanopb_fuzzer_serverlist_test_one_entry
@@ -14532,13 +13898,11 @@ target_include_directories(percent_decode_fuzzer_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(percent_decode_fuzzer_one_entry
@@ -14562,13 +13926,11 @@ target_include_directories(percent_encode_fuzzer_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(percent_encode_fuzzer_one_entry
@@ -14592,13 +13954,11 @@ target_include_directories(server_fuzzer_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(server_fuzzer_one_entry
@@ -14622,13 +13982,11 @@ target_include_directories(ssl_server_fuzzer_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(ssl_server_fuzzer_one_entry
@@ -14652,13 +14010,11 @@ target_include_directories(uri_fuzzer_test_one_entry
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
 )
 
 target_link_libraries(uri_fuzzer_test_one_entry

From f3c83bfe3da285be7263920164425f110379b73c Mon Sep 17 00:00:00 2001
From: Muxi Yan <mxyan@google.com>
Date: Tue, 16 Jan 2018 10:55:00 -0800
Subject: [PATCH 106/161] Fix sed error in podspecs on sed 4.4

---
 gRPC-Core.podspec                    |  2 +-
 src/objective-c/BoringSSL.podspec    | 14 +++++++-------
 templates/gRPC-Core.podspec.template |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index 358fad3d983..60943f5b4d6 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -1085,6 +1085,6 @@ Pod::Spec.new do |s|
 
   # TODO (mxyan): Instead of this hack, add include path "third_party" to C core's include path?
   s.prepare_command = <<-END_OF_COMMAND
-    find src/core/ -type f -exec sed -E -i '.back' 's;#include "third_party/nanopb/(.*)";#include <nanopb/\\1>;g' {} \\\;
+    find src/core/ -type f -exec sed -E -i'.back' 's;#include "third_party/nanopb/(.*)";#include <nanopb/\\1>;g' {} \\\;
   END_OF_COMMAND
 end
diff --git a/src/objective-c/BoringSSL.podspec b/src/objective-c/BoringSSL.podspec
index c61afc1a8f7..6e406b0dc9b 100644
--- a/src/objective-c/BoringSSL.podspec
+++ b/src/objective-c/BoringSSL.podspec
@@ -136,12 +136,12 @@ Pod::Spec.new do |s|
     # Replace "const BIGNUM *I" in rsa.h with a lowercase i, as the former fails when including
     # OpenSSL in a Swift bridging header (complex.h defines "I", and it's as if the compiler
     # included it in every bridged header).
-    sed -E -i '.back' 's/\\*I,/*i,/g' include/openssl/rsa.h
+    sed -E -i'.back' 's/\\*I,/*i,/g' include/openssl/rsa.h
 
     # Replace `#include "../crypto/internal.h"` in e_tls.c with `#include "../internal.h"`. The
     # former assumes crypto/ is in the headers search path, which is hard to enforce when using
     # dynamic frameworks. The latters always works, being relative to the current file.
-    sed -E -i '.back' 's/crypto\\///g' crypto/cipher/e_tls.c
+    sed -E -i'.back' 's/crypto\\///g' crypto/cipher/e_tls.c
 
     # Add a module map and an umbrella header
     cat > include/openssl/umbrella.h <<EOF
@@ -197,11 +197,11 @@ Pod::Spec.new do |s|
     # https://github.com/libgit2/libgit2/commit/1ddada422caf8e72ba97dca2568d2bf879fed5f2 and libvpx
     # in https://chromium.googlesource.com/webm/libvpx/+/1bec0c5a7e885ec792f6bb658eb3f34ad8f37b15
     # work around it by removing the include. We need four of its macros, so we expand them here.
-    sed -E -i '.back' '/<inttypes.h>/d' include/openssl/bn.h
-    sed -E -i '.back' 's/PRIu32/"u"/g' include/openssl/bn.h
-    sed -E -i '.back' 's/PRIx32/"x"/g' include/openssl/bn.h
-    sed -E -i '.back' 's/PRIu64/"llu"/g' include/openssl/bn.h
-    sed -E -i '.back' 's/PRIx64/"llx"/g' include/openssl/bn.h
+    sed -E -i'.back' '/<inttypes.h>/d' include/openssl/bn.h
+    sed -E -i'.back' 's/PRIu32/"u"/g' include/openssl/bn.h
+    sed -E -i'.back' 's/PRIx32/"x"/g' include/openssl/bn.h
+    sed -E -i'.back' 's/PRIu64/"llu"/g' include/openssl/bn.h
+    sed -E -i'.back' 's/PRIx64/"llx"/g' include/openssl/bn.h
 
     # This is a bit ridiculous, but requiring people to install Go in order to build is slightly
     # more ridiculous IMO. To save you from scrolling, this is the last part of the podspec.
diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template
index 9785d150e42..da404e2fefa 100644
--- a/templates/gRPC-Core.podspec.template
+++ b/templates/gRPC-Core.podspec.template
@@ -202,6 +202,6 @@
 
     # TODO (mxyan): Instead of this hack, add include path "third_party" to C core's include path?
     s.prepare_command = <<-END_OF_COMMAND
-      find src/core/ -type f -exec sed -E -i '.back' 's;#include "third_party/nanopb/(.*)";#include <nanopb/\\1>;g' {} \\\;
+      find src/core/ -type f -exec sed -E -i'.back' 's;#include "third_party/nanopb/(.*)";#include <nanopb/\\1>;g' {} \\\;
     END_OF_COMMAND
   end

From be448700590c5995dd2bdf95223aa5de96d02160 Mon Sep 17 00:00:00 2001
From: Craig Tiller <ctiller@google.com>
Date: Tue, 16 Jan 2018 12:49:18 -0800
Subject: [PATCH 107/161] Fix issue whereby fuzzer creates infinitely deep
 creds (since this is not actually interesting)

---
 test/core/end2end/fuzzers/api_fuzzer.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/test/core/end2end/fuzzers/api_fuzzer.cc b/test/core/end2end/fuzzers/api_fuzzer.cc
index 967a6d560f3..884cbdb3e56 100644
--- a/test/core/end2end/fuzzers/api_fuzzer.cc
+++ b/test/core/end2end/fuzzers/api_fuzzer.cc
@@ -280,7 +280,11 @@ static grpc_channel_credentials* read_ssl_channel_creds(input_stream* inp) {
   return creds;
 }
 
-static grpc_call_credentials* read_call_creds(input_stream* inp) {
+static grpc_call_credentials* read_call_creds(input_stream* inp, int depth) {
+  if (depth > 64) {
+    // prevent creating infinitely deep call creds
+    return nullptr;
+  }
   switch (next_byte(inp)) {
     default:
       end(inp);
@@ -288,8 +292,8 @@ static grpc_call_credentials* read_call_creds(input_stream* inp) {
     case 0:
       return nullptr;
     case 1: {
-      grpc_call_credentials* c1 = read_call_creds(inp);
-      grpc_call_credentials* c2 = read_call_creds(inp);
+      grpc_call_credentials* c1 = read_call_creds(inp, depth + 1);
+      grpc_call_credentials* c2 = read_call_creds(inp, depth + 1);
       if (c1 != nullptr && c2 != nullptr) {
         grpc_call_credentials* out =
             grpc_composite_call_credentials_create(c1, c2, nullptr);
@@ -338,7 +342,7 @@ static grpc_channel_credentials* read_channel_creds(input_stream* inp) {
       break;
     case 1: {
       grpc_channel_credentials* c1 = read_channel_creds(inp);
-      grpc_call_credentials* c2 = read_call_creds(inp);
+      grpc_call_credentials* c2 = read_call_creds(inp, 0);
       if (c1 != nullptr && c2 != nullptr) {
         grpc_channel_credentials* out =
             grpc_composite_channel_credentials_create(c1, c2, nullptr);

From d7ae4a1c617f319f6c8f31890bdacadc343037b7 Mon Sep 17 00:00:00 2001
From: Craig Tiller <ctiller@google.com>
Date: Tue, 16 Jan 2018 12:51:33 -0800
Subject: [PATCH 108/161] Also stop processing input stream

---
 test/core/end2end/fuzzers/api_fuzzer.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/core/end2end/fuzzers/api_fuzzer.cc b/test/core/end2end/fuzzers/api_fuzzer.cc
index 884cbdb3e56..43c9fa19c6d 100644
--- a/test/core/end2end/fuzzers/api_fuzzer.cc
+++ b/test/core/end2end/fuzzers/api_fuzzer.cc
@@ -283,6 +283,7 @@ static grpc_channel_credentials* read_ssl_channel_creds(input_stream* inp) {
 static grpc_call_credentials* read_call_creds(input_stream* inp, int depth) {
   if (depth > 64) {
     // prevent creating infinitely deep call creds
+    end(inp);
     return nullptr;
   }
   switch (next_byte(inp)) {

From 33cb50096ce04612479e8a64626e5ddc200c6db1 Mon Sep 17 00:00:00 2001
From: Craig Tiller <ctiller@google.com>
Date: Tue, 16 Jan 2018 13:16:21 -0800
Subject: [PATCH 109/161] Add fuzzed example that found this crash

---
 .../api_fuzzer_corpus/fuzz-input-d2ab5        | Bin 0 -> 48866 bytes
 tools/run_tests/generated/tests.json          |  23 ++++++++++++++++++
 2 files changed, 23 insertions(+)
 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/fuzz-input-d2ab5

diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/fuzz-input-d2ab5 b/test/core/end2end/fuzzers/api_fuzzer_corpus/fuzz-input-d2ab5
new file mode 100644
index 0000000000000000000000000000000000000000..1745798b683b60d7934b67a0944b6d2c333e053d
GIT binary patch
literal 48866
zcmeI3U6Ldz4urdHPOvZgx(C_&-)mChM;`(a3aYcRtt{%H!g1UsAs{lVZRWrK`}Mp2
zbkK+ePfdic?Q~L2K7;8UapWFVbtK|nzyBHMmJ$AW=`OTmlsXdEsn-Qu*7eeK{@2%k
z7|x|1hdpW1A^J?G8J8K!XE41Uw_5(giA7LS7gC+3o|xVdQ$;+g>PQ8VihKqyGa%7t
z_A07;S*?gOWu9L^$&k6ccPi;iiPjqN-8g|;4i(eEaO5+XUXR6}*&gO$4DuOVyi|TY
z8Y38%9N)h($<NghTWvtuGIM7ffjg88fBJq{)xO1kyzWM(p1yz%E}N-Js)&0S0u@9m
z@PM>eEVE}znu0M=22TMx5xKj-fugTpvm!Hj=O2cm%N&I$9(iI-LSlipd50?7K!W^;
z{o5|Rwu&8V4Ui}K45rs(@k=ewmp9*;GH(4DW*1E-MJ@RZHgBXh1(1lZj}$|~{DlL>
zh{TNEajr;_pI&jvkn!HH<P~;m@O8t_*K*lR2_0^nPWoROGKvcx(1)lgGc8_NJ;g-C
zE8Uw@GCi>`#l}<-n_@&o^8-G#by=Aezv+1H4?PPbw#^_*YziO|yz`ou9%T;qStY)9
zKVCPp)q_S%Q0i&A&A<3fz6HTu$lyhYN=5F-QS_VgYrtRf-1csnL0p+0T}H2EjM02O
ze*EmMwtv`}-pChQF)3=vXE41Ui(j_q^WU<y5MSPE94&dJEv=-3D)JFl{dO{`Kw=4Z
z?@nx91c+-;G92IUegHKd*KP@bR1m4iXE5SUQ2d#;F%N@~&tUULYEuA-__|Dngt;8p
zN>CGF^Ij`Vt7)!_6hr11A#jIMPR1*``;~ZyAFunEsi!ZX12(pk6QGKC5(=S$NCh5n
z5l3dvmNW%pB4RK6hLlWC>`SpRRm7$kQPDhq@r+%;NE;M&l6-_U)^;+fa7DfInwMLd
zxlgjN`;~ZyAMamTXv74io=n}7aVDZYi}*Q0W9~YNe)C1A2K-C^;(0Cm)`~@Z?*wdG
z(|SHUB}3*^oK@m$_v3XpGW86L&;c7e&o+iC;x<1#RS>Db0}lQ&klC{(O~IHbgQozU
zh}>P^K+*qzPs8mIEQ`L|o12&VmE^~CQya#9xnqMNRm6}OR6(R7A7Kr(olGjMCf&8|
z@#*<aue$`uctv-=67TTi{o5Fgm|(W<$v6|ehp(v&o+C6Sa<_T^Ec(p{Xbt$6ezzD4
zfA)6R-t^uME*nH=DdNpJBk{=gJpY=C-msBMlOQ$^Sz=QFiQwLy81K{YE<#*`lHvH>
z9lz@=0gx^?S~K|!M%)RCKhrkmVG!~eY~Dz13Lp_*m&uSY4_ix3gw1=cFs-J!E>aAc
zV}!sRN;w&?=<ZkI9e%v-W2T<IfDYK$PELR-;z=lk3L+JFz(pLHJzLTgjERW7@EcMx
zJ+Uvv##9lTVnjvr1O9&0Wo1@ePh4%or9%agihP8p!YH)OAWLitAQ8OtnwK7B?$cQ%
zzIH#}zp~JX2}(Vgx+mjIa2GOoj?kFM-RAwX=r><<YQVqrFP_(OZmn3v_fEj3HNKZR
zmJFFwaaM`1-H+GZ$ka10hK-$P8$%Uwn;)Jkh*aPK2Y(sJ?Aem0U`&+3Q-DrH?k;ek
z=zqYc;r0l+MVFy0{YUKIcIov%@LaGP<TIFFkHs&weCnSQSU+ncpTYEwIC77wIudba
zYtB!hWcaQV{Pq8byYyq(Do5UGvvxVxX~tzr@)=C8$C2ObbgW`Jr~(g2Ma44vUB}>+
zYF|vNk1FCe@1I5Ud=QO&$#jkm>NA+$5l8M(Wh2pN-gb;@?(tMn6JdHiZnZogMB16w
zj-gG<zMj5CB-p8Us48&|o)03|v0&JC{Zp5~%32;Zn;wyh>EJZ^45rs(@qgxp6;<vl
z)N!`V^9v{$GMD#GC4DK;S|h$2CveN5VmcU(d<N6&vG|`;q)qtqr^uS?TkOZfm(sqc
znN#gIbo>OGXB*R#)U`K$`BX6-RDlPiqGFjnThbJai86Q!(22<11r8Mb^F4oT3%fYZ
ztVh&m@H&dHkvnpTK6CpRm$AraFufkPTAr^szB6Uq`ZLTfnof#Z@)>O2NNoxr5nmrE
zhJ-mhz|<3)mn^X<fJAUzq!><L(UKwKy<f>I>{R<Z*K*lRX;ZjyI_ZCD$S5v&Kp&!}
z%(Qr6^%N5kuXJxt$@Ij&6dO}TY>E*T%@6p{)@5Z@{HEi%KlCh&*fxVKu_=H=@Xl*q
zdXzcbXO;Nc{dnEXRu396L8+(d>Zxv;%dR$IeNvyn&W>T^hGc+6zbU^4{7e6viLvl!
zZ_oC2aM>U_OA&9*8Hq=>=kwpPwUD{MnYp!071P0Z<Rh&5?POAc#1ih_ofz-a@NPt0
zgOVZh4qR4JUi~c?*WS3{se(vFK7)tIZXo_l+n9$T$Y-#5Bef}jM0{N)L&7|4Ej1A~
z@3q3Tn&!GlF=UPr0(U6oWW1ugUx|14@w$(hdinx7U}HNu0jh{6p%5yFRNw&@ab)&v
zNmDQ;BKE>>NXhiXz7!i%MQn-@6-|Ao(h%X%K90nbIem=H2;_-XLn4D2c*K!=RMnA+
z>7a^ygjK(tOe&CA!ettLekUYD=02TO;%oQg{VNNNm|(W<$v6`ds{Lee)6UPxJu3@S
z^qVg_HQ@6XPwOB%+_>G7`V5X^MefK^^ohL<0z~;DJ_w_<c;xCmfAOG8Y-I0Jo1Z3=
zispYkThl=mvJpK;=Kb4D^(o#1n~S2uUdC&b!K)aBi`<bz^qEft#$_z>8BDLot(H$+
z5!TPz$Y(IUBaYmos*XgQ*_!iHC>g%%1V8_Y-ExyB^7auKUKk9ANPtvS{GG?(m2zK9
ztB)$;Ht(NB^8^04+Fp6TY8}hczv~*mQe}2yRAn5Z*)F{XiDHq@VC`J+OZ-yH^FgEy
zYV8=hwe0cfDU@KR-l3}a-1T~&4<gsGVAyv3Q<uQX>Rb1sh)n^cf=ER^gAsRv;{VJG
zE2?}dQHX4r=NC{iWG?TWO8QcwwMKk5PT-b9#dI(n`3$DlWAQ(yNSpBIPmwj(x7d$|
z_e$T>%&GPpns`O2r6;LtpW}d0F&&s=>5-(!W%g`IOsANMGI$EmiOAgr4ix?KJ%4Nq
zyEx9QN7QHVI*PE7J93CVbNd*VvB+mIy&kt(p07B*GiBWRGt4fUPKsLc8EoE2Z3-X}
zUmq!kggHFG)DxSREU_toL~vcC7*1c&k|E>0U&$-%RQo&Ea@kC2Q@C+D>3?a+C@y$F
zAEKtrw0L3l6cZ7zbZ<_{^u)dt8&gGWiV+pf5BSj5Wo1_UrsKIk^el|nHiIm&DS$-q
z&TC$JlsVjImH68Ic-_oa4;nE+si*1cscxFft~OzPQlG)jj$!16WPnA#DZd7M{>z8f
zHg>piyC?M-9LI{>k)!Andm99Z@<n_QMrrZL)%*OnY%OG0YMZ;YOcm47qR5kcgjK(t
zOe$1Xeo44{@4L-mL0p59A@dGgRuVVH5&&s)^cTnm$!Bo$t)Ak~w2gTfhkOQ`H&UAd
zNW|A=G9=8YGr9JXC1&)FMQ~lD$TP>NhXs%f8ShyozIH!e_c2osmQdkkGgU_w@f`2P
zs31~-2VBIF*|Q~0!I+5H3%?;H(-ZqrY)logDMnN@^_@yXgh%^05>MvzF*YNRCsqxK
z3})aFNA6KoM=GX+D)JFl{dO{`Kw=4(Y4rJ>kPMmobXJM4-H-RLEHq+**}5m=Oh~Bq
zlfg|pKO^_7EKJdFzUb6|&tE*PgY0nQc2DXvIF1#$BS+CE_BIF*<%{?rjMCzftM~lH
zgD$a=y-RI=noKI1|MhH52UW;M^c<P@Z!_^J0W#jRO8mRpkNNIy89sl&dQ)G$A!gq?
zRzaj9pTRxq0*HUtzg8;qFbw$&HgBXh1(1j{W#*0@0(U4G{`8R@g114mZt_h#KdB;~
zl`lpGk&1i<FEdn9{F&`x9>yb|!RC$BrT`Lgrp(+)K;RA~!=FAyR<*C)kIRPFVtQvX
z_4EaF(CwXB0jh{s+VpYU2a?&dB~3*!Q3g)|IuW_Mz=5Lw>L0z#>JpTp2!2GbGABLi
z2KfwLcCwP<ms*~$6YDOu{tUB=rjw$Ud<L60Qkw!u#MeiPAz@y>Wi_Iy)?Tv2<^vMJ
zb&+B?z2cG~<2|dy*Y3ybK4$8XHIF}+%~Uy6#AB>FQbD8w54eaUvu8`1f-w=X7k)!Z
zrYH8L*qAEf-~TraQPF($1F}YxV42f56ozWZ6RU(o1|zP6k=x12P{njmMLxo+-%chK
zNG#zpjXu8<k|A@S&MNV>`|<vjg+@#;TlZv~2?^DHGPr5yXXKuhg(>>PzZ{Bi+q-23
zaXow6TNCXL`7WLSRm7995=E2P+aLgQWN;8hY4OYP`TVzREo4_}n~Rp*=7*<>>EJB+
z2&;ZOnN%RLgxlWPo-cDp5x7GMknt`nk>c!!*jOMHL@M$bjJOjNf2M8B!yx1{*u0V2
z6hI=rE|Vc)4qHz>v3bc7n*vA#*F}mUbBqwUL&=cwo>k&&_v3XRGxhWZba2^BRZ>OV
z!w{$-Qh^7gy<(X?ThbJaiHN=M8&Wbou`k8OR1uqEL`C!b#WQvVBW+OBN%9faSlh{@
z!WH$-YhFI~GWSUqcE1wu@Z<d}3yqkd)RU=uGR{P_XAwU~Xv|$l(I>tYi*Vb!R>ANT
zSEfgo(Mz2!ny<%?pS{(FO*_(v+SK0L#@8qL40iTmD1O<V=Pw?#g-unO1hIL@5}N`@
z1o!U5=0$+G1|`FDzk5HpBN6W63EXuQ{btK*K*nWK@)=C8$KwAx?sv2jU$Kr`*~8>A
YLgtkBD1lg7#2=0_nBEbqyPkx90e>O5`v3p{

literal 0
HcmV?d00001

diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index afeee0b3237..6f36dff8208 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -105417,6 +105417,29 @@
     ], 
     "uses_polling": false
   }, 
+  {
+    "args": [
+      "test/core/end2end/fuzzers/api_fuzzer_corpus/fuzz-input-d2ab5"
+    ], 
+    "ci_platforms": [
+      "linux"
+    ], 
+    "cpu_cost": 0.1, 
+    "exclude_configs": [
+      "tsan"
+    ], 
+    "exclude_iomgrs": [
+      "uv"
+    ], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "api_fuzzer_one_entry", 
+    "platforms": [
+      "mac", 
+      "linux"
+    ], 
+    "uses_polling": false
+  }, 
   {
     "args": [
       "test/core/end2end/fuzzers/api_fuzzer_corpus/poc-c726ee220e980ed6ad17809fd9efe2844ee61555ac08e4f88afd8901cc2dd53a"

From ca3798a8cb13f13ed6dea65d0b91399c1634967d Mon Sep 17 00:00:00 2001
From: Alexander Polcyn <apolcyn@google.com>
Date: Tue, 31 Oct 2017 18:14:06 -0700
Subject: [PATCH 110/161] Add ruby distrib tests that uses ruby 2.0.0

---
 .../ruby_jessie_x64_ruby_2_0_0/Dockerfile     | 40 +++++++++++++++++++
 .../artifacts/distribtest_targets.py          | 14 +++++--
 2 files changed, 50 insertions(+), 4 deletions(-)
 create mode 100644 tools/dockerfile/distribtest/ruby_jessie_x64_ruby_2_0_0/Dockerfile

diff --git a/tools/dockerfile/distribtest/ruby_jessie_x64_ruby_2_0_0/Dockerfile b/tools/dockerfile/distribtest/ruby_jessie_x64_ruby_2_0_0/Dockerfile
new file mode 100644
index 00000000000..ff43c92c9e5
--- /dev/null
+++ b/tools/dockerfile/distribtest/ruby_jessie_x64_ruby_2_0_0/Dockerfile
@@ -0,0 +1,40 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM debian:jessie
+
+# Install Git and basic packages.
+RUN apt-get update && apt-get install -y \
+  curl \
+  gcc && apt-get clean
+
+#==================
+# Ruby dependencies
+
+# Install rvm
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+RUN \curl -sSL https://get.rvm.io | bash -s stable
+
+# Install Ruby 2.1
+RUN /bin/bash -l -c "rvm install ruby-2.0"
+RUN /bin/bash -l -c "rvm use --default ruby-2.0"
+RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
+RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"
+RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.0' >> ~/.bashrc"
+RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
+
+RUN mkdir /var/local/jenkins
+
+# Define the default command.
+CMD ["bash"]
diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py
index b2cc16acd42..fdf094cd011 100644
--- a/tools/run_tests/artifacts/distribtest_targets.py
+++ b/tools/run_tests/artifacts/distribtest_targets.py
@@ -174,11 +174,13 @@ class PythonDistribTest(object):
 class RubyDistribTest(object):
     """Tests Ruby package"""
 
-    def __init__(self, platform, arch, docker_suffix):
-        self.name = 'ruby_%s_%s_%s' % (platform, arch, docker_suffix)
+    def __init__(self, platform, arch, docker_suffix, ruby_version=None):
+        self.name = 'ruby_%s_%s_%s_version_%s' % (platform, arch, docker_suffix,
+                                                  ruby_version or 'unspecified')
         self.platform = platform
         self.arch = arch
         self.docker_suffix = docker_suffix
+        self.ruby_version = ruby_version
         self.labels = ['distribtest', 'ruby', platform, arch, docker_suffix]
 
     def pre_build_jobspecs(self):
@@ -192,10 +194,13 @@ class RubyDistribTest(object):
         if not self.platform == 'linux':
             raise Exception("Not supported yet.")
 
+        dockerfile_name = 'tools/dockerfile/distribtest/ruby_%s_%s' % (
+            self.docker_suffix, self.arch)
+        if self.ruby_version is not None:
+            dockerfile_name += '_%s' % self.ruby_version
         return create_docker_jobspec(
             self.name,
-            'tools/dockerfile/distribtest/ruby_%s_%s' % (self.docker_suffix,
-                                                         self.arch),
+            dockerfile_name,
             'test/distrib/ruby/run_distrib_test.sh %s %s' %
             (arch_to_gem_arch[self.arch], self.platform),
             copy_rel_path='test/distrib')
@@ -314,6 +319,7 @@ def targets():
         RubyDistribTest('linux', 'x64', 'wheezy'),
         RubyDistribTest('linux', 'x64', 'jessie'),
         RubyDistribTest('linux', 'x86', 'jessie'),
+        RubyDistribTest('linux', 'x64', 'jessie', ruby_version='ruby_2_0_0'),
         RubyDistribTest('linux', 'x64', 'centos6'),
         RubyDistribTest('linux', 'x64', 'centos7'),
         RubyDistribTest('linux', 'x64', 'fedora20'),

From 37d9036e0695271fcd19350b8aa0f493ab15f182 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Tue, 16 Jan 2018 14:46:04 -0800
Subject: [PATCH 111/161] Fix check_windows_dlls.sh to pass shellcheck

---
 tools/distrib/check_windows_dlls.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/distrib/check_windows_dlls.sh b/tools/distrib/check_windows_dlls.sh
index 9b03e8bf3e9..c25091edfd7 100755
--- a/tools/distrib/check_windows_dlls.sh
+++ b/tools/distrib/check_windows_dlls.sh
@@ -16,7 +16,7 @@
 set -ex
 
 # change to root directory
-cd $(dirname $0)/../..
+cd "$(dirname "$0")/../.."
 
 bundle
 rake dlls

From 2628664f37e1875cb4517ea3f77829198cd886d0 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Tue, 16 Jan 2018 14:52:21 -0800
Subject: [PATCH 112/161] Fix build_artifact_php.sh to pass shellcheck

---
 tools/run_tests/artifacts/build_artifact_php.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tools/run_tests/artifacts/build_artifact_php.sh b/tools/run_tests/artifacts/build_artifact_php.sh
index bfba956f057..9372dc9b6b5 100755
--- a/tools/run_tests/artifacts/build_artifact_php.sh
+++ b/tools/run_tests/artifacts/build_artifact_php.sh
@@ -13,10 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-PHP_TARGET_ARCH=$1
 set -ex
 
-cd $(dirname $0)/../../..
+cd "$(dirname "$0")/../../.."
 
 mkdir -p "${ARTIFACTS_OUT}"
 

From cd867112e8867b21cce65ee3e18135a6c93829ee Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Tue, 16 Jan 2018 14:53:13 -0800
Subject: [PATCH 113/161] Fix build_artifact_protoc.sh to pass shellcheck

---
 tools/run_tests/artifacts/build_artifact_protoc.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/run_tests/artifacts/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh
index 5c5ab7d78f5..b531fc9a5d4 100755
--- a/tools/run_tests/artifacts/build_artifact_protoc.sh
+++ b/tools/run_tests/artifacts/build_artifact_protoc.sh
@@ -18,7 +18,7 @@ source scl_source enable devtoolset-1.1
 
 set -ex
 
-cd $(dirname $0)/../../..
+cd "$(dirname "$0")/../../.."
 
 make plugins
 

From bcd91f835b6c2c44b7627f75b345bab708cbbbe4 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Tue, 16 Jan 2018 14:59:07 -0800
Subject: [PATCH 114/161] Fix build_artifact_python.sh to pass shellcheck

---
 .../artifacts/build_artifact_python.sh        | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh
index ab5bce04f95..10d8211e238 100755
--- a/tools/run_tests/artifacts/build_artifact_python.sh
+++ b/tools/run_tests/artifacts/build_artifact_python.sh
@@ -15,7 +15,7 @@
 
 set -ex
 
-cd $(dirname $0)/../../..
+cd "$(dirname "$0")/../../.."
 
 export GRPC_PYTHON_USE_CUSTOM_BDIST=0
 export GRPC_PYTHON_BUILD_WITH_CYTHON=1
@@ -29,30 +29,30 @@ ARTIFACT_DIR="$PWD/${ARTIFACTS_OUT}"
 # Build the source distribution first because MANIFEST.in cannot override
 # exclusion of built shared objects among package resources (for some
 # inexplicable reason).
-${SETARCH_CMD} ${PYTHON} setup.py sdist
+${SETARCH_CMD} "${PYTHON}" setup.py sdist
 
 # Wheel has a bug where directories don't get excluded.
 # https://bitbucket.org/pypa/wheel/issues/99/cannot-exclude-directory
-${SETARCH_CMD} ${PYTHON} setup.py bdist_wheel
+${SETARCH_CMD} "${PYTHON}" setup.py bdist_wheel
 
 # Build gRPC tools package distribution
-${PYTHON} tools/distrib/python/make_grpcio_tools.py
+"${PYTHON}" tools/distrib/python/make_grpcio_tools.py
 
 # Build gRPC tools package source distribution
-${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py sdist
+${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py sdist
 
 # Build gRPC tools package binary distribution
-${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel
+${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py bdist_wheel
 
 if [ "$GRPC_BUILD_MANYLINUX_WHEEL" != "" ]
 then
   for wheel in dist/*.whl; do
-    ${AUDITWHEEL} repair $wheel -w "$ARTIFACT_DIR"
-    rm $wheel
+    "${AUDITWHEEL}" repair "$wheel" -w "$ARTIFACT_DIR"
+    rm "$wheel"
   done
   for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do
-    ${AUDITWHEEL} repair $wheel -w "$ARTIFACT_DIR"
-    rm $wheel
+    "${AUDITWHEEL}" repair "$wheel" -w "$ARTIFACT_DIR"
+    rm "$wheel"
   done
 fi
 
@@ -62,17 +62,17 @@ fi
 # are in a docker image or in a virtualenv.
 if [ "$GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS" != "" ]
 then
-  ${PIP} install -rrequirements.txt
-  ${PIP} install grpcio --no-index --find-links "file://$ARTIFACT_DIR/"
-  ${PIP} install grpcio-tools --no-index --find-links "file://$ARTIFACT_DIR/"
+  "${PIP}" install -rrequirements.txt
+  "${PIP}" install grpcio --no-index --find-links "file://$ARTIFACT_DIR/"
+  "${PIP}" install grpcio-tools --no-index --find-links "file://$ARTIFACT_DIR/"
 
   # Build gRPC health-checking source distribution
-  ${SETARCH_CMD} ${PYTHON} src/python/grpcio_health_checking/setup.py \
+  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_health_checking/setup.py \
       preprocess build_package_protos sdist
   cp -r src/python/grpcio_health_checking/dist/* "$ARTIFACT_DIR"
 
   # Build gRPC reflection source distribution
-  ${SETARCH_CMD} ${PYTHON} src/python/grpcio_reflection/setup.py \
+  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_reflection/setup.py \
       preprocess build_package_protos sdist
   cp -r src/python/grpcio_reflection/dist/* "$ARTIFACT_DIR"
 fi

From 2b19f4922db21a77d43808a1f1b2e9475348b1d7 Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Tue, 16 Jan 2018 15:10:27 -0800
Subject: [PATCH 115/161] Code review changes.

---
 src/core/lib/support/orphanable.h      | 9 +++++++--
 src/core/lib/support/ref_counted_ptr.h | 4 ++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/core/lib/support/orphanable.h b/src/core/lib/support/orphanable.h
index 63eda2e08b0..2f537573fd4 100644
--- a/src/core/lib/support/orphanable.h
+++ b/src/core/lib/support/orphanable.h
@@ -31,11 +31,16 @@
 
 namespace grpc_core {
 
-// A base class for orphanable objects.
+// A base class for orphanable objects, which have one external owner
+// but are not necessarily destroyed immediately when the external owner
+// gives up ownership.  Instead, the owner calls the object's Orphan()
+// method, and the object then takes responsibility for its own cleanup
+// and destruction.
 class Orphanable {
  public:
   // Gives up ownership of the object.  The implementation must arrange
-  // to destroy the object without further interaction from the caller.
+  // to eventually destroy the object without further interaction from the
+  // caller.
   virtual void Orphan() GRPC_ABSTRACT;
 
   // Not copyable or movable.
diff --git a/src/core/lib/support/ref_counted_ptr.h b/src/core/lib/support/ref_counted_ptr.h
index 8c8606ca0a1..76ff0bba668 100644
--- a/src/core/lib/support/ref_counted_ptr.h
+++ b/src/core/lib/support/ref_counted_ptr.h
@@ -79,11 +79,11 @@ class RefCountedPtr {
   bool operator==(const RefCountedPtr& other) const {
     return value_ == other.value_;
   }
-  bool operator==(T* other) const { return value_ == other; }
+  bool operator==(const T* other) const { return value_ == other; }
   bool operator!=(const RefCountedPtr& other) const {
     return value_ != other.value_;
   }
-  bool operator!=(T* other) const { return value_ != other; }
+  bool operator!=(const T* other) const { return value_ != other; }
 
  private:
   T* value_ = nullptr;

From 919de03fc99a6e3fcbcb73db3e971ef5900258dc Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Tue, 16 Jan 2018 15:11:15 -0800
Subject: [PATCH 116/161] Fix build_package_php.sh to pass shellcheck

---
 tools/run_tests/artifacts/build_package_php.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/run_tests/artifacts/build_package_php.sh b/tools/run_tests/artifacts/build_package_php.sh
index d2d1e8d4599..85e4dda40a6 100755
--- a/tools/run_tests/artifacts/build_package_php.sh
+++ b/tools/run_tests/artifacts/build_package_php.sh
@@ -15,7 +15,7 @@
 
 set -ex
 
-cd $(dirname $0)/../../..
+cd "$(dirname "$0")/../../.."
 
 mkdir -p artifacts/
-cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true
+cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true

From 368fa048e6e40a02745994caf592d9344823ff09 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Tue, 16 Jan 2018 15:12:04 -0800
Subject: [PATCH 117/161] Fix build_package_python.sh to pass shellcheck

---
 tools/run_tests/artifacts/build_package_python.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh
index 1d9d04e3c0b..d596e350007 100755
--- a/tools/run_tests/artifacts/build_package_python.sh
+++ b/tools/run_tests/artifacts/build_package_python.sh
@@ -15,13 +15,13 @@
 
 set -ex
 
-cd $(dirname $0)/../../..
+cd "$(dirname "$0")/../../.."
 
 mkdir -p artifacts/
 
 # All the python packages have been built in the artifact phase already
 # and we only collect them here to deliver them to the distribtest phase.
-cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true
+cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true
 
 # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz
 # source distribution package, and only one of them will end up

From e8b8cbd7c82fcbd2240a54e29e80637bf3419b4c Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Tue, 16 Jan 2018 15:13:59 -0800
Subject: [PATCH 118/161] Fix build_package_ruby.sh to pass shellcheck

---
 .../run_tests/artifacts/build_package_ruby.sh  | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tools/run_tests/artifacts/build_package_ruby.sh b/tools/run_tests/artifacts/build_package_ruby.sh
index b7e0965ffe7..0283c438434 100755
--- a/tools/run_tests/artifacts/build_package_ruby.sh
+++ b/tools/run_tests/artifacts/build_package_ruby.sh
@@ -15,7 +15,7 @@
 
 set -ex
 
-cd $(dirname $0)/../../..
+cd "$(dirname "$0")/../../.."
 
 base=$(pwd)
 
@@ -23,7 +23,7 @@ mkdir -p artifacts/
 
 # All the ruby packages have been built in the artifact phase already
 # and we only collect them here to deliver them to the distribtest phase.
-cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true
+cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true
 
 well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers )
 
@@ -43,16 +43,16 @@ for arch in {x86,x64}; do
   for plat in {windows,linux,macos}; do
     input_dir="$EXTERNAL_GIT_ROOT/platform=${plat}/artifacts/protoc_${plat}_${arch}"
     output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}"
-    mkdir -p $output_dir/google/protobuf
-    mkdir -p $output_dir/google/protobuf/compiler  # needed for plugin.proto
-    cp $input_dir/protoc* $output_dir/
-    cp $input_dir/grpc_ruby_plugin* $output_dir/
+    mkdir -p "$output_dir"/google/protobuf
+    mkdir -p "$output_dir"/google/protobuf/compiler  # needed for plugin.proto
+    cp "$input_dir"/protoc* "$output_dir"/
+    cp "$input_dir"/grpc_ruby_plugin* "$output_dir"/
     for proto in "${well_known_protos[@]}"; do
-      cp $base/third_party/protobuf/src/google/protobuf/$proto.proto $output_dir/google/protobuf/$proto.proto
+      cp "$base/third_party/protobuf/src/google/protobuf/$proto.proto" "$output_dir/google/protobuf/$proto.proto"
     done
   done
 done
 
-cd $base/src/ruby/tools
+cd "$base/src/ruby/tools"
 gem build grpc-tools.gemspec
-cp ./grpc-tools*.gem $base/artifacts/
+cp ./grpc-tools*.gem "$base/artifacts/"

From 02336a29ac7c2ba8ee68a7a0bae589b713805c20 Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Wed, 3 Jan 2018 22:14:28 -0800
Subject: [PATCH 119/161] Check for failures from run clang tidy

---
 tools/distrib/run_clang_tidy.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/distrib/run_clang_tidy.py b/tools/distrib/run_clang_tidy.py
index 72d7956b68e..bc61d4e79a7 100755
--- a/tools/distrib/run_clang_tidy.py
+++ b/tools/distrib/run_clang_tidy.py
@@ -69,4 +69,5 @@ for filename in args.files:
         shortname=filename,
     ))  #verbose_success=True))
 
-jobset.run(jobs, maxjobs=args.jobs)
+num_fails, res_set = jobset.run(jobs, maxjobs=args.jobs)
+sys.exit(num_fails)

From 5edcfb39c6afaabdd183da9e08fbcc56720fb92f Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Wed, 3 Jan 2018 22:28:07 -0800
Subject: [PATCH 120/161] Dont check every single file

---
 tools/run_tests/sanity/check_clang_tidy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/run_tests/sanity/check_clang_tidy.sh b/tools/run_tests/sanity/check_clang_tidy.sh
index 6c4caa1ee63..a27379253e9 100755
--- a/tools/run_tests/sanity/check_clang_tidy.sh
+++ b/tools/run_tests/sanity/check_clang_tidy.sh
@@ -17,5 +17,5 @@ set -e
 
 make buildtests \
   -j "$(python -c 'import multiprocessing; print multiprocessing.cpu_count()')"
-find src/core src/cpp test/core test/cpp -print0 -name '*.h' -or -name '*.cc' \
+find src/core src/cpp test/core test/cpp -name '*.h' -or -name '*.cc' -print0 \
   | xargs -0 tools/distrib/run_clang_tidy.py "$@"

From 365ea6309bd683d384b58e05fdb58e557b20ba19 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Fri, 12 Jan 2018 15:16:41 -0800
Subject: [PATCH 121/161] Dockerize clang tidy

---
 .../{clang_format.include => clang5.include}  |  2 -
 .../grpc_clang_format/Dockerfile.template     |  6 ++-
 .../grpc_clang_tidy/Dockerfile.template       | 26 +++++++++++++
 .../test/sanity/Dockerfile.template           |  2 +-
 tools/distrib/clang_tidy_code.sh              | 31 +++++++++++++++
 tools/distrib/run_clang_tidy.py               |  3 +-
 tools/dockerfile/grpc_clang_format/Dockerfile |  2 +-
 tools/dockerfile/grpc_clang_tidy/Dockerfile   | 39 +++++++++++++++++++
 .../clang_tidy_all_the_things.sh}             |  9 +++--
 tools/dockerfile/test/sanity/Dockerfile       |  2 -
 10 files changed, 110 insertions(+), 12 deletions(-)
 rename templates/tools/dockerfile/{clang_format.include => clang5.include} (61%)
 create mode 100644 templates/tools/dockerfile/grpc_clang_tidy/Dockerfile.template
 create mode 100755 tools/distrib/clang_tidy_code.sh
 create mode 100644 tools/dockerfile/grpc_clang_tidy/Dockerfile
 rename tools/{run_tests/sanity/check_clang_tidy.sh => dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh} (82%)

diff --git a/templates/tools/dockerfile/clang_format.include b/templates/tools/dockerfile/clang5.include
similarity index 61%
rename from templates/tools/dockerfile/clang_format.include
rename to templates/tools/dockerfile/clang5.include
index 79d0ff286f8..f16439b1d45 100644
--- a/templates/tools/dockerfile/clang_format.include
+++ b/templates/tools/dockerfile/clang5.include
@@ -1,5 +1,3 @@
 RUN apt-get update && apt-get -y install wget xz-utils
 RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
 RUN tar xf clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
-RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
-ENV CLANG_FORMAT=clang-format
diff --git a/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template
index 1ab667c95d3..75048ba70fe 100644
--- a/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template
+++ b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template
@@ -16,8 +16,10 @@
   
   FROM debian:jessie
   
-  <%include file="../clang_format.include"/>
+  <%include file="../clang5.include"/>
+  RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
+  ENV CLANG_FORMAT=clang-format
   ADD clang_format_all_the_things.sh /
   CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"]
   
-  
\ No newline at end of file
+  
diff --git a/templates/tools/dockerfile/grpc_clang_tidy/Dockerfile.template b/templates/tools/dockerfile/grpc_clang_tidy/Dockerfile.template
new file mode 100644
index 00000000000..67d56cace22
--- /dev/null
+++ b/templates/tools/dockerfile/grpc_clang_tidy/Dockerfile.template
@@ -0,0 +1,26 @@
+%YAML 1.2
+--- |
+  # Copyright 2015 gRPC authors.
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+  
+  FROM debian:jessie
+  
+  <%include file="../clang5.include"/>
+  <%include file="../python_deps.include"/>
+  RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/bin/clang-tidy
+  ENV CLANG_TIDY=clang-tidy
+  ADD clang_tidy_all_the_things.sh /
+  CMD ["echo 'Run with tools/distrib/clang_tidy_code.sh'"]
+  
+  
diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template
index c98f7d41761..7453e6c460d 100644
--- a/templates/tools/dockerfile/test/sanity/Dockerfile.template
+++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template
@@ -53,7 +53,7 @@
   RUN chmod +x ./bazel-0.4.4-installer-linux-x86_64.sh
   RUN ./bazel-0.4.4-installer-linux-x86_64.sh
   
-  <%include file="../../clang_format.include"/>
+  <%include file="../../clang5.include"/>
   <%include file="../../run_tests_addons.include"/>
   
   # Define the default command.
diff --git a/tools/distrib/clang_tidy_code.sh b/tools/distrib/clang_tidy_code.sh
new file mode 100755
index 00000000000..c5bfc5ca4cc
--- /dev/null
+++ b/tools/distrib/clang_tidy_code.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to root directory
+cd $(dirname $0)/../..
+REPO_ROOT=$(pwd)
+
+if [ "$CLANG_TIDY_SKIP_DOCKER" == "" ]
+then
+  # build clang-tidy docker image
+  docker build -t grpc_clang_tidy tools/dockerfile/grpc_clang_tidy
+
+  # run clang-tidy against the checked out codebase
+  docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_TIDY_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_tidy /clang_tidy_all_the_things.sh
+else
+  CLANG_tidy_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
+fi
diff --git a/tools/distrib/run_clang_tidy.py b/tools/distrib/run_clang_tidy.py
index bc61d4e79a7..182897e491b 100755
--- a/tools/distrib/run_clang_tidy.py
+++ b/tools/distrib/run_clang_tidy.py
@@ -57,7 +57,8 @@ cmdline = [
     clang_tidy,
     '--checks=-*,%s' % ','.join(GRPC_CHECKS),
     '--warnings-as-errors=%s' % ','.join(GRPC_CHECKS)
-] + ['--extra-arg-before=%s' % arg for arg in extra_args]
+] 
+# + ['--extra-arg-before=%s' % arg for arg in extra_args]
 
 if args.fix:
     cmdline.append('--fix')
diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile
index b3abaef4657..7c2b5650f64 100644
--- a/tools/dockerfile/grpc_clang_format/Dockerfile
+++ b/tools/dockerfile/grpc_clang_format/Dockerfile
@@ -17,8 +17,8 @@ FROM debian:jessie
 RUN apt-get update && apt-get -y install wget xz-utils
 RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
 RUN tar xf clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
+
 RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
 ENV CLANG_FORMAT=clang-format
-
 ADD clang_format_all_the_things.sh /
 CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"]
diff --git a/tools/dockerfile/grpc_clang_tidy/Dockerfile b/tools/dockerfile/grpc_clang_tidy/Dockerfile
new file mode 100644
index 00000000000..ed34fdc107f
--- /dev/null
+++ b/tools/dockerfile/grpc_clang_tidy/Dockerfile
@@ -0,0 +1,39 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM debian:jessie
+
+RUN apt-get update && apt-get -y install wget xz-utils
+RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
+RUN tar xf clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
+
+#====================
+# Python dependencies
+
+# Install dependencies
+
+RUN apt-get update && apt-get install -y \
+    python-all-dev \
+    python3-all-dev \
+    python-pip
+
+# Install Python packages from PyPI
+RUN pip install --upgrade pip==9.0.1
+RUN pip install virtualenv
+RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0
+
+RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/bin/clang-tidy
+ENV CLANG_TIDY=clang-tidy
+ADD clang_tidy_all_the_things.sh /
+CMD ["echo 'Run with tools/distrib/clang_tidy_code.sh'"]
diff --git a/tools/run_tests/sanity/check_clang_tidy.sh b/tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
similarity index 82%
rename from tools/run_tests/sanity/check_clang_tidy.sh
rename to tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
index a27379253e9..a74c7288509 100755
--- a/tools/run_tests/sanity/check_clang_tidy.sh
+++ b/tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
@@ -15,7 +15,10 @@
 
 set -e
 
-make buildtests \
-  -j "$(python -c 'import multiprocessing; print multiprocessing.cpu_count()')"
+# clang format command
+CLANG_TIDY=${CLANG_TIDY:-clang-tidy-5.0}
+
+cd ${CLANG_TIDY_ROOT}
+
 find src/core src/cpp test/core test/cpp -name '*.h' -or -name '*.cc' -print0 \
-  | xargs -0 tools/distrib/run_clang_tidy.py "$@"
+  | xargs -0 tools/distrib/run_clang_tidy.py --fix
diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile
index 6e5a133a698..f3330cfd8d8 100644
--- a/tools/dockerfile/test/sanity/Dockerfile
+++ b/tools/dockerfile/test/sanity/Dockerfile
@@ -108,8 +108,6 @@ RUN ./bazel-0.4.4-installer-linux-x86_64.sh
 RUN apt-get update && apt-get -y install wget xz-utils
 RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
 RUN tar xf clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
-RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
-ENV CLANG_FORMAT=clang-format
 
 # Prepare ccache
 RUN ln -s /usr/bin/ccache /usr/local/bin/gcc

From 1adad729797df4cb8c92c26fcdb5f59a72261fd4 Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Fri, 12 Jan 2018 15:26:30 -0800
Subject: [PATCH 122/161] Add to sanity, dont fix by default

---
 tools/distrib/clang_tidy_code.sh         | 2 +-
 tools/distrib/run_clang_tidy.py          | 3 +--
 tools/run_tests/sanity/sanity_tests.yaml | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/tools/distrib/clang_tidy_code.sh b/tools/distrib/clang_tidy_code.sh
index c5bfc5ca4cc..b9d4368997a 100755
--- a/tools/distrib/clang_tidy_code.sh
+++ b/tools/distrib/clang_tidy_code.sh
@@ -25,7 +25,7 @@ then
   docker build -t grpc_clang_tidy tools/dockerfile/grpc_clang_tidy
 
   # run clang-tidy against the checked out codebase
-  docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_TIDY_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_tidy /clang_tidy_all_the_things.sh
+  docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_TIDY_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_tidy /clang_tidy_all_the_things.sh "$@"
 else
   CLANG_tidy_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
 fi
diff --git a/tools/distrib/run_clang_tidy.py b/tools/distrib/run_clang_tidy.py
index 182897e491b..bc61d4e79a7 100755
--- a/tools/distrib/run_clang_tidy.py
+++ b/tools/distrib/run_clang_tidy.py
@@ -57,8 +57,7 @@ cmdline = [
     clang_tidy,
     '--checks=-*,%s' % ','.join(GRPC_CHECKS),
     '--warnings-as-errors=%s' % ','.join(GRPC_CHECKS)
-] 
-# + ['--extra-arg-before=%s' % arg for arg in extra_args]
+] + ['--extra-arg-before=%s' % arg for arg in extra_args]
 
 if args.fix:
     cmdline.append('--fix')
diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml
index dab991a7b17..efdb4d84b5c 100644
--- a/tools/run_tests/sanity/sanity_tests.yaml
+++ b/tools/run_tests/sanity/sanity_tests.yaml
@@ -14,11 +14,11 @@
   cpu_cost: 3
 - script: tools/distrib/check_copyright.py
 - script: tools/distrib/clang_format_code.sh
+- script: tools/distrib/clang_tidy_code.sh
 - script: tools/distrib/check_trailing_newlines.sh
 - script: tools/distrib/check_nanopb_output.sh
 - script: tools/distrib/check_include_guards.py
 - script: tools/distrib/pylint_code.sh
 - script: tools/distrib/yapf_code.sh
 - script: tools/distrib/python/check_grpcio_tools.py
-- script: tools/run_tests/sanity/check_clang_tidy.sh
   cpu_cost: 1000

From d268793ff3ff64b2b417f2cdbf5bed71feaf3b43 Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Tue, 16 Jan 2018 17:22:24 -0800
Subject: [PATCH 123/161] Fix clang format docker

---
 templates/tools/dockerfile/test/sanity/Dockerfile.template | 2 ++
 tools/dockerfile/test/sanity/Dockerfile                    | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template
index 7453e6c460d..9caa10884dc 100644
--- a/templates/tools/dockerfile/test/sanity/Dockerfile.template
+++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template
@@ -54,6 +54,8 @@
   RUN ./bazel-0.4.4-installer-linux-x86_64.sh
   
   <%include file="../../clang5.include"/>
+  RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
+  ENV CLANG_FORMAT=clang-format
   <%include file="../../run_tests_addons.include"/>
   
   # Define the default command.
diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile
index f3330cfd8d8..a243cbb8e51 100644
--- a/tools/dockerfile/test/sanity/Dockerfile
+++ b/tools/dockerfile/test/sanity/Dockerfile
@@ -109,6 +109,8 @@ RUN apt-get update && apt-get -y install wget xz-utils
 RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
 RUN tar xf clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
 
+RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
+ENV CLANG_FORMAT=clang-format
 # Prepare ccache
 RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
 RUN ln -s /usr/bin/ccache /usr/local/bin/g++

From e737ae9e32823e924c452e98798f3d044e4dd394 Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Tue, 16 Jan 2018 18:02:04 -0800
Subject: [PATCH 124/161] Stop building docker from inside docker

---
 tools/run_tests/run_tests.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index c8e917f1171..6b27d6f875f 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -1127,6 +1127,7 @@ class Sanity(object):
             environ = {'TEST': 'true'}
             if _is_use_docker_child():
                 environ['CLANG_FORMAT_SKIP_DOCKER'] = 'true'
+                environ['CLANG_TIDY_SKIP_DOCKER'] = 'true'
             return [
                 self.config.job_spec(
                     cmd['script'].split(),

From 86ddb5393d452fc30ba3941a286c79c253909abc Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Tue, 16 Jan 2018 18:13:26 -0800
Subject: [PATCH 125/161] Ensure all sanity finds clang-tidy

---
 templates/tools/dockerfile/clang5.include                   | 4 ++++
 .../tools/dockerfile/grpc_clang_format/Dockerfile.template  | 2 --
 .../tools/dockerfile/grpc_clang_tidy/Dockerfile.template    | 2 --
 templates/tools/dockerfile/test/sanity/Dockerfile.template  | 2 --
 tools/dockerfile/grpc_clang_format/Dockerfile               | 4 +++-
 tools/dockerfile/grpc_clang_tidy/Dockerfile                 | 6 ++++--
 tools/dockerfile/test/sanity/Dockerfile                     | 4 +++-
 7 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/templates/tools/dockerfile/clang5.include b/templates/tools/dockerfile/clang5.include
index f16439b1d45..11ff4427879 100644
--- a/templates/tools/dockerfile/clang5.include
+++ b/templates/tools/dockerfile/clang5.include
@@ -1,3 +1,7 @@
 RUN apt-get update && apt-get -y install wget xz-utils
 RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
 RUN tar xf clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
+RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
+ENV CLANG_FORMAT=clang-format
+RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/bin/clang-tidy
+ENV CLANG_TIDY=clang-tidy
diff --git a/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template
index 75048ba70fe..4f24a025c64 100644
--- a/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template
+++ b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template
@@ -17,8 +17,6 @@
   FROM debian:jessie
   
   <%include file="../clang5.include"/>
-  RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
-  ENV CLANG_FORMAT=clang-format
   ADD clang_format_all_the_things.sh /
   CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"]
   
diff --git a/templates/tools/dockerfile/grpc_clang_tidy/Dockerfile.template b/templates/tools/dockerfile/grpc_clang_tidy/Dockerfile.template
index 67d56cace22..f5bceaa5f38 100644
--- a/templates/tools/dockerfile/grpc_clang_tidy/Dockerfile.template
+++ b/templates/tools/dockerfile/grpc_clang_tidy/Dockerfile.template
@@ -18,8 +18,6 @@
   
   <%include file="../clang5.include"/>
   <%include file="../python_deps.include"/>
-  RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/bin/clang-tidy
-  ENV CLANG_TIDY=clang-tidy
   ADD clang_tidy_all_the_things.sh /
   CMD ["echo 'Run with tools/distrib/clang_tidy_code.sh'"]
   
diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template
index 9caa10884dc..7453e6c460d 100644
--- a/templates/tools/dockerfile/test/sanity/Dockerfile.template
+++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template
@@ -54,8 +54,6 @@
   RUN ./bazel-0.4.4-installer-linux-x86_64.sh
   
   <%include file="../../clang5.include"/>
-  RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
-  ENV CLANG_FORMAT=clang-format
   <%include file="../../run_tests_addons.include"/>
   
   # Define the default command.
diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile
index 7c2b5650f64..8801315bc9b 100644
--- a/tools/dockerfile/grpc_clang_format/Dockerfile
+++ b/tools/dockerfile/grpc_clang_format/Dockerfile
@@ -17,8 +17,10 @@ FROM debian:jessie
 RUN apt-get update && apt-get -y install wget xz-utils
 RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
 RUN tar xf clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
-
 RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
 ENV CLANG_FORMAT=clang-format
+RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/bin/clang-tidy
+ENV CLANG_TIDY=clang-tidy
+
 ADD clang_format_all_the_things.sh /
 CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"]
diff --git a/tools/dockerfile/grpc_clang_tidy/Dockerfile b/tools/dockerfile/grpc_clang_tidy/Dockerfile
index ed34fdc107f..9d9d70185b0 100644
--- a/tools/dockerfile/grpc_clang_tidy/Dockerfile
+++ b/tools/dockerfile/grpc_clang_tidy/Dockerfile
@@ -17,6 +17,10 @@ FROM debian:jessie
 RUN apt-get update && apt-get -y install wget xz-utils
 RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
 RUN tar xf clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
+RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
+ENV CLANG_FORMAT=clang-format
+RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/bin/clang-tidy
+ENV CLANG_TIDY=clang-tidy
 
 #====================
 # Python dependencies
@@ -33,7 +37,5 @@ RUN pip install --upgrade pip==9.0.1
 RUN pip install virtualenv
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0
 
-RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/bin/clang-tidy
-ENV CLANG_TIDY=clang-tidy
 ADD clang_tidy_all_the_things.sh /
 CMD ["echo 'Run with tools/distrib/clang_tidy_code.sh'"]
diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile
index a243cbb8e51..7a8e1c09b1d 100644
--- a/tools/dockerfile/test/sanity/Dockerfile
+++ b/tools/dockerfile/test/sanity/Dockerfile
@@ -108,9 +108,11 @@ RUN ./bazel-0.4.4-installer-linux-x86_64.sh
 RUN apt-get update && apt-get -y install wget xz-utils
 RUN wget http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
 RUN tar xf clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz
-
 RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-format /usr/local/bin/clang-format
 ENV CLANG_FORMAT=clang-format
+RUN ln -s /clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin/clang-tidy /usr/local/bin/clang-tidy
+ENV CLANG_TIDY=clang-tidy
+
 # Prepare ccache
 RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
 RUN ln -s /usr/bin/ccache /usr/local/bin/g++

From c4aba517a50afa868de40891bbe696947d2d50d4 Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Tue, 16 Jan 2018 20:02:22 -0800
Subject: [PATCH 126/161] s/tidy/TIDY

---
 tools/distrib/clang_tidy_code.sh                              | 2 +-
 tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/distrib/clang_tidy_code.sh b/tools/distrib/clang_tidy_code.sh
index b9d4368997a..7f3dfb116fa 100755
--- a/tools/distrib/clang_tidy_code.sh
+++ b/tools/distrib/clang_tidy_code.sh
@@ -27,5 +27,5 @@ then
   # run clang-tidy against the checked out codebase
   docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_TIDY_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_tidy /clang_tidy_all_the_things.sh "$@"
 else
-  CLANG_tidy_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
+  CLANG_TIDY_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
 fi
diff --git a/tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh b/tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
index a74c7288509..cb22d75489b 100755
--- a/tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
+++ b/tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -e
+set -ex
 
 # clang format command
 CLANG_TIDY=${CLANG_TIDY:-clang-tidy-5.0}

From 1957fd0a84c674676e155665d8d26588f8e03309 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Tue, 16 Jan 2018 17:22:01 -0800
Subject: [PATCH 127/161] Enable epoll on Python manylinux1

The Python packages built for Linux and uploaded to PyPI are required to
target a standardized platform specification dubbed `manylinux1`, which
tries to cover a vast array of Linux distributions, thereby emulating a
legacy lowest-common-denominator distribution, with an old `glibc` that
does not support `epoll_create1`, but provides the `epoll_create`
interface.  While there are race condition risks associated with
utilizing the latter interface and setting the `O_CLOEXEC` flag
immediately on the file descriptor returned by `epoll_create`, the
payoff is well worth the risks for our Python users, who currently end
up falling back on `poll` polling engine when downloading our Linux
binary packages.
---
 src/core/lib/iomgr/ev_epoll1_linux.cc | 26 ++++++++++++++++++++++++--
 src/core/lib/iomgr/port.h             |  5 ++++-
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc
index 1ab7e516de5..6ec25d761f9 100644
--- a/src/core/lib/iomgr/ev_epoll1_linux.cc
+++ b/src/core/lib/iomgr/ev_epoll1_linux.cc
@@ -26,6 +26,7 @@
 
 #include <assert.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <limits.h>
 #include <poll.h>
 #include <pthread.h>
@@ -84,11 +85,32 @@ typedef struct epoll_set {
 /* The global singleton epoll set */
 static epoll_set g_epoll_set;
 
+static int epoll_create_and_set_flag() {
+#ifdef GRPC_LINUX_EPOLL_CREATE1
+  int fd = epoll_create1(EPOLL_CLOEXEC);
+  if (fd >= 0) {
+    return fd;
+  }
+  gpr_log(GPR_ERROR, "epoll_create1 unavailable");
+  return -1;
+#else
+  int fd = epoll_create(MAX_EPOLL_EVENTS);
+  if (fd < 0) {
+    gpr_log(GPR_ERROR, "epoll_create unavailable");
+    return -1;
+  }
+  if (fcntl(fd, F_SETFD, FD_CLOEXEC) == 0) {
+    return fd;
+  }
+  gpr_log(GPR_ERROR, "fcntl following epoll_create failed");
+  return -1;
+#endif
+}
+
 /* Must be called *only* once */
 static bool epoll_set_init() {
-  g_epoll_set.epfd = epoll_create1(EPOLL_CLOEXEC);
+  g_epoll_set.epfd = epoll_create_and_set_flag();
   if (g_epoll_set.epfd < 0) {
-    gpr_log(GPR_ERROR, "epoll unavailable");
     return false;
   }
 
diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h
index 9fae8c00528..80867ee09f1 100644
--- a/src/core/lib/iomgr/port.h
+++ b/src/core/lib/iomgr/port.h
@@ -67,8 +67,11 @@
 #define GRPC_POSIX_WAKEUP_FD 1
 #define GRPC_TIMER_USE_GENERIC 1
 #ifdef __GLIBC_PREREQ
-#if __GLIBC_PREREQ(2, 9)
+#if __GLIBC_PREREQ(2, 4)
 #define GRPC_LINUX_EPOLL 1
+#endif
+#if __GLIBC_PREREQ(2, 9)
+#define GRPC_LINUX_EPOLL_CREATE1 1
 #define GRPC_LINUX_EVENTFD 1
 #endif
 #if __GLIBC_PREREQ(2, 10)

From a6ff754b93658e03e4a89b42855b8c74496b672e Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Fri, 12 Jan 2018 15:24:32 +0100
Subject: [PATCH 128/161] fix alarm test flake on windows

---
 test/core/surface/alarm_test.cc | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/test/core/surface/alarm_test.cc b/test/core/surface/alarm_test.cc
index 8950603b413..67fc6833a5d 100644
--- a/test/core/surface/alarm_test.cc
+++ b/test/core/surface/alarm_test.cc
@@ -34,8 +34,15 @@ static void* create_test_tag(void) {
 static void shutdown_and_destroy(grpc_completion_queue* cc) {
   grpc_event ev;
   grpc_completion_queue_shutdown(cc);
-  ev =
-      grpc_completion_queue_next(cc, gpr_inf_past(GPR_CLOCK_REALTIME), nullptr);
+  /* By the time grpc_completion_queue_shutdown runs, the cq's internal
+     pending event counter might not have been updated yet by a previous
+     cq_end_op_for_next (which releases a completed event first and only later
+     updates the pending event counter), so we can't rely on a no-polling
+     cq_next to never return GRPC_QUEUE_TIMEOUT. Using a deadline in the future
+     solves the problem. See https://github.com/grpc/grpc/issues/13693.
+     */
+  ev = grpc_completion_queue_next(cc, grpc_timeout_seconds_to_deadline(2),
+                                  nullptr);
   GPR_ASSERT(ev.type == GRPC_QUEUE_SHUTDOWN);
   grpc_completion_queue_destroy(cc);
 }

From 816e8f7f43aff2f3ccba5c998587e92005645236 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Wed, 17 Jan 2018 14:23:04 +0100
Subject: [PATCH 129/161] try avoid deadline_exceeded failure in
 dualstack_socket_test

---
 test/core/end2end/dualstack_socket_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/core/end2end/dualstack_socket_test.cc b/test/core/end2end/dualstack_socket_test.cc
index 2ba1c17c2cd..0c101d0f858 100644
--- a/test/core/end2end/dualstack_socket_test.cc
+++ b/test/core/end2end/dualstack_socket_test.cc
@@ -169,7 +169,7 @@ void test_connect(const char* server_host, const char* client_host, int port,
   } else {
     /* Give up faster when failure is expected.
        BUG: Setting this to 1000 reveals a memory leak (b/18608927). */
-    deadline = ms_from_now(1500);
+    deadline = ms_from_now(3000);
   }
 
   /* Send a trivial request. */

From 52e560a01eba64abd3efec8e8622ae90d5f3d03d Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Wed, 17 Jan 2018 16:01:05 +0100
Subject: [PATCH 130/161] add standalone distribtests

---
 .../linux/grpc_distribtests_standalone.cfg    | 26 ++++++++++++++++
 .../linux/grpc_distribtests_standalone.sh     | 23 ++++++++++++++
 .../windows/grpc_distribtests_standalone.bat  | 31 +++++++++++++++++++
 .../windows/grpc_distribtests_standalone.cfg  | 26 ++++++++++++++++
 4 files changed, 106 insertions(+)
 create mode 100644 tools/internal_ci/linux/grpc_distribtests_standalone.cfg
 create mode 100755 tools/internal_ci/linux/grpc_distribtests_standalone.sh
 create mode 100644 tools/internal_ci/windows/grpc_distribtests_standalone.bat
 create mode 100644 tools/internal_ci/windows/grpc_distribtests_standalone.cfg

diff --git a/tools/internal_ci/linux/grpc_distribtests_standalone.cfg b/tools/internal_ci/linux/grpc_distribtests_standalone.cfg
new file mode 100644
index 00000000000..bc6c8e8f805
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_distribtests_standalone.cfg
@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_standalone.sh"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
diff --git a/tools/internal_ci/linux/grpc_distribtests_standalone.sh b/tools/internal_ci/linux/grpc_distribtests_standalone.sh
new file mode 100755
index 00000000000..084daa9bc6a
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_distribtests_standalone.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+tools/run_tests/task_runner.py -f distribtest linux cpp -j 6
diff --git a/tools/internal_ci/windows/grpc_distribtests_standalone.bat b/tools/internal_ci/windows/grpc_distribtests_standalone.bat
new file mode 100644
index 00000000000..229840bc982
--- /dev/null
+++ b/tools/internal_ci/windows/grpc_distribtests_standalone.bat
@@ -0,0 +1,31 @@
+@rem Copyright 2017 gRPC authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder
+@rem TODO(jtattermusch): get rid of this hack
+rename C:\Python27_32bit Python27_32bits
+rename C:\Python34_32bit Python34_32bits
+rename C:\Python35_32bit Python35_32bits
+rename C:\Python36_32bit Python36_32bits
+
+@rem enter repo root
+cd /d %~dp0\..\..\..
+
+call tools/internal_ci/helper_scripts/prepare_build_windows.bat
+
+python tools/run_tests/task_runner.py -f distribtests windows cpp -j 4 || goto :error
+goto :EOF
+
+:error
+exit /b %errorlevel%
diff --git a/tools/internal_ci/windows/grpc_distribtests_standalone.cfg b/tools/internal_ci/windows/grpc_distribtests_standalone.cfg
new file mode 100644
index 00000000000..33a50fdc456
--- /dev/null
+++ b/tools/internal_ci/windows/grpc_distribtests_standalone.cfg
@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/windows/grpc_distribtests_standalone.bat"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

From 479067b2b5820709c3f5c8dcb073bf066ee648de Mon Sep 17 00:00:00 2001
From: ncteisen <ncteisen@gmail.com>
Date: Wed, 17 Jan 2018 07:55:07 -0800
Subject: [PATCH 131/161] do not fix automatically

---
 tools/distrib/clang_tidy_code.sh                              | 4 +++-
 tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/tools/distrib/clang_tidy_code.sh b/tools/distrib/clang_tidy_code.sh
index 7f3dfb116fa..5da86aa2775 100755
--- a/tools/distrib/clang_tidy_code.sh
+++ b/tools/distrib/clang_tidy_code.sh
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+echo "NOTE: to automagically apply fixes, invoke with --fix"
+
 set -ex
 
 # change to root directory
@@ -27,5 +29,5 @@ then
   # run clang-tidy against the checked out codebase
   docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_TIDY_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_tidy /clang_tidy_all_the_things.sh "$@"
 else
-  CLANG_TIDY_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
+  CLANG_TIDY_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh "$@"
 fi
diff --git a/tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh b/tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
index cb22d75489b..1a82dd52b79 100755
--- a/tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
+++ b/tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh
@@ -21,4 +21,4 @@ CLANG_TIDY=${CLANG_TIDY:-clang-tidy-5.0}
 cd ${CLANG_TIDY_ROOT}
 
 find src/core src/cpp test/core test/cpp -name '*.h' -or -name '*.cc' -print0 \
-  | xargs -0 tools/distrib/run_clang_tidy.py --fix
+  | xargs -0 tools/distrib/run_clang_tidy.py "$@"

From d9c288bba7ad2da82bb14ed48ff3aa5a288e609f Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Wed, 17 Jan 2018 09:10:06 -0800
Subject: [PATCH 132/161] generate_projects

---
 CMakeLists.txt | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 38bb92a5a87..97d68cc62c7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -10925,13 +10925,11 @@ target_include_directories(orphanable_test
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
   PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${PROTOBUF_ROOT_DIR}/src
-  PRIVATE ${BENCHMARK_ROOT_DIR}/include
-  PRIVATE ${ZLIB_ROOT_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
-  PRIVATE ${CARES_INCLUDE_DIR}
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
-  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
   PRIVATE third_party/googletest/googletest/include
   PRIVATE third_party/googletest/googletest
   PRIVATE third_party/googletest/googlemock/include

From 53e0b74a0aa649ca895a5d70f96848682541ad52 Mon Sep 17 00:00:00 2001
From: Muxi Yan <muxi@users.noreply.github.com>
Date: Wed, 17 Jan 2018 12:02:05 -0800
Subject: [PATCH 133/161] Specify minimum supported versions in README.md

---
 src/objective-c/README.md | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/objective-c/README.md b/src/objective-c/README.md
index e76ee173ea3..40aba0317b3 100644
--- a/src/objective-c/README.md
+++ b/src/objective-c/README.md
@@ -1,5 +1,12 @@
 [![Cocoapods](https://img.shields.io/cocoapods/v/gRPC.svg)](https://cocoapods.org/pods/gRPC)
 # gRPC for Objective-C
+gRPC Objective C library provides Objective C API for users to make gRPC calls on iOS or OS X
+platforms. Currently, the minimum supported iOS version is 7.0 and OS X version is 10.9 (Mavericks).
+
+While gRPC doesn't require the use of an IDL to describe the API of services, using one simplifies
+usage and adds some interoperability guarantees. Here we use [Protocol Buffers][], and provide a
+plugin for the Protobuf Compiler (_protoc_) to generate client libraries to communicate with gRPC
+services.
 
 - [Write your API declaration in proto format](#write-protos)
 - [Integrate a proto library in your project](#cocoapods)
@@ -10,11 +17,6 @@
     - [Install protoc and the gRPC plugin without using Homebrew](#no-homebrew)
     - [Integrate the generated gRPC library without using Cocoapods](#no-cocoapods)
 
-While gRPC doesn't require the use of an IDL to describe the API of services, using one simplifies
-usage and adds some interoperability guarantees. Here we use [Protocol Buffers][], and provide a
-plugin for the Protobuf Compiler (_protoc_) to generate client libraries to communicate with gRPC
-services.
-
 <a name="write-protos"></a>
 ## Write your API declaration in proto format
 

From 156d6e15bb97c803e9b6cc20f13578df6752a398 Mon Sep 17 00:00:00 2001
From: David Garcia Quintas <dgq@google.com>
Date: Wed, 17 Jan 2018 12:25:27 -0800
Subject: [PATCH 134/161] Fix bad merge

---
 src/core/lib/support/ref_counted.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/core/lib/support/ref_counted.h b/src/core/lib/support/ref_counted.h
index 1adf867d65f..48c11f7bbfe 100644
--- a/src/core/lib/support/ref_counted.h
+++ b/src/core/lib/support/ref_counted.h
@@ -45,7 +45,6 @@ class RefCounted {
   // Not copyable nor movable.
   RefCounted(const RefCounted&) = delete;
   RefCounted& operator=(const RefCounted&) = delete;
-  GRPC_ABSTRACT_BASE_CLASS
 
   GRPC_ABSTRACT_BASE_CLASS
 
@@ -101,7 +100,6 @@ class RefCountedWithTracing {
   // Not copyable nor movable.
   RefCountedWithTracing(const RefCountedWithTracing&) = delete;
   RefCountedWithTracing& operator=(const RefCountedWithTracing&) = delete;
-  GRPC_ABSTRACT_BASE_CLASS
 
   GRPC_ABSTRACT_BASE_CLASS
 

From 4adad9997164a901cd2e5d1b7a636d940a86d581 Mon Sep 17 00:00:00 2001
From: David Garcia Quintas <dgq@google.com>
Date: Wed, 17 Jan 2018 13:50:39 -0800
Subject: [PATCH 135/161] fix BUILD

---
 BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/BUILD b/BUILD
index 5e48a2106d0..186d664395a 100644
--- a/BUILD
+++ b/BUILD
@@ -940,6 +940,7 @@ grpc_cc_library(
         "grpc_base",
         "grpc_deadline_filter",
         "ref_counted",
+        "ref_counted_ptr",
     ],
 )
 

From 8b0e9fb17edab249f41391d877d8d03961f4f7ee Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Wed, 17 Jan 2018 13:42:26 -0800
Subject: [PATCH 136/161] Explicitly enable epoll on manylinux1

For some reason, the glibc version check does not enable
GRPC_LINUX_EPOLL on manylinux1. This commit:

* Explicitly enables GRPC_LINUX_LEGACY_EPOLL on MANYLINUX1
* Switches the flag to enable epoll1 IO manager to
  GRPC_LINUX_LEGACY_EPOLL instead of GRPC_LINUX_EPOLL.
  This is to ensure epollex and epollsig that are not yet
  compatible with epoll_create (not epoll_create1) do not
  get activated unintentionally.
---
 src/core/lib/iomgr/ev_epoll1_linux.cc | 27 ++++++++++++---------------
 src/core/lib/iomgr/port.h             |  4 +++-
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc
index 6ec25d761f9..c8899be9856 100644
--- a/src/core/lib/iomgr/ev_epoll1_linux.cc
+++ b/src/core/lib/iomgr/ev_epoll1_linux.cc
@@ -20,8 +20,9 @@
 
 #include <grpc/support/log.h>
 
-/* This polling engine is only relevant on linux kernels supporting epoll() */
-#ifdef GRPC_LINUX_EPOLL
+/* This polling engine is only relevant on linux kernels supporting epoll
+   epoll_create() or epoll_create1() */
+#ifdef GRPC_LINUX_LEGACY_EPOLL
 #include "src/core/lib/iomgr/ev_epoll1_linux.h"
 
 #include <assert.h>
@@ -85,31 +86,27 @@ typedef struct epoll_set {
 /* The global singleton epoll set */
 static epoll_set g_epoll_set;
 
-static int epoll_create_and_set_flag() {
+static int epoll_create_and_cloexec() {
 #ifdef GRPC_LINUX_EPOLL_CREATE1
   int fd = epoll_create1(EPOLL_CLOEXEC);
-  if (fd >= 0) {
-    return fd;
+  if (fd < 0) {
+    gpr_log(GPR_ERROR, "epoll_create1 unavailable");
   }
-  gpr_log(GPR_ERROR, "epoll_create1 unavailable");
-  return -1;
 #else
   int fd = epoll_create(MAX_EPOLL_EVENTS);
   if (fd < 0) {
     gpr_log(GPR_ERROR, "epoll_create unavailable");
+  } else if (fcntl(fd, F_SETFD, FD_CLOEXEC) != 0) {
+    gpr_log(GPR_ERROR, "fcntl following epoll_create failed");
     return -1;
   }
-  if (fcntl(fd, F_SETFD, FD_CLOEXEC) == 0) {
-    return fd;
-  }
-  gpr_log(GPR_ERROR, "fcntl following epoll_create failed");
-  return -1;
 #endif
+  return fd;
 }
 
 /* Must be called *only* once */
 static bool epoll_set_init() {
-  g_epoll_set.epfd = epoll_create_and_set_flag();
+  g_epoll_set.epfd = epoll_create_and_cloexec();
   if (g_epoll_set.epfd < 0) {
     return false;
   }
@@ -1248,7 +1245,7 @@ const grpc_event_engine_vtable* grpc_init_epoll1_linux(bool explicit_request) {
   return &vtable;
 }
 
-#else /* defined(GRPC_LINUX_EPOLL) */
+#else /* defined(GRPC_LINUX_LEGACY_EPOLL) */
 #if defined(GRPC_POSIX_SOCKET)
 #include "src/core/lib/iomgr/ev_epoll1_linux.h"
 /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return
@@ -1257,4 +1254,4 @@ const grpc_event_engine_vtable* grpc_init_epoll1_linux(bool explicit_request) {
   return nullptr;
 }
 #endif /* defined(GRPC_POSIX_SOCKET) */
-#endif /* !defined(GRPC_LINUX_EPOLL) */
+#endif /* !defined(GRPC_LINUX_LEGACY_EPOLL) */
diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h
index 80867ee09f1..2d56ded4e25 100644
--- a/src/core/lib/iomgr/port.h
+++ b/src/core/lib/iomgr/port.h
@@ -37,6 +37,7 @@
 #define GRPC_POSIX_SOCKETUTILS 1
 #define GRPC_POSIX_WAKEUP_FD 1
 #define GRPC_TIMER_USE_GENERIC 1
+#define GRPC_LINUX_LEGACY_EPOLL 1
 #elif defined(GPR_WINDOWS)
 #define GRPC_TIMER_USE_GENERIC 1
 #define GRPC_WINSOCK_SOCKET 1
@@ -68,9 +69,10 @@
 #define GRPC_TIMER_USE_GENERIC 1
 #ifdef __GLIBC_PREREQ
 #if __GLIBC_PREREQ(2, 4)
-#define GRPC_LINUX_EPOLL 1
+#define GRPC_LINUX_LEGACY_EPOLL 1
 #endif
 #if __GLIBC_PREREQ(2, 9)
+#define GRPC_LINUX_EPOLL 1
 #define GRPC_LINUX_EPOLL_CREATE1 1
 #define GRPC_LINUX_EVENTFD 1
 #endif

From fb66900a9ab7729d22aa83a0e60cb80ec8e642c6 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Wed, 17 Jan 2018 15:37:56 -0800
Subject: [PATCH 137/161] Eliminate GRPC_LINUX_LEGACY_EPOLL

Rename:

  GRPC_LINUX_LEGACY_EPOLL to GRPC_LINUX_EPOLL, and
  GRPC_LINUX_EPOLL to GRPC_LINUX_EPOLL_CREATE1
---
 src/core/lib/iomgr/ev_epoll1_linux.cc             |  6 +++---
 src/core/lib/iomgr/ev_epollex_linux.cc            | 10 +++++-----
 src/core/lib/iomgr/ev_epollsig_linux.cc           | 10 +++++-----
 src/core/lib/iomgr/ev_epollsig_linux.h            |  4 ++--
 src/core/lib/iomgr/is_epollexclusive_available.cc |  2 +-
 src/core/lib/iomgr/port.h                         |  6 +++---
 test/core/iomgr/ev_epollsig_linux_test.cc         |  6 +++---
 test/core/iomgr/pollset_set_test.cc               |  6 +++---
 8 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc
index c8899be9856..aa14d5931a3 100644
--- a/src/core/lib/iomgr/ev_epoll1_linux.cc
+++ b/src/core/lib/iomgr/ev_epoll1_linux.cc
@@ -22,7 +22,7 @@
 
 /* This polling engine is only relevant on linux kernels supporting epoll
    epoll_create() or epoll_create1() */
-#ifdef GRPC_LINUX_LEGACY_EPOLL
+#ifdef GRPC_LINUX_EPOLL
 #include "src/core/lib/iomgr/ev_epoll1_linux.h"
 
 #include <assert.h>
@@ -1245,7 +1245,7 @@ const grpc_event_engine_vtable* grpc_init_epoll1_linux(bool explicit_request) {
   return &vtable;
 }
 
-#else /* defined(GRPC_LINUX_LEGACY_EPOLL) */
+#else /* defined(GRPC_LINUX_EPOLL) */
 #if defined(GRPC_POSIX_SOCKET)
 #include "src/core/lib/iomgr/ev_epoll1_linux.h"
 /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return
@@ -1254,4 +1254,4 @@ const grpc_event_engine_vtable* grpc_init_epoll1_linux(bool explicit_request) {
   return nullptr;
 }
 #endif /* defined(GRPC_POSIX_SOCKET) */
-#endif /* !defined(GRPC_LINUX_LEGACY_EPOLL) */
+#endif /* !defined(GRPC_LINUX_EPOLL) */
diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc
index 5f5f45a7a54..e4a2d67e4b4 100644
--- a/src/core/lib/iomgr/ev_epollex_linux.cc
+++ b/src/core/lib/iomgr/ev_epollex_linux.cc
@@ -21,7 +21,7 @@
 #include <grpc/support/log.h>
 
 /* This polling engine is only relevant on linux kernels supporting epoll() */
-#ifdef GRPC_LINUX_EPOLL
+#ifdef GRPC_LINUX_EPOLL_CREATE1
 
 #include "src/core/lib/iomgr/ev_epollex_linux.h"
 
@@ -1442,15 +1442,15 @@ const grpc_event_engine_vtable* grpc_init_epollex_linux(
   return &vtable;
 }
 
-#else /* defined(GRPC_LINUX_EPOLL) */
+#else /* defined(GRPC_LINUX_EPOLL_CREATE1) */
 #if defined(GRPC_POSIX_SOCKET)
 #include "src/core/lib/iomgr/ev_epollex_linux.h"
-/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return
- * NULL */
+/* If GRPC_LINUX_EPOLL_CREATE1 is not defined, it means
+   epoll_create1 is not available. Return NULL */
 const grpc_event_engine_vtable* grpc_init_epollex_linux(
     bool explicitly_requested) {
   return nullptr;
 }
 #endif /* defined(GRPC_POSIX_SOCKET) */
 
-#endif /* !defined(GRPC_LINUX_EPOLL) */
+#endif /* !defined(GRPC_LINUX_EPOLL_CREATE1) */
diff --git a/src/core/lib/iomgr/ev_epollsig_linux.cc b/src/core/lib/iomgr/ev_epollsig_linux.cc
index 8072a6cbedc..3544d4f3a4d 100644
--- a/src/core/lib/iomgr/ev_epollsig_linux.cc
+++ b/src/core/lib/iomgr/ev_epollsig_linux.cc
@@ -22,7 +22,7 @@
 #include <grpc/support/log.h>
 
 /* This polling engine is only relevant on linux kernels supporting epoll() */
-#ifdef GRPC_LINUX_EPOLL
+#ifdef GRPC_LINUX_EPOLL_CREATE1
 
 #include "src/core/lib/iomgr/ev_epollsig_linux.h"
 
@@ -1725,11 +1725,11 @@ const grpc_event_engine_vtable* grpc_init_epollsig_linux(
   return &vtable;
 }
 
-#else /* defined(GRPC_LINUX_EPOLL) */
+#else /* defined(GRPC_LINUX_EPOLL_CREATE1) */
 #if defined(GRPC_POSIX_SOCKET)
 #include "src/core/lib/iomgr/ev_epollsig_linux.h"
-/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return
- * NULL */
+/* If GRPC_LINUX_EPOLL_CREATE1 is not defined, it means
+   epoll_create1 is not available. Return NULL */
 const grpc_event_engine_vtable* grpc_init_epollsig_linux(
     bool explicit_request) {
   return nullptr;
@@ -1737,4 +1737,4 @@ const grpc_event_engine_vtable* grpc_init_epollsig_linux(
 #endif /* defined(GRPC_POSIX_SOCKET) */
 
 void grpc_use_signal(int signum) {}
-#endif /* !defined(GRPC_LINUX_EPOLL) */
+#endif /* !defined(GRPC_LINUX_EPOLL_CREATE1) */
diff --git a/src/core/lib/iomgr/ev_epollsig_linux.h b/src/core/lib/iomgr/ev_epollsig_linux.h
index 5b8aba9d9f9..48178d3713c 100644
--- a/src/core/lib/iomgr/ev_epollsig_linux.h
+++ b/src/core/lib/iomgr/ev_epollsig_linux.h
@@ -24,10 +24,10 @@
 
 const grpc_event_engine_vtable* grpc_init_epollsig_linux(bool explicit_request);
 
-#ifdef GRPC_LINUX_EPOLL
+#ifdef GRPC_LINUX_EPOLL_CREATE1
 void* grpc_fd_get_polling_island(grpc_fd* fd);
 void* grpc_pollset_get_polling_island(grpc_pollset* ps);
 bool grpc_are_polling_islands_equal(void* p, void* q);
-#endif /* defined(GRPC_LINUX_EPOLL) */
+#endif /* defined(GRPC_LINUX_EPOLL_CREATE1) */
 
 #endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H */
diff --git a/src/core/lib/iomgr/is_epollexclusive_available.cc b/src/core/lib/iomgr/is_epollexclusive_available.cc
index e5803532e76..08f9cf2b69d 100644
--- a/src/core/lib/iomgr/is_epollexclusive_available.cc
+++ b/src/core/lib/iomgr/is_epollexclusive_available.cc
@@ -20,7 +20,7 @@
 
 #include "src/core/lib/iomgr/is_epollexclusive_available.h"
 
-#ifdef GRPC_LINUX_EPOLL
+#ifdef GRPC_LINUX_EPOLL_CREATE1
 
 #include <grpc/support/log.h>
 
diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h
index 2d56ded4e25..25090898ed1 100644
--- a/src/core/lib/iomgr/port.h
+++ b/src/core/lib/iomgr/port.h
@@ -37,7 +37,7 @@
 #define GRPC_POSIX_SOCKETUTILS 1
 #define GRPC_POSIX_WAKEUP_FD 1
 #define GRPC_TIMER_USE_GENERIC 1
-#define GRPC_LINUX_LEGACY_EPOLL 1
+#define GRPC_LINUX_EPOLL 1
 #elif defined(GPR_WINDOWS)
 #define GRPC_TIMER_USE_GENERIC 1
 #define GRPC_WINSOCK_SOCKET 1
@@ -69,10 +69,9 @@
 #define GRPC_TIMER_USE_GENERIC 1
 #ifdef __GLIBC_PREREQ
 #if __GLIBC_PREREQ(2, 4)
-#define GRPC_LINUX_LEGACY_EPOLL 1
+#define GRPC_LINUX_EPOLL 1
 #endif
 #if __GLIBC_PREREQ(2, 9)
-#define GRPC_LINUX_EPOLL 1
 #define GRPC_LINUX_EPOLL_CREATE1 1
 #define GRPC_LINUX_EVENTFD 1
 #endif
@@ -82,6 +81,7 @@
 #endif
 #ifndef __GLIBC__
 #define GRPC_LINUX_EPOLL 1
+#define GRPC_LINUX_EPOLL_CREATE1 1
 #define GRPC_LINUX_EVENTFD 1
 #define GRPC_MSG_IOVLEN_TYPE int
 #endif
diff --git a/test/core/iomgr/ev_epollsig_linux_test.cc b/test/core/iomgr/ev_epollsig_linux_test.cc
index e767e01f219..262470300e2 100644
--- a/test/core/iomgr/ev_epollsig_linux_test.cc
+++ b/test/core/iomgr/ev_epollsig_linux_test.cc
@@ -18,7 +18,7 @@
 #include "src/core/lib/iomgr/port.h"
 
 /* This test only relevant on linux systems where epoll() is available */
-#ifdef GRPC_LINUX_EPOLL
+#ifdef GRPC_LINUX_EPOLL_CREATE1
 #include "src/core/lib/iomgr/ev_epollsig_linux.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 
@@ -319,6 +319,6 @@ int main(int argc, char** argv) {
   grpc_shutdown();
   return 0;
 }
-#else  /* defined(GRPC_LINUX_EPOLL) */
+#else  /* defined(GRPC_LINUX_EPOLL_CREATE1) */
 int main(int argc, char** argv) { return 0; }
-#endif /* !defined(GRPC_LINUX_EPOLL) */
+#endif /* !defined(GRPC_LINUX_EPOLL_CREATE1) */
diff --git a/test/core/iomgr/pollset_set_test.cc b/test/core/iomgr/pollset_set_test.cc
index f27079134b8..7d2f59bed44 100644
--- a/test/core/iomgr/pollset_set_test.cc
+++ b/test/core/iomgr/pollset_set_test.cc
@@ -18,7 +18,7 @@
 #include "src/core/lib/iomgr/port.h"
 
 /* This test only relevant on linux systems where epoll is available */
-#ifdef GRPC_LINUX_EPOLL
+#ifdef GRPC_LINUX_EPOLL_CREATE1
 
 #include <errno.h>
 #include <string.h>
@@ -443,6 +443,6 @@ int main(int argc, char** argv) {
   grpc_shutdown();
   return 0;
 }
-#else  /* defined(GRPC_LINUX_EPOLL) */
+#else  /* defined(GRPC_LINUX_EPOLL_CREATE1) */
 int main(int argc, char** argv) { return 0; }
-#endif /* !defined(GRPC_LINUX_EPOLL) */
+#endif /* !defined(GRPC_LINUX_EPOLL_CREATE1) */

From f2a33b0e39a7cca7b2c6dc96f105a73f54702304 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Wed, 17 Jan 2018 18:13:26 -0800
Subject: [PATCH 138/161] Reformat Python docstrings

---
 src/python/grpcio/grpc/__init__.py | 996 +++++++++++++++--------------
 1 file changed, 506 insertions(+), 490 deletions(-)

diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py
index db410d307b1..79793a710e4 100644
--- a/src/python/grpcio/grpc/__init__.py
+++ b/src/python/grpcio/grpc/__init__.py
@@ -35,152 +35,152 @@ class FutureCancelledError(Exception):
 class Future(six.with_metaclass(abc.ABCMeta)):
     """A representation of a computation in another control flow.
 
-  Computations represented by a Future may be yet to be begun, may be ongoing,
-  or may have already completed.
-  """
+    Computations represented by a Future may be yet to be begun,
+    may be ongoing, or may have already completed.
+    """
 
     @abc.abstractmethod
     def cancel(self):
         """Attempts to cancel the computation.
 
-    This method does not block.
+        This method does not block.
 
-    Returns:
-        bool:
-        Returns True if the computation was canceled.
-        Returns False under all other circumstances, for example:
-        1. computation has begun and could not be canceled.
-        2. computation has finished
-        3. computation is scheduled for execution and it is impossible to
-           determine its state without blocking.
-    """
+        Returns:
+            bool:
+            Returns True if the computation was canceled.
+            Returns False under all other circumstances, for example:
+            1. computation has begun and could not be canceled.
+            2. computation has finished
+            3. computation is scheduled for execution and it is impossible
+              to determine its state without blocking.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def cancelled(self):
         """Describes whether the computation was cancelled.
 
-    This method does not block.
+        This method does not block.
 
-    Returns:
-        bool:
-        Returns True if the computation was cancelled before its result became
-        available.
-        False under all other circumstances, for example:
-        1. computation was not cancelled.
-        2. computation's result is available.
-    """
+        Returns:
+            bool:
+            Returns True if the computation was cancelled before its result became
+            available.
+            False under all other circumstances, for example:
+            1. computation was not cancelled.
+            2. computation's result is available.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def running(self):
         """Describes whether the computation is taking place.
 
-    This method does not block.
+        This method does not block.
 
-    Returns:
-        bool:
-        Returns True if the computation is scheduled for execution or currently
-        executing.
-        Returns False if the computation already executed or was cancelled.
-    """
+        Returns:
+            bool:
+            Returns True if the computation is scheduled for execution or
+            currently executing.
+            Returns False if the computation already executed or was cancelled.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def done(self):
         """Describes whether the computation has taken place.
 
-    This method does not block.
+        This method does not block.
 
-    Returns:
-        bool:
-        Returns True if the computation already executed or was cancelled.
-        Returns False if the computation is scheduled for execution or currently
-        executing.
-        This is exactly opposite of the running() method's result.
-    """
+        Returns:
+            bool:
+            Returns True if the computation already executed or was cancelled.
+            Returns False if the computation is scheduled for execution or
+            currently executing.
+            This is exactly opposite of the running() method's result.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def result(self, timeout=None):
         """Returns the result of the computation or raises its exception.
 
-    This method may return immediately or may block.
+        This method may return immediately or may block.
 
-    Args:
-      timeout: The length of time in seconds to wait for the computation to
-        finish or be cancelled. If None, the call will block until the
-        computations's termination.
+        Args:
+          timeout: The length of time in seconds to wait for the computation to
+            finish or be cancelled. If None, the call will block until the
+            computations's termination.
 
-    Returns:
-      The return value of the computation.
+        Returns:
+          The return value of the computation.
 
-    Raises:
-      FutureTimeoutError: If a timeout value is passed and the computation does
-        not terminate within the allotted time.
-      FutureCancelledError: If the computation was cancelled.
-      Exception: If the computation raised an exception, this call will raise
-        the same exception.
-    """
+        Raises:
+          FutureTimeoutError: If a timeout value is passed and the computation
+            does not terminate within the allotted time.
+          FutureCancelledError: If the computation was cancelled.
+          Exception: If the computation raised an exception, this call will
+            raise the same exception.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def exception(self, timeout=None):
         """Return the exception raised by the computation.
 
-    This method may return immediately or may block.
+        This method may return immediately or may block.
 
-    Args:
-      timeout: The length of time in seconds to wait for the computation to
-        terminate or be cancelled. If None, the call will block until the
-        computations's termination.
+        Args:
+          timeout: The length of time in seconds to wait for the computation to
+            terminate or be cancelled. If None, the call will block until the
+            computations's termination.
 
-    Returns:
-        The exception raised by the computation, or None if the computation did
-        not raise an exception.
+        Returns:
+            The exception raised by the computation, or None if the computation
+            did not raise an exception.
 
-    Raises:
-      FutureTimeoutError: If a timeout value is passed and the computation does
-        not terminate within the allotted time.
-      FutureCancelledError: If the computation was cancelled.
-    """
+        Raises:
+          FutureTimeoutError: If a timeout value is passed and the computation
+            does not terminate within the allotted time.
+          FutureCancelledError: If the computation was cancelled.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def traceback(self, timeout=None):
         """Access the traceback of the exception raised by the computation.
 
-    This method may return immediately or may block.
+        This method may return immediately or may block.
 
-    Args:
-      timeout: The length of time in seconds to wait for the computation to
-        terminate or be cancelled. If None, the call will block until the
-        computations's termination.
+        Args:
+          timeout: The length of time in seconds to wait for the computation
+            to terminate or be cancelled. If None, the call will block until
+            the computation's termination.
 
-    Returns:
-        The traceback of the exception raised by the computation, or None if the
-        computation did not raise an exception.
+        Returns:
+            The traceback of the exception raised by the computation, or None
+            if the computation did not raise an exception.
 
-    Raises:
-      FutureTimeoutError: If a timeout value is passed and the computation does
-        not terminate within the allotted time.
-      FutureCancelledError: If the computation was cancelled.
-    """
+        Raises:
+          FutureTimeoutError: If a timeout value is passed and the computation
+            does not terminate within the allotted time.
+          FutureCancelledError: If the computation was cancelled.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def add_done_callback(self, fn):
         """Adds a function to be called at completion of the computation.
 
-    The callback will be passed this Future object describing the outcome of
-    the computation.
+        The callback will be passed this Future object describing the outcome
+        of the computation.
 
-    If the computation has already completed, the callback will be called
-    immediately.
+        If the computation has already completed, the callback will be called
+        immediately.
 
-    Args:
-      fn: A callable taking this Future object as its single parameter.
-    """
+        Args:
+          fn: A callable taking this Future object as its single parameter.
+        """
         raise NotImplementedError()
 
 
@@ -191,14 +191,14 @@ class Future(six.with_metaclass(abc.ABCMeta)):
 class ChannelConnectivity(enum.Enum):
     """Mirrors grpc_connectivity_state in the gRPC Core.
 
-  Attributes:
-    IDLE: The channel is idle.
-    CONNECTING: The channel is connecting.
-    READY: The channel is ready to conduct RPCs.
-    TRANSIENT_FAILURE: The channel has seen a failure from which it expects to
-      recover.
-    SHUTDOWN: The channel has seen a failure from which it cannot recover.
-  """
+    Attributes:
+      IDLE: The channel is idle.
+      CONNECTING: The channel is connecting.
+      READY: The channel is ready to conduct RPCs.
+      TRANSIENT_FAILURE: The channel has seen a failure from which it expects
+        to recover.
+      SHUTDOWN: The channel has seen a failure from which it cannot recover.
+    """
     IDLE = (_cygrpc.ConnectivityState.idle, 'idle')
     CONNECTING = (_cygrpc.ConnectivityState.connecting, 'connecting')
     READY = (_cygrpc.ConnectivityState.ready, 'ready')
@@ -250,44 +250,44 @@ class RpcContext(six.with_metaclass(abc.ABCMeta)):
     def is_active(self):
         """Describes whether the RPC is active or has terminated.
 
-    Returns:
-      bool:
-      True if RPC is active, False otherwise.
-    """
+        Returns:
+          bool:
+          True if RPC is active, False otherwise.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def time_remaining(self):
         """Describes the length of allowed time remaining for the RPC.
 
-    Returns:
-      A nonnegative float indicating the length of allowed time in seconds
-      remaining for the RPC to complete before it is considered to have timed
-      out, or None if no deadline was specified for the RPC.
-    """
+        Returns:
+          A nonnegative float indicating the length of allowed time in seconds
+          remaining for the RPC to complete before it is considered to have
+          timed out, or None if no deadline was specified for the RPC.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def cancel(self):
         """Cancels the RPC.
 
-    Idempotent and has no effect if the RPC has already terminated.
-    """
+        Idempotent and has no effect if the RPC has already terminated.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def add_callback(self, callback):
         """Registers a callback to be called on RPC termination.
 
-    Args:
-      callback: A no-parameter callable to be called on RPC termination.
+        Args:
+          callback: A no-parameter callable to be called on RPC termination.
 
-    Returns:
-      bool:
-        True if the callback was added and will be called later; False if the
-        callback was not added and will not be called (because the RPC
-        already terminated or some other reason).
-    """
+        Returns:
+          bool:
+            True if the callback was added and will be called later; False if
+            the callback was not added and will not be called (because the RPC
+            already terminated or some other reason).
+        """
         raise NotImplementedError()
 
 
@@ -301,44 +301,44 @@ class Call(six.with_metaclass(abc.ABCMeta, RpcContext)):
     def initial_metadata(self):
         """Accesses the initial metadata sent by the server.
 
-    This method blocks until the value is available.
+        This method blocks until the value is available.
 
-    Returns:
-      The initial :term:`metadata`.
-    """
+        Returns:
+          The initial :term:`metadata`.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def trailing_metadata(self):
         """Accesses the trailing metadata sent by the server.
 
-    This method blocks until the value is available.
+        This method blocks until the value is available.
 
-    Returns:
-      The trailing :term:`metadata`.
-    """
+        Returns:
+          The trailing :term:`metadata`.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def code(self):
         """Accesses the status code sent by the server.
 
-    This method blocks until the value is available.
+        This method blocks until the value is available.
 
-    Returns:
-      The StatusCode value for the RPC.
-    """
+        Returns:
+          The StatusCode value for the RPC.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def details(self):
         """Accesses the details sent by the server.
 
-    This method blocks until the value is available.
+        This method blocks until the value is available.
 
-    Returns:
-      The details string of the RPC.
-    """
+        Returns:
+          The details string of the RPC.
+        """
         raise NotImplementedError()
 
 
@@ -578,9 +578,9 @@ class AuthMetadataPlugin(six.with_metaclass(abc.ABCMeta)):
 class ServerCredentials(object):
     """An encapsulation of the data required to open a secure port on a Server.
 
-  This class has no supported interface - it exists to define the type of its
-  instances and its instances exist to be passed to other functions.
-  """
+    This class has no supported interface - it exists to define the type of its
+    instances and its instances exist to be passed to other functions.
+    """
 
     def __init__(self, credentials):
         self._credentials = credentials
@@ -611,61 +611,65 @@ class UnaryUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
     def __call__(self, request, timeout=None, metadata=None, credentials=None):
         """Synchronously invokes the underlying RPC.
 
-    Args:
-      request: The request value for the RPC.
-      timeout: An optional duration of time in seconds to allow for the RPC.
-      metadata: Optional :term:`metadata` to be transmitted to the
-        service-side of the RPC.
-      credentials: An optional CallCredentials for the RPC.
+        Args:
+          request: The request value for the RPC.
+          timeout: An optional duration of time in seconds to allow
+            for the RPC.
+          metadata: Optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC.
 
-    Returns:
-      The response value for the RPC.
+        Returns:
+          The response value for the RPC.
 
-    Raises:
-      RpcError: Indicating that the RPC terminated with non-OK status. The
-        raised RpcError will also be a Call for the RPC affording the RPC's
-        metadata, status code, and details.
-    """
+        Raises:
+          RpcError: Indicating that the RPC terminated with non-OK status. The
+            raised RpcError will also be a Call for the RPC affording the RPC's
+            metadata, status code, and details.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def with_call(self, request, timeout=None, metadata=None, credentials=None):
         """Synchronously invokes the underlying RPC.
 
-    Args:
-      request: The request value for the RPC.
-      timeout: An optional durating of time in seconds to allow for the RPC.
-      metadata: Optional :term:`metadata` to be transmitted to the
-        service-side of the RPC.
-      credentials: An optional CallCredentials for the RPC.
+        Args:
+          request: The request value for the RPC.
+          timeout: An optional durating of time in seconds to allow for
+            the RPC.
+          metadata: Optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC.
 
-    Returns:
-      The response value for the RPC and a Call value for the RPC.
+        Returns:
+          The response value for the RPC and a Call value for the RPC.
 
-    Raises:
-      RpcError: Indicating that the RPC terminated with non-OK status. The
-        raised RpcError will also be a Call for the RPC affording the RPC's
-        metadata, status code, and details.
-    """
+        Raises:
+          RpcError: Indicating that the RPC terminated with non-OK status. The
+            raised RpcError will also be a Call for the RPC affording the RPC's
+            metadata, status code, and details.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def future(self, request, timeout=None, metadata=None, credentials=None):
         """Asynchronously invokes the underlying RPC.
 
-    Args:
-      request: The request value for the RPC.
-      timeout: An optional duration of time in seconds to allow for the RPC.
-      metadata: Optional :term:`metadata` to be transmitted to the
-        service-side of the RPC.
-      credentials: An optional CallCredentials for the RPC.
+        Args:
+          request: The request value for the RPC.
+          timeout: An optional duration of time in seconds to allow for
+            the RPC.
+          metadata: Optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC.
 
-    Returns:
-        An object that is both a Call for the RPC and a Future. In the event of
-        RPC completion, the return Call-Future's result value will be the
-        response message of the RPC. Should the event terminate with non-OK
-        status, the returned Call-Future's exception value will be an RpcError.
-    """
+        Returns:
+            An object that is both a Call for the RPC and a Future.
+            In the event of RPC completion, the return Call-Future's result
+            value will be the response message of the RPC.
+            Should the event terminate with non-OK status,
+            the returned Call-Future's exception value will be an RpcError.
+        """
         raise NotImplementedError()
 
 
@@ -676,19 +680,20 @@ class UnaryStreamMultiCallable(six.with_metaclass(abc.ABCMeta)):
     def __call__(self, request, timeout=None, metadata=None, credentials=None):
         """Invokes the underlying RPC.
 
-    Args:
-      request: The request value for the RPC.
-      timeout: An optional duration of time in seconds to allow for the RPC.
-               If None, the timeout is considered infinite.
-      metadata: An optional :term:`metadata` to be transmitted to the
-        service-side of the RPC.
-      credentials: An optional CallCredentials for the RPC.
+        Args:
+          request: The request value for the RPC.
+          timeout: An optional duration of time in seconds to allow for
+            the RPC. If None, the timeout is considered infinite.
+          metadata: An optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC.
 
-    Returns:
-        An object that is both a Call for the RPC and an iterator of response
-        values. Drawing response values from the returned Call-iterator may
-        raise RpcError indicating termination of the RPC with non-OK status.
-    """
+        Returns:
+            An object that is both a Call for the RPC and an iterator of
+            response values. Drawing response values from the returned
+            Call-iterator may raise RpcError indicating termination of the
+            RPC with non-OK status.
+        """
         raise NotImplementedError()
 
 
@@ -703,22 +708,23 @@ class StreamUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
                  credentials=None):
         """Synchronously invokes the underlying RPC.
 
-    Args:
-      request_iterator: An iterator that yields request values for the RPC.
-      timeout: An optional duration of time in seconds to allow for the RPC.
-               If None, the timeout is considered infinite.
-      metadata: Optional :term:`metadata` to be transmitted to the
-        service-side of the RPC.
-      credentials: An optional CallCredentials for the RPC.
+        Args:
+          request_iterator: An iterator that yields request values for
+            the RPC.
+          timeout: An optional duration of time in seconds to allow for
+            the RPC. If None, the timeout is considered infinite.
+          metadata: Optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC.
 
-    Returns:
-      The response value for the RPC.
+        Returns:
+          The response value for the RPC.
 
-    Raises:
-      RpcError: Indicating that the RPC terminated with non-OK status. The
-        raised RpcError will also implement grpc.Call, affording methods
-        such as metadata, code, and details.
-    """
+        Raises:
+          RpcError: Indicating that the RPC terminated with non-OK status. The
+            raised RpcError will also implement grpc.Call, affording methods
+            such as metadata, code, and details.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
@@ -729,22 +735,23 @@ class StreamUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
                   credentials=None):
         """Synchronously invokes the underlying RPC on the client.
 
-    Args:
-      request_iterator: An iterator that yields request values for the RPC.
-      timeout: An optional duration of time in seconds to allow for the RPC.
-               If None, the timeout is considered infinite.
-      metadata: Optional :term:`metadata` to be transmitted to the
-        service-side of the RPC.
-      credentials: An optional CallCredentials for the RPC.
+        Args:
+          request_iterator: An iterator that yields request values for
+            the RPC.
+          timeout: An optional duration of time in seconds to allow for
+            the RPC. If None, the timeout is considered infinite.
+          metadata: Optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC.
 
-    Returns:
-      The response value for the RPC and a Call object for the RPC.
+        Returns:
+          The response value for the RPC and a Call object for the RPC.
 
-    Raises:
-      RpcError: Indicating that the RPC terminated with non-OK status. The
-        raised RpcError will also be a Call for the RPC affording the RPC's
-        metadata, status code, and details.
-    """
+        Raises:
+          RpcError: Indicating that the RPC terminated with non-OK status. The
+            raised RpcError will also be a Call for the RPC affording the RPC's
+            metadata, status code, and details.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
@@ -755,20 +762,21 @@ class StreamUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
                credentials=None):
         """Asynchronously invokes the underlying RPC on the client.
 
-    Args:
-      request_iterator: An iterator that yields request values for the RPC.
-      timeout: An optional duration of time in seconds to allow for the RPC.
-               If None, the timeout is considered infinite.
-      metadata: Optional :term:`metadata` to be transmitted to the
-        service-side of the RPC.
-      credentials: An optional CallCredentials for the RPC.
+        Args:
+          request_iterator: An iterator that yields request values for the RPC.
+          timeout: An optional duration of time in seconds to allow for
+            the RPC. If None, the timeout is considered infinite.
+          metadata: Optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC.
 
-    Returns:
-        An object that is both a Call for the RPC and a Future. In the event of
-        RPC completion, the return Call-Future's result value will be the
-        response message of the RPC. Should the event terminate with non-OK
-        status, the returned Call-Future's exception value will be an RpcError.
-    """
+        Returns:
+            An object that is both a Call for the RPC and a Future.
+            In the event of RPC completion, the return Call-Future's result value
+            will be the response message of the RPC. Should the event terminate
+            with non-OK status, the returned Call-Future's exception value will
+            be an RpcError.
+        """
         raise NotImplementedError()
 
 
@@ -783,19 +791,20 @@ class StreamStreamMultiCallable(six.with_metaclass(abc.ABCMeta)):
                  credentials=None):
         """Invokes the underlying RPC on the client.
 
-    Args:
-      request_iterator: An iterator that yields request values for the RPC.
-      timeout: An optional duration of time in seconds to allow for the RPC.
-               if not specified the timeout is considered infinite.
-      metadata: Optional :term:`metadata` to be transmitted to the
-        service-side of the RPC.
-      credentials: An optional CallCredentials for the RPC.
+        Args:
+          request_iterator: An iterator that yields request values for the RPC.
+          timeout: An optional duration of time in seconds to allow for
+            the RPC. If not specified, the timeout is considered infinite.
+          metadata: Optional :term:`metadata` to be transmitted to the
+            service-side of the RPC.
+          credentials: An optional CallCredentials for the RPC.
 
-    Returns:
-        An object that is both a Call for the RPC and an iterator of response
-        values. Drawing response values from the returned Call-iterator may
-        raise RpcError indicating termination of the RPC with non-OK status.
-    """
+        Returns:
+            An object that is both a Call for the RPC and an iterator of
+            response values. Drawing response values from the returned
+            Call-iterator may raise RpcError indicating termination of the
+            RPC with non-OK status.
+        """
         raise NotImplementedError()
 
 
@@ -809,31 +818,31 @@ class Channel(six.with_metaclass(abc.ABCMeta)):
     def subscribe(self, callback, try_to_connect=False):
         """Subscribe to this Channel's connectivity state machine.
 
-    A Channel may be in any of the states described by ChannelConnectivity.
-    This method allows application to monitor the state transitions.
-    The typical use case is to debug or gain better visibility into gRPC
-    runtime's state.
+        A Channel may be in any of the states described by ChannelConnectivity.
+        This method allows application to monitor the state transitions.
+        The typical use case is to debug or gain better visibility into gRPC
+        runtime's state.
 
-    Args:
-      callback: A callable to be invoked with ChannelConnectivity argument.
-        ChannelConnectivity describes current state of the channel.
-        The callable will be invoked immediately upon subscription and again for
-        every change to ChannelConnectivity until it is unsubscribed or this
-        Channel object goes out of scope.
-      try_to_connect: A boolean indicating whether or not this Channel should
-        attempt to connect immediately. If set to False, gRPC runtime decides
-        when to connect.
-    """
+        Args:
+          callback: A callable to be invoked with ChannelConnectivity argument.
+            ChannelConnectivity describes current state of the channel.
+            The callable will be invoked immediately upon subscription
+            and again for every change to ChannelConnectivity until it
+            is unsubscribed or this Channel object goes out of scope.
+          try_to_connect: A boolean indicating whether or not this Channel
+            should attempt to connect immediately. If set to False, gRPC
+            runtime decides when to connect.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def unsubscribe(self, callback):
         """Unsubscribes a subscribed callback from this Channel's connectivity.
 
-    Args:
-      callback: A callable previously registered with this Channel from having
-        been passed to its "subscribe" method.
-    """
+        Args:
+          callback: A callable previously registered with this Channel from
+          having been passed to its "subscribe" method.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
@@ -843,16 +852,17 @@ class Channel(six.with_metaclass(abc.ABCMeta)):
                     response_deserializer=None):
         """Creates a UnaryUnaryMultiCallable for a unary-unary method.
 
-    Args:
-      method: The name of the RPC method.
-      request_serializer: Optional behaviour for serializing the request
-        message. Request goes unserialized in case None is passed.
-      response_deserializer: Optional behaviour for deserializing the response
-        message. Response goes undeserialized in case None is passed.
+        Args:
+          method: The name of the RPC method.
+          request_serializer: Optional behaviour for serializing the request
+            message. Request goes unserialized in case None is passed.
+          response_deserializer: Optional behaviour for deserializing the
+            response message. Response goes undeserialized in case None
+            is passed.
 
-    Returns:
-      A UnaryUnaryMultiCallable value for the named unary-unary method.
-    """
+        Returns:
+          A UnaryUnaryMultiCallable value for the named unary-unary method.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
@@ -862,16 +872,17 @@ class Channel(six.with_metaclass(abc.ABCMeta)):
                      response_deserializer=None):
         """Creates a UnaryStreamMultiCallable for a unary-stream method.
 
-    Args:
-      method: The name of the RPC method.
-      request_serializer: Optional behaviour for serializing the request
-        message. Request goes unserialized in case None is passed.
-      response_deserializer: Optional behaviour for deserializing the response
-        message. Response goes undeserialized in case None is passed.
+        Args:
+          method: The name of the RPC method.
+          request_serializer: Optional behaviour for serializing the request
+            message. Request goes unserialized in case None is passed.
+          response_deserializer: Optional behaviour for deserializing the
+            response message. Response goes undeserialized in case None is
+            passed.
 
-    Returns:
-      A UnaryStreamMultiCallable value for the name unary-stream method.
-    """
+        Returns:
+          A UnaryStreamMultiCallable value for the name unary-stream method.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
@@ -881,16 +892,17 @@ class Channel(six.with_metaclass(abc.ABCMeta)):
                      response_deserializer=None):
         """Creates a StreamUnaryMultiCallable for a stream-unary method.
 
-    Args:
-      method: The name of the RPC method.
-      request_serializer: Optional behaviour for serializing the request
-        message. Request goes unserialized in case None is passed.
-      response_deserializer: Optional behaviour for deserializing the response
-        message. Response goes undeserialized in case None is passed.
+        Args:
+          method: The name of the RPC method.
+          request_serializer: Optional behaviour for serializing the request
+            message. Request goes unserialized in case None is passed.
+          response_deserializer: Optional behaviour for deserializing the
+            response message. Response goes undeserialized in case None is
+            passed.
 
-    Returns:
-      A StreamUnaryMultiCallable value for the named stream-unary method.
-    """
+        Returns:
+          A StreamUnaryMultiCallable value for the named stream-unary method.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
@@ -900,16 +912,17 @@ class Channel(six.with_metaclass(abc.ABCMeta)):
                       response_deserializer=None):
         """Creates a StreamStreamMultiCallable for a stream-stream method.
 
-    Args:
-      method: The name of the RPC method.
-      request_serializer: Optional behaviour for serializing the request
-        message. Request goes unserialized in case None is passed.
-      response_deserializer: Optional behaviour for deserializing the response
-        message. Response goes undeserialized in case None is passed.
+        Args:
+          method: The name of the RPC method.
+          request_serializer: Optional behaviour for serializing the request
+            message. Request goes unserialized in case None is passed.
+          response_deserializer: Optional behaviour for deserializing the
+            response message. Response goes undeserialized in case None
+            is passed.
 
-    Returns:
-      A StreamStreamMultiCallable value for the named stream-stream method.
-    """
+        Returns:
+          A StreamStreamMultiCallable value for the named stream-stream method.
+        """
         raise NotImplementedError()
 
 
@@ -923,79 +936,79 @@ class ServicerContext(six.with_metaclass(abc.ABCMeta, RpcContext)):
     def invocation_metadata(self):
         """Accesses the metadata from the sent by the client.
 
-    Returns:
-      The invocation :term:`metadata`.
-    """
+        Returns:
+          The invocation :term:`metadata`.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def peer(self):
         """Identifies the peer that invoked the RPC being serviced.
 
-    Returns:
-      A string identifying the peer that invoked the RPC being serviced.
-      The string format is determined by gRPC runtime.
-    """
+        Returns:
+          A string identifying the peer that invoked the RPC being serviced.
+          The string format is determined by gRPC runtime.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def peer_identities(self):
         """Gets one or more peer identity(s).
 
-      Equivalent to
-      servicer_context.auth_context().get(
-          servicer_context.peer_identity_key())
+        Equivalent to
+        servicer_context.auth_context().get(
+            servicer_context.peer_identity_key())
 
-    Returns:
-      An iterable of the identities, or None if the call is not authenticated.
-      Each identity is returned as a raw bytes type.
-     """
+        Returns:
+          An iterable of the identities, or None if the call is not
+          authenticated. Each identity is returned as a raw bytes type.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def peer_identity_key(self):
         """The auth property used to identify the peer.
 
-    For example, "x509_common_name" or "x509_subject_alternative_name" are
-    used to identify an SSL peer.
+        For example, "x509_common_name" or "x509_subject_alternative_name" are
+        used to identify an SSL peer.
 
-    Returns:
-      The auth property (string) that indicates the
-      peer identity, or None if the call is not authenticated.
-    """
+        Returns:
+          The auth property (string) that indicates the
+          peer identity, or None if the call is not authenticated.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def auth_context(self):
         """Gets the auth context for the call.
 
-      Returns:
-        A map of strings to an iterable of bytes for each auth property.
-      """
+        Returns:
+          A map of strings to an iterable of bytes for each auth property.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def send_initial_metadata(self, initial_metadata):
         """Sends the initial metadata value to the client.
 
-    This method need not be called by implementations if they have no
-    metadata to add to what the gRPC runtime will transmit.
+        This method need not be called by implementations if they have no
+        metadata to add to what the gRPC runtime will transmit.
 
-    Args:
-      initial_metadata: The initial :term:`metadata`.
-    """
+        Args:
+          initial_metadata: The initial :term:`metadata`.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def set_trailing_metadata(self, trailing_metadata):
         """Sends the trailing metadata for the RPC.
 
-    This method need not be called by implementations if they have no
-    metadata to add to what the gRPC runtime will transmit.
+        This method need not be called by implementations if they have no
+        metadata to add to what the gRPC runtime will transmit.
 
-    Args:
-      trailing_metadata: The trailing :term:`metadata`.
-    """
+        Args:
+          trailing_metadata: The trailing :term:`metadata`.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
@@ -1049,44 +1062,45 @@ class ServicerContext(six.with_metaclass(abc.ABCMeta, RpcContext)):
 class RpcMethodHandler(six.with_metaclass(abc.ABCMeta)):
     """An implementation of a single RPC method.
 
-  Attributes:
-    request_streaming: Whether the RPC supports exactly one request message or
-      any arbitrary number of request messages.
-    response_streaming: Whether the RPC supports exactly one response message or
-      any arbitrary number of response messages.
-    request_deserializer: A callable behavior that accepts a byte string and
-      returns an object suitable to be passed to this object's business logic,
-      or None to indicate that this object's business logic should be passed the
-      raw request bytes.
-    response_serializer: A callable behavior that accepts an object produced by
-      this object's business logic and returns a byte string, or None to
-      indicate that the byte strings produced by this object's business logic
-      should be transmitted on the wire as they are.
-    unary_unary: This object's application-specific business logic as a callable
-      value that takes a request value and a ServicerContext object and returns
-      a response value. Only non-None if both request_streaming and
-      response_streaming are False.
-    unary_stream: This object's application-specific business logic as a
-      callable value that takes a request value and a ServicerContext object and
-      returns an iterator of response values. Only non-None if request_streaming
-      is False and response_streaming is True.
-    stream_unary: This object's application-specific business logic as a
-      callable value that takes an iterator of request values and a
-      ServicerContext object and returns a response value. Only non-None if
-      request_streaming is True and response_streaming is False.
-    stream_stream: This object's application-specific business logic as a
-      callable value that takes an iterator of request values and a
-      ServicerContext object and returns an iterator of response values. Only
-      non-None if request_streaming and response_streaming are both True.
-  """
+    Attributes:
+      request_streaming: Whether the RPC supports exactly one request message
+        or any arbitrary number of request messages.
+      response_streaming: Whether the RPC supports exactly one response message
+        or any arbitrary number of response messages.
+      request_deserializer: A callable behavior that accepts a byte string and
+        returns an object suitable to be passed to this object's business
+        logic, or None to indicate that this object's business logic should be
+        passed the raw request bytes.
+      response_serializer: A callable behavior that accepts an object produced
+        by this object's business logic and returns a byte string, or None to
+        indicate that the byte strings produced by this object's business logic
+        should be transmitted on the wire as they are.
+      unary_unary: This object's application-specific business logic as a
+        callable value that takes a request value and a ServicerContext object
+        and returns a response value. Only non-None if both request_streaming
+        and response_streaming are False.
+      unary_stream: This object's application-specific business logic as a
+        callable value that takes a request value and a ServicerContext object
+        and returns an iterator of response values. Only non-None if
+        request_streaming is False and response_streaming is True.
+      stream_unary: This object's application-specific business logic as a
+        callable value that takes an iterator of request values and a
+        ServicerContext object and returns a response value. Only non-None if
+        request_streaming is True and response_streaming is False.
+      stream_stream: This object's application-specific business logic as a
+        callable value that takes an iterator of request values and a
+        ServicerContext object and returns an iterator of response values.
+        Only non-None if request_streaming and response_streaming are both
+        True.
+    """
 
 
 class HandlerCallDetails(six.with_metaclass(abc.ABCMeta)):
     """Describes an RPC that has just arrived for service.
-  Attributes:
-    method: The method name of the RPC.
-    invocation_metadata: The :term:`metadata` sent by the client.
-  """
+    Attributes:
+      method: The method name of the RPC.
+      invocation_metadata: The :term:`metadata` sent by the client.
+    """
 
 
 class GenericRpcHandler(six.with_metaclass(abc.ABCMeta)):
@@ -1096,33 +1110,33 @@ class GenericRpcHandler(six.with_metaclass(abc.ABCMeta)):
     def service(self, handler_call_details):
         """Returns the handler for servicing the RPC.
 
-    Args:
-      handler_call_details: A HandlerCallDetails describing the RPC.
+        Args:
+          handler_call_details: A HandlerCallDetails describing the RPC.
 
-    Returns:
-      An RpcMethodHandler with which the RPC may be serviced if the
-      implementation chooses to service this RPC, or None otherwise.
-    """
+        Returns:
+          An RpcMethodHandler with which the RPC may be serviced if the
+          implementation chooses to service this RPC, or None otherwise.
+        """
         raise NotImplementedError()
 
 
 class ServiceRpcHandler(six.with_metaclass(abc.ABCMeta, GenericRpcHandler)):
     """An implementation of RPC methods belonging to a service.
 
-  A service handles RPC methods with structured names of the form
-  '/Service.Name/Service.Method', where 'Service.Name' is the value
-  returned by service_name(), and 'Service.Method' is the method
-  name.  A service can have multiple method names, but only a single
-  service name.
-  """
+    A service handles RPC methods with structured names of the form
+    '/Service.Name/Service.Method', where 'Service.Name' is the value
+    returned by service_name(), and 'Service.Method' is the method
+    name.  A service can have multiple method names, but only a single
+    service name.
+    """
 
     @abc.abstractmethod
     def service_name(self):
         """Returns this service's name.
 
-    Returns:
-      The service name.
-    """
+        Returns:
+          The service name.
+        """
         raise NotImplementedError()
 
 
@@ -1164,83 +1178,84 @@ class Server(six.with_metaclass(abc.ABCMeta)):
     def add_generic_rpc_handlers(self, generic_rpc_handlers):
         """Registers GenericRpcHandlers with this Server.
 
-    This method is only safe to call before the server is started.
+        This method is only safe to call before the server is started.
 
-    Args:
-      generic_rpc_handlers: An iterable of GenericRpcHandlers that will be used
-      to service RPCs.
-    """
+        Args:
+          generic_rpc_handlers: An iterable of GenericRpcHandlers that will be
+          used to service RPCs.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def add_insecure_port(self, address):
         """Opens an insecure port for accepting RPCs.
 
-    This method may only be called before starting the server.
+        This method may only be called before starting the server.
 
-    Args:
-      address: The address for which to open a port.
-      if the port is 0, or not specified in the address, then gRPC runtime
-      will choose a port.
+        Args:
+          address: The address for which to open a port.
+          if the port is 0, or not specified in the address, then gRPC runtime
+          will choose a port.
 
-    Returns:
-      integer:
-      An integer port on which server will accept RPC requests.
-    """
+        Returns:
+          integer:
+          An integer port on which server will accept RPC requests.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def add_secure_port(self, address, server_credentials):
         """Opens a secure port for accepting RPCs.
 
-    This method may only be called before starting the server.
+        This method may only be called before starting the server.
 
-    Args:
-      address: The address for which to open a port.
-        if the port is 0, or not specified in the address, then gRPC runtime
-        will choose a port.
-      server_credentials: A ServerCredentials object.
+        Args:
+          address: The address for which to open a port.
+            if the port is 0, or not specified in the address, then gRPC
+            runtime will choose a port.
+          server_credentials: A ServerCredentials object.
 
-    Returns:
-      integer:
-      An integer port on which server will accept RPC requests.
-    """
+        Returns:
+          integer:
+          An integer port on which server will accept RPC requests.
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def start(self):
         """Starts this Server.
 
-    This method may only be called once. (i.e. it is not idempotent).
-    """
+        This method may only be called once. (i.e. it is not idempotent).
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
     def stop(self, grace):
         """Stops this Server.
 
-    This method immediately stop service of new RPCs in all cases.
-    If a grace period is specified, this method returns immediately
-    and all RPCs active at the end of the grace period are aborted.
+        This method immediately stop service of new RPCs in all cases.
+        If a grace period is specified, this method returns immediately
+        and all RPCs active at the end of the grace period are aborted.
 
-    If a grace period is not specified, then all existing RPCs are
-    teriminated immediately and the this method blocks until the last
-    RPC handler terminates.
+        If a grace period is not specified, then all existing RPCs are
+        teriminated immediately and the this method blocks until the last
+        RPC handler terminates.
 
-    This method is idempotent and may be called at any time. Passing a smaller
-    grace value in subsequentcall will have the effect of stopping the Server
-    sooner. Passing a larger grace value in subsequent call *will not* have the
-    effect of stopping the server later (i.e. the most restrictive grace
-    value is used).
+        This method is idempotent and may be called at any time.
+        Passing a smaller grace value in subsequent call will have
+        the effect of stopping the Server sooner. Passing a larger
+        grace value in subsequent call *will not* have the effect of
+        stopping the server later (i.e. the most restrictive grace
+        value is used).
 
-    Args:
-      grace: A duration of time in seconds or None.
+        Args:
+          grace: A duration of time in seconds or None.
 
-    Returns:
-      A threading.Event that will be set when this Server has completely
-      stopped, i.e. when running RPCs either complete or are aborted and
-      all handlers have terminated.
-    """
+        Returns:
+          A threading.Event that will be set when this Server has completely
+          stopped, i.e. when running RPCs either complete or are aborted and
+          all handlers have terminated.
+        """
         raise NotImplementedError()
 
 
@@ -1252,15 +1267,15 @@ def unary_unary_rpc_method_handler(behavior,
                                    response_serializer=None):
     """Creates an RpcMethodHandler for a unary-unary RPC method.
 
-  Args:
-    behavior: The implementation of an RPC that accepts one request and returns
-    one response.
-    request_deserializer: An optional behavior for request deserialization.
-    response_serializer: An optional behavior for response serialization.
+    Args:
+      behavior: The implementation of an RPC that accepts one request
+        and returns one response.
+      request_deserializer: An optional behavior for request deserialization.
+      response_serializer: An optional behavior for response serialization.
 
-  Returns:
-    An RpcMethodHandler object that is typically used by grpc.Server.
-  """
+    Returns:
+      An RpcMethodHandler object that is typically used by grpc.Server.
+    """
     from grpc import _utilities  # pylint: disable=cyclic-import
     return _utilities.RpcMethodHandler(False, False, request_deserializer,
                                        response_serializer, behavior, None,
@@ -1272,15 +1287,15 @@ def unary_stream_rpc_method_handler(behavior,
                                     response_serializer=None):
     """Creates an RpcMethodHandler for a unary-stream RPC method.
 
-  Args:
-    behavior: The implementation of an RPC that accepts one request and returns
-      an iterator of response values.
-    request_deserializer: An optional behavior for request deserialization.
-    response_serializer: An optional behavior for response serialization.
+    Args:
+      behavior: The implementation of an RPC that accepts one request
+        and returns an iterator of response values.
+      request_deserializer: An optional behavior for request deserialization.
+      response_serializer: An optional behavior for response serialization.
 
-  Returns:
-    An RpcMethodHandler object that is typically used by grpc.Server.
-  """
+    Returns:
+      An RpcMethodHandler object that is typically used by grpc.Server.
+    """
     from grpc import _utilities  # pylint: disable=cyclic-import
     return _utilities.RpcMethodHandler(False, True, request_deserializer,
                                        response_serializer, None, behavior,
@@ -1292,15 +1307,15 @@ def stream_unary_rpc_method_handler(behavior,
                                     response_serializer=None):
     """Creates an RpcMethodHandler for a stream-unary RPC method.
 
-  Args:
-    behavior: The implementation of an RPC that accepts an iterator of request
-    values and returns a single response value.
-    request_deserializer: An optional behavior for request deserialization.
-    response_serializer: An optional behavior for response serialization.
+    Args:
+      behavior: The implementation of an RPC that accepts an iterator of
+        request values and returns a single response value.
+      request_deserializer: An optional behavior for request deserialization.
+      response_serializer: An optional behavior for response serialization.
 
-  Returns:
-    An RpcMethodHandler object that is typically used by grpc.Server.
-  """
+    Returns:
+      An RpcMethodHandler object that is typically used by grpc.Server.
+    """
     from grpc import _utilities  # pylint: disable=cyclic-import
     return _utilities.RpcMethodHandler(True, False, request_deserializer,
                                        response_serializer, None, None,
@@ -1312,15 +1327,15 @@ def stream_stream_rpc_method_handler(behavior,
                                      response_serializer=None):
     """Creates an RpcMethodHandler for a stream-stream RPC method.
 
-  Args:
-    behavior: The implementation of an RPC that accepts an iterator of request
-    values and returns an iterator of response values.
-    request_deserializer: An optional behavior for request deserialization.
-    response_serializer: An optional behavior for response serialization.
+    Args:
+      behavior: The implementation of an RPC that accepts an iterator of
+        request values and returns an iterator of response values.
+      request_deserializer: An optional behavior for request deserialization.
+      response_serializer: An optional behavior for response serialization.
 
-  Returns:
-    An RpcMethodHandler object that is typically used by grpc.Server.
-  """
+    Returns:
+      An RpcMethodHandler object that is typically used by grpc.Server.
+    """
     from grpc import _utilities  # pylint: disable=cyclic-import
     return _utilities.RpcMethodHandler(True, True, request_deserializer,
                                        response_serializer, None, None, None,
@@ -1330,15 +1345,16 @@ def stream_stream_rpc_method_handler(behavior,
 def method_handlers_generic_handler(service, method_handlers):
     """Creates a GenericRpcHandler from RpcMethodHandlers.
 
-  Args:
-    service: The name of the service that is implemented by the method_handlers.
-    method_handlers: A dictionary that maps method names to corresponding
-    RpcMethodHandler.
+    Args:
+      service: The name of the service that is implemented by the
+        method_handlers.
+      method_handlers: A dictionary that maps method names to corresponding
+        RpcMethodHandler.
 
-  Returns:
-    A GenericRpcHandler. This is typically added to the grpc.Server object
-    with add_generic_rpc_handlers() before starting the server.
-  """
+    Returns:
+      A GenericRpcHandler. This is typically added to the grpc.Server object
+      with add_generic_rpc_handlers() before starting the server.
+    """
     from grpc import _utilities  # pylint: disable=cyclic-import
     return _utilities.DictionaryGenericHandler(service, method_handlers)
 
@@ -1435,20 +1451,20 @@ def ssl_server_credentials(private_key_certificate_chain_pairs,
                            require_client_auth=False):
     """Creates a ServerCredentials for use with an SSL-enabled Server.
 
-  Args:
-    private_key_certificate_chain_pairs: A list of pairs of the form
-      [PEM-encoded private key, PEM-encoded certificate chain].
-    root_certificates: An optional byte string of PEM-encoded client root
-      certificates that the server will use to verify client authentication.
-      If omitted, require_client_auth must also be False.
-    require_client_auth: A boolean indicating whether or not to require
-      clients to be authenticated. May only be True if root_certificates
-      is not None.
-
-  Returns:
-    A ServerCredentials for use with an SSL-enabled Server. Typically, this
-    object is an argument to add_secure_port() method during server setup.
-  """
+    Args:
+      private_key_certificate_chain_pairs: A list of pairs of the form
+        [PEM-encoded private key, PEM-encoded certificate chain].
+      root_certificates: An optional byte string of PEM-encoded client root
+        certificates that the server will use to verify client authentication.
+        If omitted, require_client_auth must also be False.
+      require_client_auth: A boolean indicating whether or not to require
+        clients to be authenticated. May only be True if root_certificates
+        is not None.
+
+    Returns:
+      A ServerCredentials for use with an SSL-enabled Server. Typically, this
+      object is an argument to add_secure_port() method during server setup.
+    """
     if len(private_key_certificate_chain_pairs) == 0:
         raise ValueError(
             'At least one private key-certificate chain pair is required!')
@@ -1522,16 +1538,16 @@ def dynamic_ssl_server_credentials(initial_certificate_configuration,
 def channel_ready_future(channel):
     """Creates a Future that tracks when a Channel is ready.
 
-  Cancelling the Future does not affect the channel's state machine.
-  It merely decouples the Future from channel state machine.
+    Cancelling the Future does not affect the channel's state machine.
+    It merely decouples the Future from channel state machine.
 
-  Args:
-    channel: A Channel object.
+    Args:
+      channel: A Channel object.
 
-  Returns:
-    A Future object that matures when the channel connectivity is
-    ChannelConnectivity.READY.
-  """
+    Returns:
+      A Future object that matures when the channel connectivity is
+      ChannelConnectivity.READY.
+    """
     from grpc import _utilities  # pylint: disable=cyclic-import
     return _utilities.channel_ready_future(channel)
 
@@ -1539,14 +1555,14 @@ def channel_ready_future(channel):
 def insecure_channel(target, options=None):
     """Creates an insecure Channel to a server.
 
-  Args:
-    target: The server address
-    options: An optional list of key-value pairs (channel args in gRPC runtime)
-    to configure the channel.
+    Args:
+      target: The server address
+      options: An optional list of key-value pairs (channel args
+        in gRPC Core runtime) to configure the channel.
 
-  Returns:
-    A Channel object.
-  """
+    Returns:
+      A Channel object.
+    """
     from grpc import _channel  # pylint: disable=cyclic-import
     return _channel.Channel(target, () if options is None else options, None)
 
@@ -1554,15 +1570,15 @@ def insecure_channel(target, options=None):
 def secure_channel(target, credentials, options=None):
     """Creates a secure Channel to a server.
 
-  Args:
-    target: The server address.
-    credentials: A ChannelCredentials instance.
-    options: An optional list of key-value pairs (channel args in gRPC runtime)
-    to configure the channel.
+    Args:
+      target: The server address.
+      credentials: A ChannelCredentials instance.
+      options: An optional list of key-value pairs (channel args
+        in gRPC Core runtime) to configure the channel.
 
-  Returns:
-    A Channel object.
-  """
+    Returns:
+      A Channel object.
+    """
     from grpc import _channel  # pylint: disable=cyclic-import
     return _channel.Channel(target, () if options is None else options,
                             credentials._credentials)

From 0f5f46109b995aab3bb5f8ffeadd96547de7e100 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Thu, 18 Jan 2018 09:26:32 +0100
Subject: [PATCH 139/161] use grpc_timeout_milliseconds_to_deadline directly

---
 test/core/end2end/dualstack_socket_test.cc | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/test/core/end2end/dualstack_socket_test.cc b/test/core/end2end/dualstack_socket_test.cc
index 0c101d0f858..04c727e689a 100644
--- a/test/core/end2end/dualstack_socket_test.cc
+++ b/test/core/end2end/dualstack_socket_test.cc
@@ -43,14 +43,11 @@
 
 static void* tag(intptr_t i) { return (void*)i; }
 
-static gpr_timespec ms_from_now(int ms) {
-  return grpc_timeout_milliseconds_to_deadline(ms);
-}
-
 static void drain_cq(grpc_completion_queue* cq) {
   grpc_event ev;
   do {
-    ev = grpc_completion_queue_next(cq, ms_from_now(5000), nullptr);
+    ev = grpc_completion_queue_next(
+        cq, grpc_timeout_milliseconds_to_deadline(5000), nullptr);
   } while (ev.type != GRPC_QUEUE_SHUTDOWN);
 }
 
@@ -165,11 +162,11 @@ void test_connect(const char* server_host, const char* client_host, int port,
 
   if (expect_ok) {
     /* Normal deadline, shouldn't be reached. */
-    deadline = ms_from_now(60000);
+    deadline = grpc_timeout_milliseconds_to_deadline(60000);
   } else {
     /* Give up faster when failure is expected.
        BUG: Setting this to 1000 reveals a memory leak (b/18608927). */
-    deadline = ms_from_now(3000);
+    deadline = grpc_timeout_milliseconds_to_deadline(3000);
   }
 
   /* Send a trivial request. */

From 631ccc2c798a668cc3bc396dcd849cbac37b833c Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Thu, 18 Jan 2018 10:10:39 +0100
Subject: [PATCH 140/161] distribtests_standalone: fixing nits

---
 tools/dockerfile/push_testing_images.sh                    | 2 +-
 tools/internal_ci/windows/grpc_distribtests_standalone.bat | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/dockerfile/push_testing_images.sh b/tools/dockerfile/push_testing_images.sh
index e9151303f75..b76ceea8f6f 100755
--- a/tools/dockerfile/push_testing_images.sh
+++ b/tools/dockerfile/push_testing_images.sh
@@ -29,7 +29,7 @@ cd -
 
 DOCKERHUB_ORGANIZATION=grpctesting
 
-for DOCKERFILE_DIR in tools/dockerfile/test/* tools/dockerfile/grpc_artifact_* tools/dockerfile/interoptest/* third_party/rake-compiler-dock
+for DOCKERFILE_DIR in tools/dockerfile/test/* tools/dockerfile/grpc_artifact_* tools/dockerfile/interoptest/* tools/dockerfile/distribtest/cpp_jessie_x64 third_party/rake-compiler-dock
 do
   # Generate image name based on Dockerfile checksum. That works well as long
   # as can count on dockerfiles being written in a way that changing the logical 
diff --git a/tools/internal_ci/windows/grpc_distribtests_standalone.bat b/tools/internal_ci/windows/grpc_distribtests_standalone.bat
index 229840bc982..3eb33b1548c 100644
--- a/tools/internal_ci/windows/grpc_distribtests_standalone.bat
+++ b/tools/internal_ci/windows/grpc_distribtests_standalone.bat
@@ -24,7 +24,7 @@ cd /d %~dp0\..\..\..
 
 call tools/internal_ci/helper_scripts/prepare_build_windows.bat
 
-python tools/run_tests/task_runner.py -f distribtests windows cpp -j 4 || goto :error
+python tools/run_tests/task_runner.py -f distribtest windows cpp -j 4 || goto :error
 goto :EOF
 
 :error

From 41723fa68f12cbe344f3bff9d3a6bad1745ba488 Mon Sep 17 00:00:00 2001
From: Carlos O'Ryan <coryan@users.noreply.github.com>
Date: Thu, 18 Jan 2018 10:21:27 -0500
Subject: [PATCH 141/161] Fixed typo in documentation.

s/opitons/options/
---
 src/core/lib/iomgr/socket_mutator.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/core/lib/iomgr/socket_mutator.h b/src/core/lib/iomgr/socket_mutator.h
index 0a97cf657f0..f8fd21d15a3 100644
--- a/src/core/lib/iomgr/socket_mutator.h
+++ b/src/core/lib/iomgr/socket_mutator.h
@@ -26,7 +26,7 @@
 
 /** The virtual table of grpc_socket_mutator */
 typedef struct {
-  /** Mutates the socket opitons of \a fd */
+  /** Mutates the socket options of \a fd */
   bool (*mutate_fd)(int fd, grpc_socket_mutator* mutator);
   /** Compare socket mutator \a a and \a b */
   int (*compare)(grpc_socket_mutator* a, grpc_socket_mutator* b);

From b5089297dbe528c773233ed1a7bb429e880f4d5a Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Thu, 18 Jan 2018 16:21:08 +0100
Subject: [PATCH 142/161] fix "not enough actual parameters for macro" warning

---
 .../filters/client_channel/lb_policy/round_robin/round_robin.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
index dca345566ad..5a36acaa572 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
@@ -484,7 +484,7 @@ static void rr_connectivity_changed_locked(void* arg, grpc_error* error) {
       break;
     }
     case GRPC_CHANNEL_SHUTDOWN:
-      GPR_UNREACHABLE_CODE();
+      GPR_UNREACHABLE_CODE(return );
     case GRPC_CHANNEL_CONNECTING:
     case GRPC_CHANNEL_IDLE:;  // fallthrough
   }

From 44fe6e282c848d876d516fe6cd79a008d8104604 Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Thu, 18 Jan 2018 08:54:45 -0800
Subject: [PATCH 143/161] Combine BackOff Begin() and Step() methods.

---
 .../client_channel/lb_policy/grpclb/grpclb.cc |  2 +-
 .../resolver/dns/c_ares/dns_resolver_ares.cc  |  2 +-
 .../resolver/dns/native/dns_resolver.cc       |  2 +-
 .../ext/filters/client_channel/subchannel.cc  |  3 +-
 src/core/lib/backoff/backoff.cc               | 23 ++++++++-----
 src/core/lib/backoff/backoff.h                | 16 ++++-----
 test/core/backoff/backoff_test.cc             | 34 +++++++++----------
 7 files changed, 42 insertions(+), 40 deletions(-)

diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index 272b3617b25..97dd93144e9 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -1158,7 +1158,7 @@ static void maybe_restart_lb_call(glb_lb_policy* glb_policy) {
     glb_policy->updating_lb_call = false;
   } else if (!glb_policy->shutting_down) {
     /* if we aren't shutting down, restart the LB client call after some time */
-    grpc_millis next_try = glb_policy->lb_call_backoff->Step();
+    grpc_millis next_try = glb_policy->lb_call_backoff->NextAttemptTime();
     if (grpc_lb_glb_trace.enabled()) {
       gpr_log(GPR_DEBUG, "[grpclb %p] Connection to LB server lost...",
               glb_policy);
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
index 4659a5f3ede..002c6f3f771 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
@@ -264,7 +264,7 @@ static void dns_ares_on_resolved_locked(void* arg, grpc_error* error) {
   } else {
     const char* msg = grpc_error_string(error);
     gpr_log(GPR_DEBUG, "dns resolution failed: %s", msg);
-    grpc_millis next_try = r->backoff->Step();
+    grpc_millis next_try = r->backoff->NextAttemptTime();
     grpc_millis timeout = next_try - grpc_core::ExecCtx::Get()->Now();
     gpr_log(GPR_INFO, "dns resolution failed (will retry): %s",
             grpc_error_string(error));
diff --git a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
index 1c2cfc08e7b..bfbec3ac59f 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
+++ b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
@@ -161,7 +161,7 @@ static void dns_on_resolved_locked(void* arg, grpc_error* error) {
     grpc_resolved_addresses_destroy(r->addresses);
     grpc_lb_addresses_destroy(addresses);
   } else {
-    grpc_millis next_try = r->backoff->Step();
+    grpc_millis next_try = r->backoff->NextAttemptTime();
     grpc_millis timeout = next_try - grpc_core::ExecCtx::Get()->Now();
     gpr_log(GPR_INFO, "dns resolution failed (will retry): %s",
             grpc_error_string(error));
diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc
index a604c55c58c..c1c69168faf 100644
--- a/src/core/ext/filters/client_channel/subchannel.cc
+++ b/src/core/ext/filters/client_channel/subchannel.cc
@@ -392,6 +392,7 @@ static void continue_connect_locked(grpc_subchannel* c) {
   args.interested_parties = c->pollset_set;
   const grpc_millis min_deadline =
       c->min_connect_timeout_ms + grpc_core::ExecCtx::Get()->Now();
+  c->next_attempt_deadline = c->backoff->NextAttemptTime();
   args.deadline = std::max(c->next_attempt_deadline, min_deadline);
   args.channel_args = c->args;
   grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_CONNECTING,
@@ -437,7 +438,6 @@ static void on_alarm(void* arg, grpc_error* error) {
   }
   if (error == GRPC_ERROR_NONE) {
     gpr_log(GPR_INFO, "Failed to connect to channel, retrying");
-    c->next_attempt_deadline = c->backoff->Step();
     continue_connect_locked(c);
     gpr_mu_unlock(&c->mu);
   } else {
@@ -473,7 +473,6 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) {
 
   if (!c->backoff_begun) {
     c->backoff_begun = true;
-    c->next_attempt_deadline = c->backoff->Begin();
     continue_connect_locked(c);
   } else {
     GPR_ASSERT(!c->have_alarm);
diff --git a/src/core/lib/backoff/backoff.cc b/src/core/lib/backoff/backoff.cc
index 41f625a636c..d561fc7460d 100644
--- a/src/core/lib/backoff/backoff.cc
+++ b/src/core/lib/backoff/backoff.cc
@@ -41,18 +41,20 @@ double generate_uniform_random_number_between(uint32_t* rng_state, double a,
   const double range = b - a;
   return a + generate_uniform_random_number(rng_state) * range;
 }
-}  // namespace
 
-BackOff::BackOff(const Options& options) : options_(options) {
-  rng_state_ = static_cast<uint32_t>(gpr_now(GPR_CLOCK_REALTIME).tv_nsec);
-}
+}  // namespace
 
-grpc_millis BackOff::Begin() {
-  current_backoff_ = options_.initial_backoff();
-  return current_backoff_ + grpc_core::ExecCtx::Get()->Now();
+BackOff::BackOff(const Options& options)
+    : options_(options),
+      rng_state_(static_cast<uint32_t>(gpr_now(GPR_CLOCK_REALTIME).tv_nsec)) {
+  Reset();
 }
 
-grpc_millis BackOff::Step() {
+grpc_millis BackOff::NextAttemptTime() {
+  if (initial_) {
+    initial_ = false;
+    return current_backoff_ + grpc_core::ExecCtx::Get()->Now();
+  }
   current_backoff_ =
       (grpc_millis)(std::min(current_backoff_ * options_.multiplier(),
                              (double)options_.max_backoff()));
@@ -63,7 +65,10 @@ grpc_millis BackOff::Step() {
   return next_timeout + grpc_core::ExecCtx::Get()->Now();
 }
 
-void BackOff::Reset() { current_backoff_ = options_.initial_backoff(); }
+void BackOff::Reset() {
+  current_backoff_ = options_.initial_backoff();
+  initial_ = true;
+}
 
 void BackOff::SetRandomSeed(uint32_t seed) { rng_state_ = seed; }
 
diff --git a/src/core/lib/backoff/backoff.h b/src/core/lib/backoff/backoff.h
index 84ef9b82e41..de30e5268b3 100644
--- a/src/core/lib/backoff/backoff.h
+++ b/src/core/lib/backoff/backoff.h
@@ -32,14 +32,11 @@ class BackOff {
   /// Initialize backoff machinery - does not need to be destroyed
   explicit BackOff(const Options& options);
 
-  /// Begin retry loop: returns the deadline to be used for the next attempt,
-  /// following the backoff strategy.
-  grpc_millis Begin();
-  /// Step a retry loop: returns the deadline to be used for the next attempt,
-  /// following the backoff strategy.
-  grpc_millis Step();
-  /// Reset the backoff, so the next grpc_backoff_step will be a
-  /// grpc_backoff_begin.
+  /// Returns the time at which the next attempt should start.
+  grpc_millis NextAttemptTime();
+
+  /// Reset the backoff, so the next value returned by NextAttemptTime()
+  /// will be the time of the second attempt (rather than the Nth).
   void Reset();
 
   void SetRandomSeed(unsigned int seed);
@@ -80,9 +77,10 @@ class BackOff {
 
  private:
   const Options options_;
+  uint32_t rng_state_;
+  bool initial_;
   /// current delay before retries
   grpc_millis current_backoff_;
-  uint32_t rng_state_;
 };
 
 }  // namespace grpc_core
diff --git a/test/core/backoff/backoff_test.cc b/test/core/backoff/backoff_test.cc
index 7bc4d14ce67..2e61243831a 100644
--- a/test/core/backoff/backoff_test.cc
+++ b/test/core/backoff/backoff_test.cc
@@ -45,11 +45,11 @@ TEST(BackOffTest, ConstantBackOff) {
       .set_max_backoff(max_backoff);
   BackOff backoff(options);
 
-  grpc_millis next_attempt_start_time = backoff.Begin();
+  grpc_millis next_attempt_start_time = backoff.NextAttemptTime();
   EXPECT_EQ(next_attempt_start_time - grpc_core::ExecCtx::Get()->Now(),
             initial_backoff);
   for (int i = 0; i < 10000; i++) {
-    next_attempt_start_time = backoff.Step();
+    next_attempt_start_time = backoff.NextAttemptTime();
     EXPECT_EQ(next_attempt_start_time - grpc_core::ExecCtx::Get()->Now(),
               initial_backoff);
   }
@@ -67,7 +67,7 @@ TEST(BackOffTest, MinConnect) {
       .set_jitter(jitter)
       .set_max_backoff(max_backoff);
   BackOff backoff(options);
-  grpc_millis next = backoff.Begin();
+  grpc_millis next = backoff.NextAttemptTime();
   EXPECT_EQ(next - grpc_core::ExecCtx::Get()->Now(), initial_backoff);
 }
 
@@ -86,42 +86,42 @@ TEST(BackOffTest, NoJitterBackOff) {
   // x_n = 2**i + x_{i-1} ( = 2**(n+1) - 2 )
   grpc_core::ExecCtx exec_ctx;
   grpc_core::ExecCtx::Get()->TestOnlySetNow(0);
-  grpc_millis next = backoff.Begin();
+  grpc_millis next = backoff.NextAttemptTime();
   EXPECT_EQ(next, 2);
   grpc_core::ExecCtx::Get()->TestOnlySetNow(next);
-  next = backoff.Step();
+  next = backoff.NextAttemptTime();
   EXPECT_EQ(next, 6);
   grpc_core::ExecCtx::Get()->TestOnlySetNow(next);
-  next = backoff.Step();
+  next = backoff.NextAttemptTime();
   EXPECT_EQ(next, 14);
   grpc_core::ExecCtx::Get()->TestOnlySetNow(next);
-  next = backoff.Step();
+  next = backoff.NextAttemptTime();
   EXPECT_EQ(next, 30);
   grpc_core::ExecCtx::Get()->TestOnlySetNow(next);
-  next = backoff.Step();
+  next = backoff.NextAttemptTime();
   EXPECT_EQ(next, 62);
   grpc_core::ExecCtx::Get()->TestOnlySetNow(next);
-  next = backoff.Step();
+  next = backoff.NextAttemptTime();
   EXPECT_EQ(next, 126);
   grpc_core::ExecCtx::Get()->TestOnlySetNow(next);
-  next = backoff.Step();
+  next = backoff.NextAttemptTime();
   EXPECT_EQ(next, 254);
   grpc_core::ExecCtx::Get()->TestOnlySetNow(next);
-  next = backoff.Step();
+  next = backoff.NextAttemptTime();
   EXPECT_EQ(next, 510);
   grpc_core::ExecCtx::Get()->TestOnlySetNow(next);
-  next = backoff.Step();
+  next = backoff.NextAttemptTime();
   EXPECT_EQ(next, 1022);
   grpc_core::ExecCtx::Get()->TestOnlySetNow(next);
-  next = backoff.Step();
+  next = backoff.NextAttemptTime();
   // Hit the maximum timeout. From this point onwards, retries will increase
   // only by max timeout.
   EXPECT_EQ(next, 1535);
   grpc_core::ExecCtx::Get()->TestOnlySetNow(next);
-  next = backoff.Step();
+  next = backoff.NextAttemptTime();
   EXPECT_EQ(next, 2048);
   grpc_core::ExecCtx::Get()->TestOnlySetNow(next);
-  next = backoff.Step();
+  next = backoff.NextAttemptTime();
   EXPECT_EQ(next, 2561);
 }
 
@@ -141,7 +141,7 @@ TEST(BackOffTest, JitterBackOff) {
   backoff.SetRandomSeed(0);  // force consistent PRNG
 
   grpc_core::ExecCtx exec_ctx;
-  grpc_millis next = backoff.Begin();
+  grpc_millis next = backoff.NextAttemptTime();
   EXPECT_EQ(next - grpc_core::ExecCtx::Get()->Now(), initial_backoff);
 
   grpc_millis expected_next_lower_bound =
@@ -150,7 +150,7 @@ TEST(BackOffTest, JitterBackOff) {
       (grpc_millis)((double)current_backoff * (1 + jitter));
 
   for (int i = 0; i < 10000; i++) {
-    next = backoff.Step();
+    next = backoff.NextAttemptTime();
     // next-now must be within (jitter*100)% of the current backoff (which
     // increases by * multiplier up to max_backoff).
     const grpc_millis timeout_millis = next - grpc_core::ExecCtx::Get()->Now();

From 8725b77e0b3406481bd6c8e6a7916896bc30652d Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Thu, 18 Jan 2018 09:47:44 -0800
Subject: [PATCH 144/161] Make run_in_workspace.sh pass shellcheck (with
 suppressions)

---
 tools/run_tests/artifacts/run_in_workspace.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tools/run_tests/artifacts/run_in_workspace.sh b/tools/run_tests/artifacts/run_in_workspace.sh
index 5b8af6ab534..20181e077c2 100755
--- a/tools/run_tests/artifacts/run_in_workspace.sh
+++ b/tools/run_tests/artifacts/run_in_workspace.sh
@@ -18,15 +18,19 @@
 # All cmdline args will be executed as a command.
 set -ex
 
-cd $(dirname $0)/../../..
+cd "$(dirname "$0")/../../.."
 export repo_root=$(pwd)
 
+# TODO: fix file to pass shellcheck
+
 rm -rf "${WORKSPACE_NAME}"
 git clone . "${WORKSPACE_NAME}"
 # clone gRPC submodules, use data from locally cloned submodules where possible
+# shellcheck disable=SC1004,SC2016
 git submodule foreach 'cd "${repo_root}/${WORKSPACE_NAME}" \
     && git submodule update --init --reference ${repo_root}/${name} ${name}'
 
 echo "Running in workspace ${WORKSPACE_NAME}"
-cd ${WORKSPACE_NAME}
+cd "${WORKSPACE_NAME}"
+# shellcheck disable=SC2068
 $@

From adf6c95ab7edea73b7c5ae921678026c27526458 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Thu, 18 Jan 2018 09:57:15 -0800
Subject: [PATCH 145/161] Make build_artifact_ruby.sh pass shellcheck (with
 suppressions)

---
 tools/run_tests/artifacts/build_artifact_ruby.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tools/run_tests/artifacts/build_artifact_ruby.sh b/tools/run_tests/artifacts/build_artifact_ruby.sh
index 9165a224840..5ab4cf21b4e 100755
--- a/tools/run_tests/artifacts/build_artifact_ruby.sh
+++ b/tools/run_tests/artifacts/build_artifact_ruby.sh
@@ -14,9 +14,9 @@
 # limitations under the License.
 set -ex
 
-SYSTEM=`uname | cut -f 1 -d_`
+SYSTEM=$(uname | cut -f 1 -d_)
 
-cd $(dirname $0)/../../..
+cd "$(dirname "$0")/../../.."
 set +ex
 [[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
 [[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm"
@@ -46,7 +46,9 @@ export DOCKERHUB_ORGANIZATION=grpctesting
 rake gem:native
 
 if [ "$SYSTEM" == "Darwin" ] ; then
-  rm `ls pkg/*.gem | grep -v darwin`
+  # TODO: consider rewriting this to pass shellcheck
+  # shellcheck disable=SC2046,SC2010
+  rm $(ls pkg/*.gem | grep -v darwin)
 fi
 
 mkdir -p "${ARTIFACTS_OUT}"

From b24cff136466f338839d013189f81486c59084ce Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Thu, 18 Jan 2018 10:06:12 -0800
Subject: [PATCH 146/161] Fix build_artifact_csharp.sh to pass shellcheck

---
 tools/run_tests/artifacts/build_artifact_csharp.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/run_tests/artifacts/build_artifact_csharp.sh b/tools/run_tests/artifacts/build_artifact_csharp.sh
index 0884a9e5d69..d65340261df 100755
--- a/tools/run_tests/artifacts/build_artifact_csharp.sh
+++ b/tools/run_tests/artifacts/build_artifact_csharp.sh
@@ -15,7 +15,7 @@
 
 set -ex
 
-cd $(dirname $0)/../../..
+cd "$(dirname "$0")/../../.."
 
 make grpc_csharp_ext
 

From 05668d49a6f9b28cdb6bca430c539e128a5cd578 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Thu, 18 Jan 2018 10:06:44 -0800
Subject: [PATCH 147/161] Enforce shellcheck on tools/run_tests/artifacts

---
 tools/run_tests/sanity/check_shellcheck.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/run_tests/sanity/check_shellcheck.sh b/tools/run_tests/sanity/check_shellcheck.sh
index f2cba1892b1..b30febab043 100755
--- a/tools/run_tests/sanity/check_shellcheck.sh
+++ b/tools/run_tests/sanity/check_shellcheck.sh
@@ -19,6 +19,7 @@ set -e
 ROOT="$(dirname "$0")/../../.."
 
 DIRS=(
+    'tools/run_tests/artifacts'
     'tools/run_tests/helper_scripts'
     'tools/run_tests/sanity'
 )

From 1c42023ae60abf0a913aa268ef1b8818b0675300 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Thu, 18 Jan 2018 17:31:21 +0100
Subject: [PATCH 148/161] fix distribtest typo

---
 test/distrib/cpp/run_distrib_test_cmake.bat | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/distrib/cpp/run_distrib_test_cmake.bat b/test/distrib/cpp/run_distrib_test_cmake.bat
index 047846b0f1a..c4b0b359a46 100644
--- a/test/distrib/cpp/run_distrib_test_cmake.bat
+++ b/test/distrib/cpp/run_distrib_test_cmake.bat
@@ -58,7 +58,7 @@ cmake -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% -DOPENSSL_ROOT_DIR=%OPENSSL_DIR% -DOP
 cmake --build . --config Release --target install || goto :error
 cd ../..
 
-# Build helloworld example using cmake
+@rem Build helloworld example using cmake
 cd examples/cpp/helloworld
 mkdir cmake
 cd cmake

From 1e9543deb9c4f8de31dd0dce4dcc429aaa53530d Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Thu, 18 Jan 2018 17:45:18 +0100
Subject: [PATCH 149/161] fix finding openssl

---
 test/distrib/cpp/run_distrib_test_cmake.bat | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/distrib/cpp/run_distrib_test_cmake.bat b/test/distrib/cpp/run_distrib_test_cmake.bat
index c4b0b359a46..f920768ae39 100644
--- a/test/distrib/cpp/run_distrib_test_cmake.bat
+++ b/test/distrib/cpp/run_distrib_test_cmake.bat
@@ -64,7 +64,7 @@ mkdir cmake
 cd cmake
 mkdir build
 cd build
-cmake -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% ../.. || goto :error
+cmake -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% -DOPENSSL_ROOT_DIR=%OPENSSL_DIR% -DOPENSSL_INCLUDE_DIR=%OPENSSL_DIR%/include ../.. || goto :error
 cmake --build . --config Release || goto :error
 cd ../../../../..
 

From dec076c4abe7bd5518abab7a8da4e624c49ec655 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Thu, 18 Jan 2018 18:26:54 +0100
Subject: [PATCH 150/161] fix protobuf usage in the example

---
 examples/cpp/helloworld/CMakeLists.txt | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/examples/cpp/helloworld/CMakeLists.txt b/examples/cpp/helloworld/CMakeLists.txt
index 49684a13b04..c3ce4d5ba6b 100644
--- a/examples/cpp/helloworld/CMakeLists.txt
+++ b/examples/cpp/helloworld/CMakeLists.txt
@@ -16,15 +16,22 @@ endif()
 find_package(Protobuf REQUIRED)
 message(STATUS "Using protobuf ${protobuf_VERSION}")
 
-if(Protobuf_FOUND)
-  # Protobuf_FOUND is set for package type "CONFIG"
-  set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)
-  set(_PROTOBUF_PROTOC protobuf::protoc)
-elseif(PROTOBUF_FOUND)
-  # PROTOBUF_FOUND is set for package type "MODULE"
-  set(_PROTOBUF_LIBPROTOBUF ${PROTOBUF_LIBRARIES})
-  set(_PROTOBUF_PROTOC ${PROTOBUF_PROTOC_EXECUTABLE})
-  include_directories(${PROTOBUF_INCLUDE_DIRS})
+# {Protobuf,PROTOBUF}_FOUND is defined based on find_package type ("MODULE" vs "CONFIG").
+# For "MODULE", the case has also changed between cmake 3.5 and 3.6.
+# We use the legacy uppercase version for *_LIBRARIES AND *_INCLUDE_DIRS variables
+# as newer cmake versions provide them too for backward compatibility.
+if(Protobuf_FOUND OR PROTOBUF_FOUND)
+  if(TARGET protobuf::libprotobuf)
+    set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)
+  else()
+    set(_PROTOBUF_LIBPROTOBUF ${PROTOBUF_LIBRARIES})
+    include_directories(${PROTOBUF_INCLUDE_DIRS})
+  endif()
+  if(TARGET protobuf::protoc)
+    set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
+  else()
+    set(_PROTOBUF_PROTOC ${PROTOBUF_PROTOC_EXECUTABLE})
+  endif()
 else()
   message(WARNING "Failed to locate libprotobuf and protoc!")
 endif()

From fb227b0968e668cf9a3b7b6e1e2f5445f29eafb0 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Thu, 18 Jan 2018 19:32:27 +0100
Subject: [PATCH 151/161] improve comments on find_package(Protobuf)

---
 cmake/protobuf.cmake | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cmake/protobuf.cmake b/cmake/protobuf.cmake
index fccabd455a8..cb799b52953 100644
--- a/cmake/protobuf.cmake
+++ b/cmake/protobuf.cmake
@@ -53,6 +53,11 @@ if("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "module")
   endif()
 elseif("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "package")
   find_package(Protobuf REQUIRED ${gRPC_PROTOBUF_PACKAGE_TYPE})
+
+  # {Protobuf,PROTOBUF}_FOUND is defined based on find_package type ("MODULE" vs "CONFIG").
+  # For "MODULE", the case has also changed between cmake 3.5 and 3.6.
+  # We use the legacy uppercase version for *_LIBRARIES AND *_INCLUDE_DIRS variables
+  # as newer cmake versions provide them too for backward compatibility.
   if(Protobuf_FOUND OR PROTOBUF_FOUND)
     if(TARGET protobuf::${_gRPC_PROTOBUF_LIBRARY_NAME})
       set(_gRPC_PROTOBUF_LIBRARIES protobuf::${_gRPC_PROTOBUF_LIBRARY_NAME})

From dbdf495f61d099521bab45cf93f6f7d8dfb5c06f Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Thu, 18 Jan 2018 11:21:12 -0800
Subject: [PATCH 152/161] Split lib/support into lib/gpr and lib/gpr++.

---
 BUILD                                         | 158 ++++++-----
 CMakeLists.txt                                | 204 ++++++-------
 Makefile                                      | 268 +++++++++---------
 build.yaml                                    | 200 ++++++-------
 config.m4                                     |  86 +++---
 config.w32                                    |  86 +++---
 gRPC-Core.podspec                             | 178 ++++++------
 grpc.gemspec                                  | 128 ++++-----
 grpc.gyp                                      |  84 +++---
 package.xml                                   | 128 ++++-----
 .../filters/client_channel/backup_poller.cc   |   4 +-
 .../filters/client_channel/client_channel.cc  |   2 +-
 .../client_channel/http_connect_handshaker.cc |   4 +-
 .../ext/filters/client_channel/http_proxy.cc  |   4 +-
 .../ext/filters/client_channel/lb_policy.h    |   2 +-
 .../client_channel/lb_policy/grpclb/grpclb.cc |   4 +-
 .../lb_policy/grpclb/grpclb_channel.cc        |   2 +-
 .../lb_policy/grpclb/grpclb_channel_secure.cc |   2 +-
 .../lb_policy/round_robin/round_robin.cc      |   2 +-
 .../lb_policy/subchannel_list.h               |   2 +-
 .../client_channel/lb_policy_registry.cc      |   2 +-
 .../filters/client_channel/parse_address.cc   |   2 +-
 .../resolver/dns/c_ares/dns_resolver_ares.cc  |   6 +-
 .../dns/c_ares/grpc_ares_ev_driver_posix.cc   |   2 +-
 .../resolver/dns/c_ares/grpc_ares_wrapper.cc  |   2 +-
 .../resolver/dns/native/dns_resolver.cc       |   6 +-
 .../resolver/fake/fake_resolver.cc            |   2 +-
 .../resolver/sockaddr/sockaddr_resolver.cc    |   2 +-
 .../ext/filters/client_channel/subchannel.cc  |   4 +-
 .../ext/filters/client_channel/subchannel.h   |   6 +-
 .../ext/filters/client_channel/uri_parser.cc  |   2 +-
 .../filters/http/client/http_client_filter.cc |   2 +-
 .../message_compress_filter.cc                |   2 +-
 .../message_size/message_size_filter.cc       |   2 +-
 .../transport/chttp2/transport/bin_decoder.cc |   2 +-
 .../chttp2/transport/chttp2_plugin.cc         |   2 +-
 .../chttp2/transport/chttp2_transport.cc      |   4 +-
 .../chttp2/transport/flow_control.cc          |   2 +-
 .../transport/chttp2/transport/flow_control.h |   4 +-
 .../transport/chttp2/transport/frame_data.cc  |   2 +-
 .../chttp2/transport/hpack_parser.cc          |   2 +-
 .../transport/chttp2/transport/hpack_table.cc |   2 +-
 .../ext/transport/chttp2/transport/internal.h |   2 +-
 .../cronet/transport/cronet_transport.cc      |   2 +-
 src/core/lib/channel/channel_args.cc          |   2 +-
 src/core/lib/channel/channel_stack.h          |   2 +-
 src/core/lib/channel/connected_channel.cc     |   2 +-
 src/core/lib/debug/stats.cc                   |   2 +-
 src/core/lib/debug/trace.cc                   |   2 +-
 src/core/lib/gpr++/README.md                  |  16 ++
 src/core/lib/{support => gpr++}/abstract.h    |   6 +-
 src/core/lib/{support => gpr++}/atomic.h      |  10 +-
 .../lib/{support => gpr++}/atomic_with_atm.h  |   6 +-
 .../lib/{support => gpr++}/atomic_with_std.h  |   6 +-
 .../lib/{support => gpr++}/debug_location.h   |   6 +-
 .../vector.h => gpr++/inlined_vector.h}       |   8 +-
 .../{support => gpr++}/manual_constructor.h   |   4 +-
 src/core/lib/{support => gpr++}/memory.h      |   6 +-
 src/core/lib/{support => gpr++}/orphanable.h  |  12 +-
 src/core/lib/{support => gpr++}/ref_counted.h |  12 +-
 .../lib/{support => gpr++}/ref_counted_ptr.h  |   8 +-
 src/core/lib/gpr/README.md                    |   8 +
 src/core/lib/{support => gpr}/alloc.cc        |   0
 src/core/lib/{support => gpr}/arena.cc        |   2 +-
 src/core/lib/{support => gpr}/arena.h         |   6 +-
 src/core/lib/{support => gpr}/atm.cc          |   0
 src/core/lib/{support => gpr}/avl.cc          |   0
 src/core/lib/{support => gpr}/cmdline.cc      |   2 +-
 src/core/lib/{support => gpr}/cpu_iphone.cc   |   0
 src/core/lib/{support => gpr}/cpu_linux.cc    |   0
 src/core/lib/{support => gpr}/cpu_posix.cc    |   0
 src/core/lib/{support => gpr}/cpu_windows.cc  |   0
 src/core/lib/{support => gpr}/env.h           |   6 +-
 src/core/lib/{support => gpr}/env_linux.cc    |   4 +-
 src/core/lib/{support => gpr}/env_posix.cc    |   4 +-
 src/core/lib/{support => gpr}/env_windows.cc  |   6 +-
 src/core/lib/{support => gpr}/fork.cc         |   4 +-
 src/core/lib/{support => gpr}/fork.h          |   6 +-
 src/core/lib/{support => gpr}/host_port.cc    |   2 +-
 src/core/lib/{support => gpr}/log.cc          |   4 +-
 src/core/lib/{support => gpr}/log_android.cc  |   0
 src/core/lib/{support => gpr}/log_linux.cc    |   0
 src/core/lib/{support => gpr}/log_posix.cc    |   0
 src/core/lib/{support => gpr}/log_windows.cc  |   4 +-
 src/core/lib/{support => gpr}/mpscq.cc        |   2 +-
 src/core/lib/{support => gpr}/mpscq.h         |   6 +-
 src/core/lib/{support => gpr}/murmur_hash.cc  |   2 +-
 src/core/lib/{support => gpr}/murmur_hash.h   |   6 +-
 src/core/lib/{support => gpr}/spinlock.h      |   6 +-
 src/core/lib/{support => gpr}/string.cc       |   2 +-
 src/core/lib/{support => gpr}/string.h        |   6 +-
 src/core/lib/{support => gpr}/string_posix.cc |   0
 .../{support => gpr}/string_util_windows.cc   |   4 +-
 .../lib/{support => gpr}/string_windows.cc    |   2 +-
 .../lib/{support => gpr}/string_windows.h     |   6 +-
 .../lib/{support => gpr}/subprocess_posix.cc  |   0
 .../{support => gpr}/subprocess_windows.cc    |   4 +-
 src/core/lib/{support => gpr}/sync.cc         |   0
 src/core/lib/{support => gpr}/sync_posix.cc   |   0
 src/core/lib/{support => gpr}/sync_windows.cc |   0
 src/core/lib/{support => gpr}/thd.cc          |   0
 src/core/lib/{support => gpr}/thd_internal.h  |   6 +-
 src/core/lib/{support => gpr}/thd_posix.cc    |   2 +-
 src/core/lib/{support => gpr}/thd_windows.cc  |   0
 src/core/lib/{support => gpr}/time.cc         |   0
 src/core/lib/{support => gpr}/time_posix.cc   |   2 +-
 src/core/lib/{support => gpr}/time_precise.cc |   2 +-
 src/core/lib/{support => gpr}/time_precise.h  |   6 +-
 src/core/lib/{support => gpr}/time_windows.cc |   2 +-
 src/core/lib/{support => gpr}/tls_pthread.cc  |   0
 src/core/lib/{support => gpr}/tmpfile.h       |   6 +-
 src/core/lib/{support => gpr}/tmpfile_msys.cc |   4 +-
 .../lib/{support => gpr}/tmpfile_posix.cc     |   4 +-
 .../lib/{support => gpr}/tmpfile_windows.cc   |   4 +-
 src/core/lib/{support => gpr}/wrap_memcpy.cc  |   0
 src/core/lib/http/format_request.cc           |   2 +-
 src/core/lib/http/httpcli.cc                  |   2 +-
 .../lib/http/httpcli_security_connector.cc    |   2 +-
 src/core/lib/iomgr/call_combiner.h            |   2 +-
 src/core/lib/iomgr/closure.h                  |   2 +-
 src/core/lib/iomgr/combiner.h                 |   2 +-
 src/core/lib/iomgr/endpoint_pair_posix.cc     |   2 +-
 src/core/lib/iomgr/ev_epoll1_linux.cc         |   4 +-
 src/core/lib/iomgr/ev_epollex_linux.cc        |   4 +-
 src/core/lib/iomgr/ev_epollsig_linux.cc       |   2 +-
 src/core/lib/iomgr/ev_poll_posix.cc           |   2 +-
 src/core/lib/iomgr/ev_posix.cc                |   2 +-
 src/core/lib/iomgr/executor.cc                |   2 +-
 src/core/lib/iomgr/fork_posix.cc              |   6 +-
 src/core/lib/iomgr/iomgr.cc                   |   4 +-
 src/core/lib/iomgr/load_file.cc               |   2 +-
 src/core/lib/iomgr/resolve_address_posix.cc   |   2 +-
 src/core/lib/iomgr/resolve_address_windows.cc |   2 +-
 src/core/lib/iomgr/sockaddr_utils.cc          |   2 +-
 .../lib/iomgr/socket_utils_common_posix.cc    |   2 +-
 src/core/lib/iomgr/tcp_client_posix.cc        |   2 +-
 src/core/lib/iomgr/tcp_posix.cc               |   2 +-
 src/core/lib/iomgr/tcp_server_posix.cc        |   2 +-
 src/core/lib/iomgr/tcp_uv.cc                  |   2 +-
 src/core/lib/iomgr/timer_generic.cc           |   2 +-
 src/core/lib/iomgr/udp_server.cc              |   2 +-
 src/core/lib/profiling/basic_timers.cc        |   2 +-
 .../lib/security/context/security_context.cc  |   2 +-
 .../lib/security/credentials/credentials.cc   |   2 +-
 .../credentials/fake/fake_credentials.cc      |   2 +-
 .../google_default/credentials_generic.cc     |   4 +-
 .../google_default_credentials.cc             |   4 +-
 .../security/credentials/jwt/json_token.cc    |   2 +-
 .../security/credentials/jwt/jwt_verifier.cc  |   2 +-
 .../security/transport/client_auth_filter.cc  |   2 +-
 .../lib/security/transport/secure_endpoint.cc |   2 +-
 .../security/transport/security_connector.cc  |   4 +-
 src/core/lib/slice/slice_intern.cc            |   2 +-
 src/core/lib/slice/slice_string_helpers.cc    |   2 +-
 src/core/lib/slice/slice_string_helpers.h     |   2 +-
 src/core/lib/surface/call.cc                  |   4 +-
 src/core/lib/surface/call_log_batch.cc        |   2 +-
 src/core/lib/surface/channel.cc               |   2 +-
 src/core/lib/surface/completion_queue.cc      |   4 +-
 src/core/lib/surface/event_string.cc          |   2 +-
 src/core/lib/surface/init.cc                  |   4 +-
 src/core/lib/surface/lame_client.cc           |   4 +-
 src/core/lib/surface/server.cc                |   6 +-
 src/core/lib/transport/metadata.cc            |   4 +-
 src/core/lib/transport/service_config.cc      |   2 +-
 src/core/lib/transport/timeout_encoding.cc    |   2 +-
 src/core/lib/transport/timeout_encoding.h     |   2 +-
 src/core/lib/transport/transport.cc           |   2 +-
 src/core/lib/transport/transport.h            |   2 +-
 src/core/lib/transport/transport_op_string.cc |   2 +-
 src/cpp/client/channel_cc.cc                  |   4 +-
 src/csharp/ext/grpc_csharp_ext.c              |   2 +-
 src/objective-c/examples/Sample/Podfile       |   2 +-
 src/objective-c/examples/SwiftSample/Podfile  |   2 +-
 src/objective-c/tests/Connectivity/Podfile    |   2 +-
 .../CoreCronetEnd2EndTests.mm                 |   6 +-
 .../tests/CronetUnitTests/CronetUnitTests.m   |   6 +-
 src/objective-c/tests/Podfile                 |   2 +-
 src/python/grpcio/grpc_core_dependencies.py   |  84 +++---
 templates/gRPC-Core.podspec.template          |   2 +-
 test/core/bad_client/bad_client.cc            |   4 +-
 test/core/bad_client/tests/large_metadata.cc  |   2 +-
 test/core/bad_ssl/bad_ssl_test.cc             |   4 +-
 .../channel/minimal_stack_is_minimal_test.cc  |   2 +-
 test/core/client_channel/lb_policies_test.cc  |   2 +-
 .../core/compression/message_compress_test.cc |   2 +-
 test/core/end2end/bad_server_response_test.cc |   2 +-
 test/core/end2end/cq_verifier.cc              |   2 +-
 test/core/end2end/dualstack_socket_test.cc    |   2 +-
 test/core/end2end/fixtures/h2_full+trace.cc   |   2 +-
 test/core/end2end/fixtures/h2_http_proxy.cc   |   2 +-
 .../end2end/fixtures/h2_sockpair+trace.cc     |   2 +-
 test/core/end2end/fixtures/h2_ssl.cc          |   6 +-
 test/core/end2end/fixtures/h2_ssl_proxy.cc    |   6 +-
 test/core/end2end/fixtures/h2_uds.cc          |   2 +-
 .../end2end/fixtures/http_proxy_fixture.cc    |   2 +-
 test/core/end2end/fuzzers/api_fuzzer.cc       |   2 +-
 test/core/end2end/h2_ssl_cert_test.cc         |   6 +-
 test/core/end2end/tests/bad_hostname.cc       |   2 +-
 test/core/end2end/tests/call_creds.cc         |   2 +-
 test/core/end2end/tests/cancel_with_status.cc |   2 +-
 test/core/end2end/tests/default_host.cc       |   2 +-
 test/core/end2end/tests/empty_batch.cc        |   2 +-
 test/core/end2end/tests/high_initial_seqno.cc |   2 +-
 test/core/end2end/tests/hpack_size.cc         |   2 +-
 test/core/end2end/tests/idempotent_request.cc |   2 +-
 test/core/end2end/tests/keepalive_timeout.cc  |   2 +-
 test/core/end2end/tests/negative_deadline.cc  |   2 +-
 test/core/end2end/tests/no_logging.cc         |   2 +-
 test/core/end2end/tests/proxy_auth.cc         |   2 +-
 test/core/end2end/tests/registered_call.cc    |   2 +-
 .../end2end/tests/server_finishes_request.cc  |   2 +-
 test/core/end2end/tests/simple_request.cc     |   2 +-
 test/core/fling/fling_stream_test.cc          |   2 +-
 test/core/fling/fling_test.cc                 |   2 +-
 test/core/gpr++/BUILD                         |  95 +++++++
 .../inlined_vector_test.cc}                   |   4 +-
 .../manual_constructor_test.cc                |   4 +-
 test/core/{support => gpr++}/memory_test.cc   |   2 +-
 .../{support => gpr++}/orphanable_test.cc     |   4 +-
 .../ref_counted_ptr_test.cc                   |   6 +-
 .../{support => gpr++}/ref_counted_test.cc    |   4 +-
 test/core/{support => gpr}/BUILD              |  78 +----
 test/core/{support => gpr}/alloc_test.cc      |   0
 test/core/{support => gpr}/arena_test.cc      |   4 +-
 test/core/{support => gpr}/avl_test.cc        |   0
 test/core/{support => gpr}/cmdline_test.cc    |   0
 test/core/{support => gpr}/cpu_test.cc        |   0
 test/core/{support => gpr}/env_test.cc        |   4 +-
 test/core/{support => gpr}/host_port_test.cc  |   0
 test/core/{support => gpr}/log_test.cc        |   2 +-
 test/core/{support => gpr}/mpscq_test.cc      |   2 +-
 .../core/{support => gpr}/murmur_hash_test.cc |   2 +-
 test/core/{support => gpr}/spinlock_test.cc   |   2 +-
 test/core/{support => gpr}/string_test.cc     |   2 +-
 test/core/{support => gpr}/sync_test.cc       |   0
 test/core/{support => gpr}/thd_test.cc        |   0
 test/core/{support => gpr}/time_test.cc       |   0
 test/core/{support => gpr}/tls_test.cc        |   0
 test/core/{support => gpr}/useful_test.cc     |   0
 test/core/iomgr/load_file_test.cc             |   4 +-
 test/core/iomgr/wakeup_fd_cv_test.cc          |   2 +-
 test/core/json/json_test.cc                   |   2 +-
 test/core/memory_usage/client.cc              |   4 +-
 test/core/memory_usage/memory_usage_test.cc   |   2 +-
 test/core/security/auth_context_test.cc       |   2 +-
 test/core/security/credentials_test.cc        |   6 +-
 .../print_google_default_creds_token.cc       |   2 +-
 test/core/security/security_connector_test.cc |   6 +-
 test/core/slice/percent_encoding_test.cc      |   2 +-
 test/core/slice/slice_string_helpers_test.cc  |   2 +-
 test/core/transport/bdp_estimator_test.cc     |   2 +-
 .../core/transport/chttp2/bin_decoder_test.cc |   2 +-
 .../core/transport/chttp2/bin_encoder_test.cc |   2 +-
 .../transport/chttp2/hpack_encoder_test.cc    |   2 +-
 .../core/transport/chttp2/hpack_table_test.cc |   2 +-
 test/core/transport/metadata_test.cc          |   2 +-
 test/core/transport/pid_controller_test.cc    |   2 +-
 test/core/transport/timeout_encoding_test.cc  |   4 +-
 test/core/tsi/transport_security_test.cc      |   2 +-
 test/core/util/test_config.cc                 |   4 +-
 test/cpp/end2end/async_end2end_test.cc        |   2 +-
 test/cpp/end2end/client_lb_end2end_test.cc    |   2 +-
 test/cpp/end2end/end2end_test.cc              |   2 +-
 test/cpp/end2end/grpclb_end2end_test.cc       |   2 +-
 test/cpp/end2end/shutdown_test.cc             |   2 +-
 test/cpp/grpclb/grpclb_test.cc                |   6 +-
 test/cpp/interop/client.cc                    |   2 +-
 test/cpp/interop/http2_client.cc              |   2 +-
 test/cpp/interop/interop_server.cc            |   2 +-
 test/cpp/interop/interop_test.cc              |   2 +-
 test/cpp/microbenchmarks/bm_arena.cc          |   2 +-
 test/cpp/microbenchmarks/bm_closure.cc        |   2 +-
 test/cpp/naming/resolver_component_test.cc    |   4 +-
 ...resolver_component_tests_runner_invoker.cc |   2 +-
 test/cpp/qps/driver.cc                        |   2 +-
 test/cpp/qps/json_run_localhost.cc            |   2 +-
 tools/doxygen/Doxyfile.c++.internal           |  44 +--
 tools/doxygen/Doxyfile.core.internal          | 130 ++++-----
 .../generated/sources_and_headers.json        | 260 ++++++++---------
 tools/run_tests/generated/tests.json          |  48 ++--
 .../run_tests/sanity/core_banned_functions.py |   4 +-
 282 files changed, 1537 insertions(+), 1464 deletions(-)
 create mode 100644 src/core/lib/gpr++/README.md
 rename src/core/lib/{support => gpr++}/abstract.h (90%)
 rename src/core/lib/{support => gpr++}/atomic.h (75%)
 rename src/core/lib/{support => gpr++}/atomic_with_atm.h (90%)
 rename src/core/lib/{support => gpr++}/atomic_with_std.h (83%)
 rename src/core/lib/{support => gpr++}/debug_location.h (90%)
 rename src/core/lib/{support/vector.h => gpr++/inlined_vector.h} (94%)
 rename src/core/lib/{support => gpr++}/manual_constructor.h (98%)
 rename src/core/lib/{support => gpr++}/memory.h (95%)
 rename src/core/lib/{support => gpr++}/orphanable.h (94%)
 rename src/core/lib/{support => gpr++}/ref_counted.h (93%)
 rename src/core/lib/{support => gpr++}/ref_counted_ptr.h (93%)
 create mode 100644 src/core/lib/gpr/README.md
 rename src/core/lib/{support => gpr}/alloc.cc (100%)
 rename src/core/lib/{support => gpr}/arena.cc (98%)
 rename src/core/lib/{support => gpr}/arena.h (91%)
 rename src/core/lib/{support => gpr}/atm.cc (100%)
 rename src/core/lib/{support => gpr}/avl.cc (100%)
 rename src/core/lib/{support => gpr}/cmdline.cc (99%)
 rename src/core/lib/{support => gpr}/cpu_iphone.cc (100%)
 rename src/core/lib/{support => gpr}/cpu_linux.cc (100%)
 rename src/core/lib/{support => gpr}/cpu_posix.cc (100%)
 rename src/core/lib/{support => gpr}/cpu_windows.cc (100%)
 rename src/core/lib/{support => gpr}/env.h (92%)
 rename src/core/lib/{support => gpr}/env_linux.cc (96%)
 rename src/core/lib/{support => gpr}/env_posix.cc (93%)
 rename src/core/lib/{support => gpr}/env_windows.cc (93%)
 rename src/core/lib/{support => gpr}/fork.cc (95%)
 rename src/core/lib/{support => gpr}/fork.h (89%)
 rename src/core/lib/{support => gpr}/host_port.cc (98%)
 rename src/core/lib/{support => gpr}/log.cc (97%)
 rename src/core/lib/{support => gpr}/log_android.cc (100%)
 rename src/core/lib/{support => gpr}/log_linux.cc (100%)
 rename src/core/lib/{support => gpr}/log_posix.cc (100%)
 rename src/core/lib/{support => gpr}/log_windows.cc (96%)
 rename src/core/lib/{support => gpr}/mpscq.cc (98%)
 rename src/core/lib/{support => gpr}/mpscq.h (96%)
 rename src/core/lib/{support => gpr}/murmur_hash.cc (97%)
 rename src/core/lib/{support => gpr}/murmur_hash.h (85%)
 rename src/core/lib/{support => gpr}/spinlock.h (90%)
 rename src/core/lib/{support => gpr}/string.cc (99%)
 rename src/core/lib/{support => gpr}/string.h (96%)
 rename src/core/lib/{support => gpr}/string_posix.cc (100%)
 rename src/core/lib/{support => gpr}/string_util_windows.cc (96%)
 rename src/core/lib/{support => gpr}/string_windows.cc (97%)
 rename src/core/lib/{support => gpr}/string_windows.h (85%)
 rename src/core/lib/{support => gpr}/subprocess_posix.cc (100%)
 rename src/core/lib/{support => gpr}/subprocess_windows.cc (97%)
 rename src/core/lib/{support => gpr}/sync.cc (100%)
 rename src/core/lib/{support => gpr}/sync_posix.cc (100%)
 rename src/core/lib/{support => gpr}/sync_windows.cc (100%)
 rename src/core/lib/{support => gpr}/thd.cc (100%)
 rename src/core/lib/{support => gpr}/thd_internal.h (85%)
 rename src/core/lib/{support => gpr}/thd_posix.cc (99%)
 rename src/core/lib/{support => gpr}/thd_windows.cc (100%)
 rename src/core/lib/{support => gpr}/time.cc (100%)
 rename src/core/lib/{support => gpr}/time_posix.cc (98%)
 rename src/core/lib/{support => gpr}/time_precise.cc (97%)
 rename src/core/lib/{support => gpr}/time_precise.h (83%)
 rename src/core/lib/{support => gpr}/time_windows.cc (98%)
 rename src/core/lib/{support => gpr}/tls_pthread.cc (100%)
 rename src/core/lib/{support => gpr}/tmpfile.h (88%)
 rename src/core/lib/{support => gpr}/tmpfile_msys.cc (94%)
 rename src/core/lib/{support => gpr}/tmpfile_posix.cc (95%)
 rename src/core/lib/{support => gpr}/tmpfile_windows.cc (95%)
 rename src/core/lib/{support => gpr}/wrap_memcpy.cc (100%)
 create mode 100644 test/core/gpr++/BUILD
 rename test/core/{support/vector_test.cc => gpr++/inlined_vector_test.cc} (95%)
 rename test/core/{support => gpr++}/manual_constructor_test.cc (96%)
 rename test/core/{support => gpr++}/memory_test.cc (97%)
 rename test/core/{support => gpr++}/orphanable_test.cc (96%)
 rename test/core/{support => gpr++}/ref_counted_ptr_test.cc (97%)
 rename test/core/{support => gpr++}/ref_counted_test.cc (95%)
 rename test/core/{support => gpr}/BUILD (70%)
 rename test/core/{support => gpr}/alloc_test.cc (100%)
 rename test/core/{support => gpr}/arena_test.cc (97%)
 rename test/core/{support => gpr}/avl_test.cc (100%)
 rename test/core/{support => gpr}/cmdline_test.cc (100%)
 rename test/core/{support => gpr}/cpu_test.cc (100%)
 rename test/core/{support => gpr}/env_test.cc (94%)
 rename test/core/{support => gpr}/host_port_test.cc (100%)
 rename test/core/{support => gpr}/log_test.cc (98%)
 rename test/core/{support => gpr}/mpscq_test.cc (99%)
 rename test/core/{support => gpr}/murmur_hash_test.cc (97%)
 rename test/core/{support => gpr}/spinlock_test.cc (99%)
 rename test/core/{support => gpr}/string_test.cc (99%)
 rename test/core/{support => gpr}/sync_test.cc (100%)
 rename test/core/{support => gpr}/thd_test.cc (100%)
 rename test/core/{support => gpr}/time_test.cc (100%)
 rename test/core/{support => gpr}/tls_test.cc (100%)
 rename test/core/{support => gpr}/useful_test.cc (100%)

diff --git a/BUILD b/BUILD
index 186d664395a..c52a95cf91b 100644
--- a/BUILD
+++ b/BUILD
@@ -448,75 +448,67 @@ grpc_cc_library(
     srcs = [
         "src/core/lib/profiling/basic_timers.cc",
         "src/core/lib/profiling/stap_timers.cc",
-        "src/core/lib/support/alloc.cc",
-        "src/core/lib/support/arena.cc",
-        "src/core/lib/support/atm.cc",
-        "src/core/lib/support/avl.cc",
-        "src/core/lib/support/cmdline.cc",
-        "src/core/lib/support/cpu_iphone.cc",
-        "src/core/lib/support/cpu_linux.cc",
-        "src/core/lib/support/cpu_posix.cc",
-        "src/core/lib/support/cpu_windows.cc",
-        "src/core/lib/support/env_linux.cc",
-        "src/core/lib/support/env_posix.cc",
-        "src/core/lib/support/env_windows.cc",
-        "src/core/lib/support/fork.cc",
-        "src/core/lib/support/host_port.cc",
-        "src/core/lib/support/log.cc",
-        "src/core/lib/support/log_android.cc",
-        "src/core/lib/support/log_linux.cc",
-        "src/core/lib/support/log_posix.cc",
-        "src/core/lib/support/log_windows.cc",
-        "src/core/lib/support/mpscq.cc",
-        "src/core/lib/support/murmur_hash.cc",
-        "src/core/lib/support/string.cc",
-        "src/core/lib/support/string_posix.cc",
-        "src/core/lib/support/string_util_windows.cc",
-        "src/core/lib/support/string_windows.cc",
-        "src/core/lib/support/subprocess_posix.cc",
-        "src/core/lib/support/subprocess_windows.cc",
-        "src/core/lib/support/sync.cc",
-        "src/core/lib/support/sync_posix.cc",
-        "src/core/lib/support/sync_windows.cc",
-        "src/core/lib/support/thd.cc",
-        "src/core/lib/support/thd_posix.cc",
-        "src/core/lib/support/thd_windows.cc",
-        "src/core/lib/support/time.cc",
-        "src/core/lib/support/time_posix.cc",
-        "src/core/lib/support/time_precise.cc",
-        "src/core/lib/support/time_windows.cc",
-        "src/core/lib/support/tls_pthread.cc",
-        "src/core/lib/support/tmpfile_msys.cc",
-        "src/core/lib/support/tmpfile_posix.cc",
-        "src/core/lib/support/tmpfile_windows.cc",
-        "src/core/lib/support/wrap_memcpy.cc",
+        "src/core/lib/gpr/alloc.cc",
+        "src/core/lib/gpr/arena.cc",
+        "src/core/lib/gpr/atm.cc",
+        "src/core/lib/gpr/avl.cc",
+        "src/core/lib/gpr/cmdline.cc",
+        "src/core/lib/gpr/cpu_iphone.cc",
+        "src/core/lib/gpr/cpu_linux.cc",
+        "src/core/lib/gpr/cpu_posix.cc",
+        "src/core/lib/gpr/cpu_windows.cc",
+        "src/core/lib/gpr/env_linux.cc",
+        "src/core/lib/gpr/env_posix.cc",
+        "src/core/lib/gpr/env_windows.cc",
+        "src/core/lib/gpr/fork.cc",
+        "src/core/lib/gpr/host_port.cc",
+        "src/core/lib/gpr/log.cc",
+        "src/core/lib/gpr/log_android.cc",
+        "src/core/lib/gpr/log_linux.cc",
+        "src/core/lib/gpr/log_posix.cc",
+        "src/core/lib/gpr/log_windows.cc",
+        "src/core/lib/gpr/mpscq.cc",
+        "src/core/lib/gpr/murmur_hash.cc",
+        "src/core/lib/gpr/string.cc",
+        "src/core/lib/gpr/string_posix.cc",
+        "src/core/lib/gpr/string_util_windows.cc",
+        "src/core/lib/gpr/string_windows.cc",
+        "src/core/lib/gpr/subprocess_posix.cc",
+        "src/core/lib/gpr/subprocess_windows.cc",
+        "src/core/lib/gpr/sync.cc",
+        "src/core/lib/gpr/sync_posix.cc",
+        "src/core/lib/gpr/sync_windows.cc",
+        "src/core/lib/gpr/thd.cc",
+        "src/core/lib/gpr/thd_posix.cc",
+        "src/core/lib/gpr/thd_windows.cc",
+        "src/core/lib/gpr/time.cc",
+        "src/core/lib/gpr/time_posix.cc",
+        "src/core/lib/gpr/time_precise.cc",
+        "src/core/lib/gpr/time_windows.cc",
+        "src/core/lib/gpr/tls_pthread.cc",
+        "src/core/lib/gpr/tmpfile_msys.cc",
+        "src/core/lib/gpr/tmpfile_posix.cc",
+        "src/core/lib/gpr/tmpfile_windows.cc",
+        "src/core/lib/gpr/wrap_memcpy.cc",
     ],
     hdrs = [
         "src/core/lib/profiling/timers.h",
-        "src/core/lib/support/abstract.h",
-        "src/core/lib/support/arena.h",
-        "src/core/lib/support/atomic.h",
-        "src/core/lib/support/atomic_with_atm.h",
-        "src/core/lib/support/atomic_with_std.h",
-        "src/core/lib/support/env.h",
-        "src/core/lib/support/fork.h",
-        "src/core/lib/support/manual_constructor.h",
-        "src/core/lib/support/memory.h",
-        "src/core/lib/support/mpscq.h",
-        "src/core/lib/support/murmur_hash.h",
-        "src/core/lib/support/spinlock.h",
-        "src/core/lib/support/string.h",
-        "src/core/lib/support/string_windows.h",
-        "src/core/lib/support/thd_internal.h",
-        "src/core/lib/support/time_precise.h",
-        "src/core/lib/support/tmpfile.h",
-        "src/core/lib/support/vector.h",
+        "src/core/lib/gpr/arena.h",
+        "src/core/lib/gpr/env.h",
+        "src/core/lib/gpr/fork.h",
+        "src/core/lib/gpr/mpscq.h",
+        "src/core/lib/gpr/murmur_hash.h",
+        "src/core/lib/gpr/spinlock.h",
+        "src/core/lib/gpr/string.h",
+        "src/core/lib/gpr/string_windows.h",
+        "src/core/lib/gpr/thd_internal.h",
+        "src/core/lib/gpr/time_precise.h",
+        "src/core/lib/gpr/tmpfile.h",
     ],
     language = "c++",
     public_hdrs = GPR_PUBLIC_HDRS,
     deps = [
         "gpr_codegen",
-        "@com_google_absl//absl/container:inlined_vector",
     ],
 )
 
@@ -552,16 +544,49 @@ grpc_cc_library(
     ],
 )
 
+grpc_cc_library(
+    name = "gpr++_base",
+    language = "c++",
+    public_hdrs = [
+        "src/core/lib/gpr++/abstract.h",
+        "src/core/lib/gpr++/manual_constructor.h",
+        "src/core/lib/gpr++/memory.h",
+    ],
+)
+
+grpc_cc_library(
+    name = "atomic",
+    language = "c++",
+    public_hdrs = [
+        "src/core/lib/gpr++/atomic.h",
+    ],
+    hdrs = [
+        "src/core/lib/gpr++/atomic_with_atm.h",
+        "src/core/lib/gpr++/atomic_with_std.h",
+    ],
+    deps = [
+        "gpr",
+    ],
+)
+
+grpc_cc_library(
+    name = "inlined_vector",
+    language = "c++",
+    public_hdrs = [
+        "src/core/lib/gpr++/inlined_vector.h",
+    ],
+)
+
 grpc_cc_library(
     name = "debug_location",
     language = "c++",
-    public_hdrs = ["src/core/lib/support/debug_location.h"],
+    public_hdrs = ["src/core/lib/gpr++/debug_location.h"],
 )
 
 grpc_cc_library(
     name = "orphanable",
     language = "c++",
-    public_hdrs = ["src/core/lib/support/orphanable.h"],
+    public_hdrs = ["src/core/lib/gpr++/orphanable.h"],
     deps = [
         "debug_location",
         "grpc_trace",
@@ -571,7 +596,7 @@ grpc_cc_library(
 grpc_cc_library(
     name = "ref_counted",
     language = "c++",
-    public_hdrs = ["src/core/lib/support/ref_counted.h"],
+    public_hdrs = ["src/core/lib/gpr++/ref_counted.h"],
     deps = [
         "debug_location",
         "grpc_trace",
@@ -581,7 +606,7 @@ grpc_cc_library(
 grpc_cc_library(
     name = "ref_counted_ptr",
     language = "c++",
-    public_hdrs = ["src/core/lib/support/ref_counted_ptr.h"],
+    public_hdrs = ["src/core/lib/gpr++/ref_counted_ptr.h"],
 )
 
 grpc_cc_library(
@@ -848,6 +873,7 @@ grpc_cc_library(
     public_hdrs = GRPC_PUBLIC_HDRS,
     deps = [
         "gpr_base",
+        "gpr++_base",
         "grpc_codegen",
         "grpc_trace",
     ],
@@ -860,6 +886,7 @@ grpc_cc_library(
     ],
     language = "c++",
     deps = [
+        "atomic",
         "grpc_base_c",
     ],
 )
@@ -1320,6 +1347,7 @@ grpc_cc_library(
     ],
     language = "c++",
     deps = [
+        "gpr++_base",
         "grpc_base",
         "grpc_http_filters",
         "grpc_transport_chttp2_alpn",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 97d68cc62c7..4450a73e9d5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -544,6 +544,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_cxx http2_client)
 endif()
 add_dependencies(buildtests_cxx hybrid_end2end_test)
+add_dependencies(buildtests_cxx inlined_vector_test)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_cxx inproc_sync_unary_ping_pong_test)
 endif()
@@ -600,7 +601,6 @@ add_dependencies(buildtests_cxx stress_test)
 add_dependencies(buildtests_cxx thread_manager_test)
 add_dependencies(buildtests_cxx thread_stress_test)
 add_dependencies(buildtests_cxx transport_pid_controller_test)
-add_dependencies(buildtests_cxx vector_test)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_cxx writes_per_rpc_test)
 endif()
@@ -623,50 +623,50 @@ endif (gRPC_BUILD_TESTS)
 
 
 add_library(gpr
+  src/core/lib/gpr/alloc.cc
+  src/core/lib/gpr/arena.cc
+  src/core/lib/gpr/atm.cc
+  src/core/lib/gpr/avl.cc
+  src/core/lib/gpr/cmdline.cc
+  src/core/lib/gpr/cpu_iphone.cc
+  src/core/lib/gpr/cpu_linux.cc
+  src/core/lib/gpr/cpu_posix.cc
+  src/core/lib/gpr/cpu_windows.cc
+  src/core/lib/gpr/env_linux.cc
+  src/core/lib/gpr/env_posix.cc
+  src/core/lib/gpr/env_windows.cc
+  src/core/lib/gpr/fork.cc
+  src/core/lib/gpr/host_port.cc
+  src/core/lib/gpr/log.cc
+  src/core/lib/gpr/log_android.cc
+  src/core/lib/gpr/log_linux.cc
+  src/core/lib/gpr/log_posix.cc
+  src/core/lib/gpr/log_windows.cc
+  src/core/lib/gpr/mpscq.cc
+  src/core/lib/gpr/murmur_hash.cc
+  src/core/lib/gpr/string.cc
+  src/core/lib/gpr/string_posix.cc
+  src/core/lib/gpr/string_util_windows.cc
+  src/core/lib/gpr/string_windows.cc
+  src/core/lib/gpr/subprocess_posix.cc
+  src/core/lib/gpr/subprocess_windows.cc
+  src/core/lib/gpr/sync.cc
+  src/core/lib/gpr/sync_posix.cc
+  src/core/lib/gpr/sync_windows.cc
+  src/core/lib/gpr/thd.cc
+  src/core/lib/gpr/thd_posix.cc
+  src/core/lib/gpr/thd_windows.cc
+  src/core/lib/gpr/time.cc
+  src/core/lib/gpr/time_posix.cc
+  src/core/lib/gpr/time_precise.cc
+  src/core/lib/gpr/time_windows.cc
+  src/core/lib/gpr/tls_pthread.cc
+  src/core/lib/gpr/tmpfile_msys.cc
+  src/core/lib/gpr/tmpfile_posix.cc
+  src/core/lib/gpr/tmpfile_windows.cc
+  src/core/lib/gpr/wrap_memcpy.cc
   src/core/lib/profiling/basic_timers.cc
   src/core/lib/profiling/stap_timers.cc
-  src/core/lib/support/alloc.cc
-  src/core/lib/support/arena.cc
-  src/core/lib/support/atm.cc
-  src/core/lib/support/avl.cc
-  src/core/lib/support/cmdline.cc
-  src/core/lib/support/cpu_iphone.cc
-  src/core/lib/support/cpu_linux.cc
-  src/core/lib/support/cpu_posix.cc
-  src/core/lib/support/cpu_windows.cc
-  src/core/lib/support/env_linux.cc
-  src/core/lib/support/env_posix.cc
-  src/core/lib/support/env_windows.cc
-  src/core/lib/support/fork.cc
-  src/core/lib/support/host_port.cc
-  src/core/lib/support/log.cc
-  src/core/lib/support/log_android.cc
-  src/core/lib/support/log_linux.cc
-  src/core/lib/support/log_posix.cc
-  src/core/lib/support/log_windows.cc
-  src/core/lib/support/mpscq.cc
-  src/core/lib/support/murmur_hash.cc
-  src/core/lib/support/string.cc
-  src/core/lib/support/string_posix.cc
-  src/core/lib/support/string_util_windows.cc
-  src/core/lib/support/string_windows.cc
-  src/core/lib/support/subprocess_posix.cc
-  src/core/lib/support/subprocess_windows.cc
-  src/core/lib/support/sync.cc
-  src/core/lib/support/sync_posix.cc
-  src/core/lib/support/sync_windows.cc
-  src/core/lib/support/thd.cc
-  src/core/lib/support/thd_posix.cc
-  src/core/lib/support/thd_windows.cc
-  src/core/lib/support/time.cc
-  src/core/lib/support/time_posix.cc
-  src/core/lib/support/time_precise.cc
-  src/core/lib/support/time_windows.cc
-  src/core/lib/support/tls_pthread.cc
-  src/core/lib/support/tmpfile_msys.cc
-  src/core/lib/support/tmpfile_posix.cc
-  src/core/lib/support/tmpfile_windows.cc
-  src/core/lib/support/wrap_memcpy.cc
 )
 
 if(WIN32 AND MSVC)
@@ -4784,7 +4784,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(alloc_test
-  test/core/support/alloc_test.cc
+  test/core/gpr/alloc_test.cc
 )
 
 
@@ -4836,7 +4836,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(arena_test
-  test/core/support/arena_test.cc
+  test/core/gpr/arena_test.cc
 )
 
 
@@ -5635,7 +5635,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_avl_test
-  test/core/support/avl_test.cc
+  test/core/gpr/avl_test.cc
 )
 
 
@@ -5660,7 +5660,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_cmdline_test
-  test/core/support/cmdline_test.cc
+  test/core/gpr/cmdline_test.cc
 )
 
 
@@ -5685,7 +5685,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_cpu_test
-  test/core/support/cpu_test.cc
+  test/core/gpr/cpu_test.cc
 )
 
 
@@ -5710,7 +5710,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_env_test
-  test/core/support/env_test.cc
+  test/core/gpr/env_test.cc
 )
 
 
@@ -5735,7 +5735,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_host_port_test
-  test/core/support/host_port_test.cc
+  test/core/gpr/host_port_test.cc
 )
 
 
@@ -5760,7 +5760,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_log_test
-  test/core/support/log_test.cc
+  test/core/gpr/log_test.cc
 )
 
 
@@ -5785,7 +5785,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_manual_constructor_test
-  test/core/support/manual_constructor_test.cc
+  test/core/gpr++/manual_constructor_test.cc
 )
 
 
@@ -5810,7 +5810,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_mpscq_test
-  test/core/support/mpscq_test.cc
+  test/core/gpr/mpscq_test.cc
 )
 
 
@@ -5835,7 +5835,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_spinlock_test
-  test/core/support/spinlock_test.cc
+  test/core/gpr/spinlock_test.cc
 )
 
 
@@ -5860,7 +5860,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_string_test
-  test/core/support/string_test.cc
+  test/core/gpr/string_test.cc
 )
 
 
@@ -5885,7 +5885,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_sync_test
-  test/core/support/sync_test.cc
+  test/core/gpr/sync_test.cc
 )
 
 
@@ -5910,7 +5910,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_thd_test
-  test/core/support/thd_test.cc
+  test/core/gpr/thd_test.cc
 )
 
 
@@ -5935,7 +5935,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_time_test
-  test/core/support/time_test.cc
+  test/core/gpr/time_test.cc
 )
 
 
@@ -5960,7 +5960,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_tls_test
-  test/core/support/tls_test.cc
+  test/core/gpr/tls_test.cc
 )
 
 
@@ -5985,7 +5985,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_useful_test
-  test/core/support/useful_test.cc
+  test/core/gpr/useful_test.cc
 )
 
 
@@ -7202,7 +7202,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(murmur_hash_test
-  test/core/support/murmur_hash_test.cc
+  test/core/gpr/murmur_hash_test.cc
 )
 
 
@@ -10552,6 +10552,43 @@ target_link_libraries(hybrid_end2end_test
   ${_gRPC_GFLAGS_LIBRARIES}
 )
 
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
+add_executable(inlined_vector_test
+  test/core/gpr++/inlined_vector_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+
+target_include_directories(inlined_vector_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(inlined_vector_test
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc_test_util
+  grpc++
+  grpc
+  gpr_test_util
+  gpr
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
@@ -10764,7 +10801,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(memory_test
-  test/core/support/memory_test.cc
+  test/core/gpr++/memory_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
   third_party/googletest/googlemock/src/gmock-all.cc
 )
@@ -10915,7 +10952,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(orphanable_test
-  test/core/support/orphanable_test.cc
+  test/core/gpr++/orphanable_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
   third_party/googletest/googlemock/src/gmock-all.cc
 )
@@ -11315,7 +11352,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(ref_counted_ptr_test
-  test/core/support/ref_counted_ptr_test.cc
+  test/core/gpr++/ref_counted_ptr_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
   third_party/googletest/googlemock/src/gmock-all.cc
 )
@@ -11352,7 +11389,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(ref_counted_test
-  test/core/support/ref_counted_test.cc
+  test/core/gpr++/ref_counted_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
   third_party/googletest/googlemock/src/gmock-all.cc
 )
@@ -12057,43 +12094,6 @@ target_link_libraries(transport_pid_controller_test
   ${_gRPC_GFLAGS_LIBRARIES}
 )
 
-endif (gRPC_BUILD_TESTS)
-if (gRPC_BUILD_TESTS)
-
-add_executable(vector_test
-  test/core/support/vector_test.cc
-  third_party/googletest/googletest/src/gtest-all.cc
-  third_party/googletest/googlemock/src/gmock-all.cc
-)
-
-
-target_include_directories(vector_test
-  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
-  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
-  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
-  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
-  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
-  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
-  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
-  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
-  PRIVATE third_party/googletest/googletest/include
-  PRIVATE third_party/googletest/googletest
-  PRIVATE third_party/googletest/googlemock/include
-  PRIVATE third_party/googletest/googlemock
-  PRIVATE ${_gRPC_PROTO_GENS_DIR}
-)
-
-target_link_libraries(vector_test
-  ${_gRPC_PROTOBUF_LIBRARIES}
-  ${_gRPC_ALLTARGETS_LIBRARIES}
-  grpc_test_util
-  grpc++
-  grpc
-  gpr_test_util
-  gpr
-  ${_gRPC_GFLAGS_LIBRARIES}
-)
-
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
diff --git a/Makefile b/Makefile
index 3265cea362e..69a68a6d66a 100644
--- a/Makefile
+++ b/Makefile
@@ -1151,6 +1151,7 @@ h2_ssl_cert_test: $(BINDIR)/$(CONFIG)/h2_ssl_cert_test
 health_service_end2end_test: $(BINDIR)/$(CONFIG)/health_service_end2end_test
 http2_client: $(BINDIR)/$(CONFIG)/http2_client
 hybrid_end2end_test: $(BINDIR)/$(CONFIG)/hybrid_end2end_test
+inlined_vector_test: $(BINDIR)/$(CONFIG)/inlined_vector_test
 inproc_sync_unary_ping_pong_test: $(BINDIR)/$(CONFIG)/inproc_sync_unary_ping_pong_test
 interop_client: $(BINDIR)/$(CONFIG)/interop_client
 interop_server: $(BINDIR)/$(CONFIG)/interop_server
@@ -1187,7 +1188,6 @@ stress_test: $(BINDIR)/$(CONFIG)/stress_test
 thread_manager_test: $(BINDIR)/$(CONFIG)/thread_manager_test
 thread_stress_test: $(BINDIR)/$(CONFIG)/thread_stress_test
 transport_pid_controller_test: $(BINDIR)/$(CONFIG)/transport_pid_controller_test
-vector_test: $(BINDIR)/$(CONFIG)/vector_test
 writes_per_rpc_test: $(BINDIR)/$(CONFIG)/writes_per_rpc_test
 public_headers_must_be_c89: $(BINDIR)/$(CONFIG)/public_headers_must_be_c89
 gen_hpack_tables: $(BINDIR)/$(CONFIG)/gen_hpack_tables
@@ -1594,6 +1594,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/health_service_end2end_test \
   $(BINDIR)/$(CONFIG)/http2_client \
   $(BINDIR)/$(CONFIG)/hybrid_end2end_test \
+  $(BINDIR)/$(CONFIG)/inlined_vector_test \
   $(BINDIR)/$(CONFIG)/inproc_sync_unary_ping_pong_test \
   $(BINDIR)/$(CONFIG)/interop_client \
   $(BINDIR)/$(CONFIG)/interop_server \
@@ -1630,7 +1631,6 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/thread_manager_test \
   $(BINDIR)/$(CONFIG)/thread_stress_test \
   $(BINDIR)/$(CONFIG)/transport_pid_controller_test \
-  $(BINDIR)/$(CONFIG)/vector_test \
   $(BINDIR)/$(CONFIG)/writes_per_rpc_test \
   $(BINDIR)/$(CONFIG)/boringssl_aes_test \
   $(BINDIR)/$(CONFIG)/boringssl_asn1_test \
@@ -1726,6 +1726,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/health_service_end2end_test \
   $(BINDIR)/$(CONFIG)/http2_client \
   $(BINDIR)/$(CONFIG)/hybrid_end2end_test \
+  $(BINDIR)/$(CONFIG)/inlined_vector_test \
   $(BINDIR)/$(CONFIG)/inproc_sync_unary_ping_pong_test \
   $(BINDIR)/$(CONFIG)/interop_client \
   $(BINDIR)/$(CONFIG)/interop_server \
@@ -1762,7 +1763,6 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/thread_manager_test \
   $(BINDIR)/$(CONFIG)/thread_stress_test \
   $(BINDIR)/$(CONFIG)/transport_pid_controller_test \
-  $(BINDIR)/$(CONFIG)/vector_test \
   $(BINDIR)/$(CONFIG)/writes_per_rpc_test \
   $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure \
   $(BINDIR)/$(CONFIG)/resolver_component_test \
@@ -2135,6 +2135,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/h2_ssl_cert_test || ( echo test h2_ssl_cert_test failed ; exit 1 )
 	$(E) "[RUN]     Testing health_service_end2end_test"
 	$(Q) $(BINDIR)/$(CONFIG)/health_service_end2end_test || ( echo test health_service_end2end_test failed ; exit 1 )
+	$(E) "[RUN]     Testing inlined_vector_test"
+	$(Q) $(BINDIR)/$(CONFIG)/inlined_vector_test || ( echo test inlined_vector_test failed ; exit 1 )
 	$(E) "[RUN]     Testing inproc_sync_unary_ping_pong_test"
 	$(Q) $(BINDIR)/$(CONFIG)/inproc_sync_unary_ping_pong_test || ( echo test inproc_sync_unary_ping_pong_test failed ; exit 1 )
 	$(E) "[RUN]     Testing interop_test"
@@ -2185,8 +2187,6 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/thread_stress_test || ( echo test thread_stress_test failed ; exit 1 )
 	$(E) "[RUN]     Testing transport_pid_controller_test"
 	$(Q) $(BINDIR)/$(CONFIG)/transport_pid_controller_test || ( echo test transport_pid_controller_test failed ; exit 1 )
-	$(E) "[RUN]     Testing vector_test"
-	$(Q) $(BINDIR)/$(CONFIG)/vector_test || ( echo test vector_test failed ; exit 1 )
 	$(E) "[RUN]     Testing writes_per_rpc_test"
 	$(Q) $(BINDIR)/$(CONFIG)/writes_per_rpc_test || ( echo test writes_per_rpc_test failed ; exit 1 )
 	$(E) "[RUN]     Testing resolver_component_tests_runner_invoker_unsecure"
@@ -2840,50 +2840,50 @@ clean:
 
 
 LIBGPR_SRC = \
+    src/core/lib/gpr/alloc.cc \
+    src/core/lib/gpr/arena.cc \
+    src/core/lib/gpr/atm.cc \
+    src/core/lib/gpr/avl.cc \
+    src/core/lib/gpr/cmdline.cc \
+    src/core/lib/gpr/cpu_iphone.cc \
+    src/core/lib/gpr/cpu_linux.cc \
+    src/core/lib/gpr/cpu_posix.cc \
+    src/core/lib/gpr/cpu_windows.cc \
+    src/core/lib/gpr/env_linux.cc \
+    src/core/lib/gpr/env_posix.cc \
+    src/core/lib/gpr/env_windows.cc \
+    src/core/lib/gpr/fork.cc \
+    src/core/lib/gpr/host_port.cc \
+    src/core/lib/gpr/log.cc \
+    src/core/lib/gpr/log_android.cc \
+    src/core/lib/gpr/log_linux.cc \
+    src/core/lib/gpr/log_posix.cc \
+    src/core/lib/gpr/log_windows.cc \
+    src/core/lib/gpr/mpscq.cc \
+    src/core/lib/gpr/murmur_hash.cc \
+    src/core/lib/gpr/string.cc \
+    src/core/lib/gpr/string_posix.cc \
+    src/core/lib/gpr/string_util_windows.cc \
+    src/core/lib/gpr/string_windows.cc \
+    src/core/lib/gpr/subprocess_posix.cc \
+    src/core/lib/gpr/subprocess_windows.cc \
+    src/core/lib/gpr/sync.cc \
+    src/core/lib/gpr/sync_posix.cc \
+    src/core/lib/gpr/sync_windows.cc \
+    src/core/lib/gpr/thd.cc \
+    src/core/lib/gpr/thd_posix.cc \
+    src/core/lib/gpr/thd_windows.cc \
+    src/core/lib/gpr/time.cc \
+    src/core/lib/gpr/time_posix.cc \
+    src/core/lib/gpr/time_precise.cc \
+    src/core/lib/gpr/time_windows.cc \
+    src/core/lib/gpr/tls_pthread.cc \
+    src/core/lib/gpr/tmpfile_msys.cc \
+    src/core/lib/gpr/tmpfile_posix.cc \
+    src/core/lib/gpr/tmpfile_windows.cc \
+    src/core/lib/gpr/wrap_memcpy.cc \
     src/core/lib/profiling/basic_timers.cc \
     src/core/lib/profiling/stap_timers.cc \
-    src/core/lib/support/alloc.cc \
-    src/core/lib/support/arena.cc \
-    src/core/lib/support/atm.cc \
-    src/core/lib/support/avl.cc \
-    src/core/lib/support/cmdline.cc \
-    src/core/lib/support/cpu_iphone.cc \
-    src/core/lib/support/cpu_linux.cc \
-    src/core/lib/support/cpu_posix.cc \
-    src/core/lib/support/cpu_windows.cc \
-    src/core/lib/support/env_linux.cc \
-    src/core/lib/support/env_posix.cc \
-    src/core/lib/support/env_windows.cc \
-    src/core/lib/support/fork.cc \
-    src/core/lib/support/host_port.cc \
-    src/core/lib/support/log.cc \
-    src/core/lib/support/log_android.cc \
-    src/core/lib/support/log_linux.cc \
-    src/core/lib/support/log_posix.cc \
-    src/core/lib/support/log_windows.cc \
-    src/core/lib/support/mpscq.cc \
-    src/core/lib/support/murmur_hash.cc \
-    src/core/lib/support/string.cc \
-    src/core/lib/support/string_posix.cc \
-    src/core/lib/support/string_util_windows.cc \
-    src/core/lib/support/string_windows.cc \
-    src/core/lib/support/subprocess_posix.cc \
-    src/core/lib/support/subprocess_windows.cc \
-    src/core/lib/support/sync.cc \
-    src/core/lib/support/sync_posix.cc \
-    src/core/lib/support/sync_windows.cc \
-    src/core/lib/support/thd.cc \
-    src/core/lib/support/thd_posix.cc \
-    src/core/lib/support/thd_windows.cc \
-    src/core/lib/support/time.cc \
-    src/core/lib/support/time_posix.cc \
-    src/core/lib/support/time_precise.cc \
-    src/core/lib/support/time_windows.cc \
-    src/core/lib/support/tls_pthread.cc \
-    src/core/lib/support/tmpfile_msys.cc \
-    src/core/lib/support/tmpfile_posix.cc \
-    src/core/lib/support/tmpfile_windows.cc \
-    src/core/lib/support/wrap_memcpy.cc \
 
 PUBLIC_HEADERS_C += \
     include/grpc/support/alloc.h \
@@ -8798,7 +8798,7 @@ endif
 
 
 ALLOC_TEST_SRC = \
-    test/core/support/alloc_test.cc \
+    test/core/gpr/alloc_test.cc \
 
 ALLOC_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(ALLOC_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -8818,7 +8818,7 @@ $(BINDIR)/$(CONFIG)/alloc_test: $(ALLOC_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_te
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/alloc_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/alloc_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_alloc_test: $(ALLOC_TEST_OBJS:.o=.dep)
 
@@ -8894,7 +8894,7 @@ endif
 
 
 ARENA_TEST_SRC = \
-    test/core/support/arena_test.cc \
+    test/core/gpr/arena_test.cc \
 
 ARENA_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(ARENA_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -8914,7 +8914,7 @@ $(BINDIR)/$(CONFIG)/arena_test: $(ARENA_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_te
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/arena_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/arena_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_arena_test: $(ARENA_TEST_OBJS:.o=.dep)
 
@@ -9857,7 +9857,7 @@ endif
 
 
 GPR_AVL_TEST_SRC = \
-    test/core/support/avl_test.cc \
+    test/core/gpr/avl_test.cc \
 
 GPR_AVL_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_AVL_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -9877,7 +9877,7 @@ $(BINDIR)/$(CONFIG)/gpr_avl_test: $(GPR_AVL_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgp
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/avl_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/avl_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_avl_test: $(GPR_AVL_TEST_OBJS:.o=.dep)
 
@@ -9889,7 +9889,7 @@ endif
 
 
 GPR_CMDLINE_TEST_SRC = \
-    test/core/support/cmdline_test.cc \
+    test/core/gpr/cmdline_test.cc \
 
 GPR_CMDLINE_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_CMDLINE_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -9909,7 +9909,7 @@ $(BINDIR)/$(CONFIG)/gpr_cmdline_test: $(GPR_CMDLINE_TEST_OBJS) $(LIBDIR)/$(CONFI
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/cmdline_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/cmdline_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_cmdline_test: $(GPR_CMDLINE_TEST_OBJS:.o=.dep)
 
@@ -9921,7 +9921,7 @@ endif
 
 
 GPR_CPU_TEST_SRC = \
-    test/core/support/cpu_test.cc \
+    test/core/gpr/cpu_test.cc \
 
 GPR_CPU_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_CPU_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -9941,7 +9941,7 @@ $(BINDIR)/$(CONFIG)/gpr_cpu_test: $(GPR_CPU_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgp
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/cpu_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/cpu_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_cpu_test: $(GPR_CPU_TEST_OBJS:.o=.dep)
 
@@ -9953,7 +9953,7 @@ endif
 
 
 GPR_ENV_TEST_SRC = \
-    test/core/support/env_test.cc \
+    test/core/gpr/env_test.cc \
 
 GPR_ENV_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_ENV_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -9973,7 +9973,7 @@ $(BINDIR)/$(CONFIG)/gpr_env_test: $(GPR_ENV_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgp
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/env_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/env_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_env_test: $(GPR_ENV_TEST_OBJS:.o=.dep)
 
@@ -9985,7 +9985,7 @@ endif
 
 
 GPR_HOST_PORT_TEST_SRC = \
-    test/core/support/host_port_test.cc \
+    test/core/gpr/host_port_test.cc \
 
 GPR_HOST_PORT_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_HOST_PORT_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10005,7 +10005,7 @@ $(BINDIR)/$(CONFIG)/gpr_host_port_test: $(GPR_HOST_PORT_TEST_OBJS) $(LIBDIR)/$(C
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/host_port_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/host_port_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_host_port_test: $(GPR_HOST_PORT_TEST_OBJS:.o=.dep)
 
@@ -10017,7 +10017,7 @@ endif
 
 
 GPR_LOG_TEST_SRC = \
-    test/core/support/log_test.cc \
+    test/core/gpr/log_test.cc \
 
 GPR_LOG_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_LOG_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10037,7 +10037,7 @@ $(BINDIR)/$(CONFIG)/gpr_log_test: $(GPR_LOG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgp
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/log_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/log_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_log_test: $(GPR_LOG_TEST_OBJS:.o=.dep)
 
@@ -10049,7 +10049,7 @@ endif
 
 
 GPR_MANUAL_CONSTRUCTOR_TEST_SRC = \
-    test/core/support/manual_constructor_test.cc \
+    test/core/gpr++/manual_constructor_test.cc \
 
 GPR_MANUAL_CONSTRUCTOR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_MANUAL_CONSTRUCTOR_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10069,7 +10069,7 @@ $(BINDIR)/$(CONFIG)/gpr_manual_constructor_test: $(GPR_MANUAL_CONSTRUCTOR_TEST_O
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/manual_constructor_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr++/manual_constructor_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_manual_constructor_test: $(GPR_MANUAL_CONSTRUCTOR_TEST_OBJS:.o=.dep)
 
@@ -10081,7 +10081,7 @@ endif
 
 
 GPR_MPSCQ_TEST_SRC = \
-    test/core/support/mpscq_test.cc \
+    test/core/gpr/mpscq_test.cc \
 
 GPR_MPSCQ_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_MPSCQ_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10101,7 +10101,7 @@ $(BINDIR)/$(CONFIG)/gpr_mpscq_test: $(GPR_MPSCQ_TEST_OBJS) $(LIBDIR)/$(CONFIG)/l
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/mpscq_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/mpscq_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_mpscq_test: $(GPR_MPSCQ_TEST_OBJS:.o=.dep)
 
@@ -10113,7 +10113,7 @@ endif
 
 
 GPR_SPINLOCK_TEST_SRC = \
-    test/core/support/spinlock_test.cc \
+    test/core/gpr/spinlock_test.cc \
 
 GPR_SPINLOCK_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_SPINLOCK_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10133,7 +10133,7 @@ $(BINDIR)/$(CONFIG)/gpr_spinlock_test: $(GPR_SPINLOCK_TEST_OBJS) $(LIBDIR)/$(CON
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/spinlock_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/spinlock_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_spinlock_test: $(GPR_SPINLOCK_TEST_OBJS:.o=.dep)
 
@@ -10145,7 +10145,7 @@ endif
 
 
 GPR_STRING_TEST_SRC = \
-    test/core/support/string_test.cc \
+    test/core/gpr/string_test.cc \
 
 GPR_STRING_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_STRING_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10165,7 +10165,7 @@ $(BINDIR)/$(CONFIG)/gpr_string_test: $(GPR_STRING_TEST_OBJS) $(LIBDIR)/$(CONFIG)
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/string_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/string_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_string_test: $(GPR_STRING_TEST_OBJS:.o=.dep)
 
@@ -10177,7 +10177,7 @@ endif
 
 
 GPR_SYNC_TEST_SRC = \
-    test/core/support/sync_test.cc \
+    test/core/gpr/sync_test.cc \
 
 GPR_SYNC_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_SYNC_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10197,7 +10197,7 @@ $(BINDIR)/$(CONFIG)/gpr_sync_test: $(GPR_SYNC_TEST_OBJS) $(LIBDIR)/$(CONFIG)/lib
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/sync_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/sync_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_sync_test: $(GPR_SYNC_TEST_OBJS:.o=.dep)
 
@@ -10209,7 +10209,7 @@ endif
 
 
 GPR_THD_TEST_SRC = \
-    test/core/support/thd_test.cc \
+    test/core/gpr/thd_test.cc \
 
 GPR_THD_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_THD_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10229,7 +10229,7 @@ $(BINDIR)/$(CONFIG)/gpr_thd_test: $(GPR_THD_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgp
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/thd_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/thd_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_thd_test: $(GPR_THD_TEST_OBJS:.o=.dep)
 
@@ -10241,7 +10241,7 @@ endif
 
 
 GPR_TIME_TEST_SRC = \
-    test/core/support/time_test.cc \
+    test/core/gpr/time_test.cc \
 
 GPR_TIME_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_TIME_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10261,7 +10261,7 @@ $(BINDIR)/$(CONFIG)/gpr_time_test: $(GPR_TIME_TEST_OBJS) $(LIBDIR)/$(CONFIG)/lib
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/time_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/time_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_time_test: $(GPR_TIME_TEST_OBJS:.o=.dep)
 
@@ -10273,7 +10273,7 @@ endif
 
 
 GPR_TLS_TEST_SRC = \
-    test/core/support/tls_test.cc \
+    test/core/gpr/tls_test.cc \
 
 GPR_TLS_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_TLS_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10293,7 +10293,7 @@ $(BINDIR)/$(CONFIG)/gpr_tls_test: $(GPR_TLS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgp
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/tls_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/tls_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_tls_test: $(GPR_TLS_TEST_OBJS:.o=.dep)
 
@@ -10305,7 +10305,7 @@ endif
 
 
 GPR_USEFUL_TEST_SRC = \
-    test/core/support/useful_test.cc \
+    test/core/gpr/useful_test.cc \
 
 GPR_USEFUL_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_USEFUL_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10325,7 +10325,7 @@ $(BINDIR)/$(CONFIG)/gpr_useful_test: $(GPR_USEFUL_TEST_OBJS) $(LIBDIR)/$(CONFIG)
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/useful_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/useful_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_useful_test: $(GPR_USEFUL_TEST_OBJS:.o=.dep)
 
@@ -11879,7 +11879,7 @@ endif
 
 
 MURMUR_HASH_TEST_SRC = \
-    test/core/support/murmur_hash_test.cc \
+    test/core/gpr/murmur_hash_test.cc \
 
 MURMUR_HASH_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(MURMUR_HASH_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -11899,7 +11899,7 @@ $(BINDIR)/$(CONFIG)/murmur_hash_test: $(MURMUR_HASH_TEST_OBJS) $(LIBDIR)/$(CONFI
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/murmur_hash_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr/murmur_hash_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_murmur_hash_test: $(MURMUR_HASH_TEST_OBJS:.o=.dep)
 
@@ -15778,6 +15778,49 @@ endif
 endif
 
 
+INLINED_VECTOR_TEST_SRC = \
+    test/core/gpr++/inlined_vector_test.cc \
+
+INLINED_VECTOR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(INLINED_VECTOR_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/inlined_vector_test: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
+
+$(BINDIR)/$(CONFIG)/inlined_vector_test: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/inlined_vector_test: $(PROTOBUF_DEP) $(INLINED_VECTOR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(INLINED_VECTOR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/inlined_vector_test
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/gpr++/inlined_vector_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_inlined_vector_test: $(INLINED_VECTOR_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(INLINED_VECTOR_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 INPROC_SYNC_UNARY_PING_PONG_TEST_SRC = \
     test/cpp/qps/inproc_sync_unary_ping_pong_test.cc \
 
@@ -15970,7 +16013,7 @@ endif
 
 
 MEMORY_TEST_SRC = \
-    test/core/support/memory_test.cc \
+    test/core/gpr++/memory_test.cc \
 
 MEMORY_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(MEMORY_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -16001,7 +16044,7 @@ endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/memory_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr++/memory_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_memory_test: $(MEMORY_TEST_OBJS:.o=.dep)
 
@@ -16147,7 +16190,7 @@ endif
 
 
 ORPHANABLE_TEST_SRC = \
-    test/core/support/orphanable_test.cc \
+    test/core/gpr++/orphanable_test.cc \
 
 ORPHANABLE_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(ORPHANABLE_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -16178,7 +16221,7 @@ endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/orphanable_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr++/orphanable_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_orphanable_test: $(ORPHANABLE_TEST_OBJS:.o=.dep)
 
@@ -16554,7 +16597,7 @@ $(OBJDIR)/$(CONFIG)/test/cpp/interop/reconnect_interop_server.o: $(GENDIR)/src/p
 
 
 REF_COUNTED_PTR_TEST_SRC = \
-    test/core/support/ref_counted_ptr_test.cc \
+    test/core/gpr++/ref_counted_ptr_test.cc \
 
 REF_COUNTED_PTR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(REF_COUNTED_PTR_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -16585,7 +16628,7 @@ endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/ref_counted_ptr_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr++/ref_counted_ptr_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_ref_counted_ptr_test: $(REF_COUNTED_PTR_TEST_OBJS:.o=.dep)
 
@@ -16597,7 +16640,7 @@ endif
 
 
 REF_COUNTED_TEST_SRC = \
-    test/core/support/ref_counted_test.cc \
+    test/core/gpr++/ref_counted_test.cc \
 
 REF_COUNTED_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(REF_COUNTED_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -16628,7 +16671,7 @@ endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/support/ref_counted_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gpr++/ref_counted_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_ref_counted_test: $(REF_COUNTED_TEST_OBJS:.o=.dep)
 
@@ -17366,49 +17409,6 @@ endif
 endif
 
 
-VECTOR_TEST_SRC = \
-    test/core/support/vector_test.cc \
-
-VECTOR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(VECTOR_TEST_SRC))))
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure targets if you don't have OpenSSL.
-
-$(BINDIR)/$(CONFIG)/vector_test: openssl_dep_error
-
-else
-
-
-
-
-ifeq ($(NO_PROTOBUF),true)
-
-# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
-
-$(BINDIR)/$(CONFIG)/vector_test: protobuf_dep_error
-
-else
-
-$(BINDIR)/$(CONFIG)/vector_test: $(PROTOBUF_DEP) $(VECTOR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-	$(E) "[LD]      Linking $@"
-	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(VECTOR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/vector_test
-
-endif
-
-endif
-
-$(OBJDIR)/$(CONFIG)/test/core/support/vector_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-
-deps_vector_test: $(VECTOR_TEST_OBJS:.o=.dep)
-
-ifneq ($(NO_SECURE),true)
-ifneq ($(NO_DEPS),true)
--include $(VECTOR_TEST_OBJS:.o=.dep)
-endif
-endif
-
-
 WRITES_PER_RPC_TEST_SRC = \
     test/cpp/performance/writes_per_rpc_test.cc \
 
diff --git a/build.yaml b/build.yaml
index c9adc9b44e7..a8639a8b0b8 100644
--- a/build.yaml
+++ b/build.yaml
@@ -26,50 +26,50 @@ filegroups:
   - nanopb
 - name: gpr_base
   src:
+  - src/core/lib/gpr/alloc.cc
+  - src/core/lib/gpr/arena.cc
+  - src/core/lib/gpr/atm.cc
+  - src/core/lib/gpr/avl.cc
+  - src/core/lib/gpr/cmdline.cc
+  - src/core/lib/gpr/cpu_iphone.cc
+  - src/core/lib/gpr/cpu_linux.cc
+  - src/core/lib/gpr/cpu_posix.cc
+  - src/core/lib/gpr/cpu_windows.cc
+  - src/core/lib/gpr/env_linux.cc
+  - src/core/lib/gpr/env_posix.cc
+  - src/core/lib/gpr/env_windows.cc
+  - src/core/lib/gpr/fork.cc
+  - src/core/lib/gpr/host_port.cc
+  - src/core/lib/gpr/log.cc
+  - src/core/lib/gpr/log_android.cc
+  - src/core/lib/gpr/log_linux.cc
+  - src/core/lib/gpr/log_posix.cc
+  - src/core/lib/gpr/log_windows.cc
+  - src/core/lib/gpr/mpscq.cc
+  - src/core/lib/gpr/murmur_hash.cc
+  - src/core/lib/gpr/string.cc
+  - src/core/lib/gpr/string_posix.cc
+  - src/core/lib/gpr/string_util_windows.cc
+  - src/core/lib/gpr/string_windows.cc
+  - src/core/lib/gpr/subprocess_posix.cc
+  - src/core/lib/gpr/subprocess_windows.cc
+  - src/core/lib/gpr/sync.cc
+  - src/core/lib/gpr/sync_posix.cc
+  - src/core/lib/gpr/sync_windows.cc
+  - src/core/lib/gpr/thd.cc
+  - src/core/lib/gpr/thd_posix.cc
+  - src/core/lib/gpr/thd_windows.cc
+  - src/core/lib/gpr/time.cc
+  - src/core/lib/gpr/time_posix.cc
+  - src/core/lib/gpr/time_precise.cc
+  - src/core/lib/gpr/time_windows.cc
+  - src/core/lib/gpr/tls_pthread.cc
+  - src/core/lib/gpr/tmpfile_msys.cc
+  - src/core/lib/gpr/tmpfile_posix.cc
+  - src/core/lib/gpr/tmpfile_windows.cc
+  - src/core/lib/gpr/wrap_memcpy.cc
   - src/core/lib/profiling/basic_timers.cc
   - src/core/lib/profiling/stap_timers.cc
-  - src/core/lib/support/alloc.cc
-  - src/core/lib/support/arena.cc
-  - src/core/lib/support/atm.cc
-  - src/core/lib/support/avl.cc
-  - src/core/lib/support/cmdline.cc
-  - src/core/lib/support/cpu_iphone.cc
-  - src/core/lib/support/cpu_linux.cc
-  - src/core/lib/support/cpu_posix.cc
-  - src/core/lib/support/cpu_windows.cc
-  - src/core/lib/support/env_linux.cc
-  - src/core/lib/support/env_posix.cc
-  - src/core/lib/support/env_windows.cc
-  - src/core/lib/support/fork.cc
-  - src/core/lib/support/host_port.cc
-  - src/core/lib/support/log.cc
-  - src/core/lib/support/log_android.cc
-  - src/core/lib/support/log_linux.cc
-  - src/core/lib/support/log_posix.cc
-  - src/core/lib/support/log_windows.cc
-  - src/core/lib/support/mpscq.cc
-  - src/core/lib/support/murmur_hash.cc
-  - src/core/lib/support/string.cc
-  - src/core/lib/support/string_posix.cc
-  - src/core/lib/support/string_util_windows.cc
-  - src/core/lib/support/string_windows.cc
-  - src/core/lib/support/subprocess_posix.cc
-  - src/core/lib/support/subprocess_windows.cc
-  - src/core/lib/support/sync.cc
-  - src/core/lib/support/sync_posix.cc
-  - src/core/lib/support/sync_windows.cc
-  - src/core/lib/support/thd.cc
-  - src/core/lib/support/thd_posix.cc
-  - src/core/lib/support/thd_windows.cc
-  - src/core/lib/support/time.cc
-  - src/core/lib/support/time_posix.cc
-  - src/core/lib/support/time_precise.cc
-  - src/core/lib/support/time_windows.cc
-  - src/core/lib/support/tls_pthread.cc
-  - src/core/lib/support/tmpfile_msys.cc
-  - src/core/lib/support/tmpfile_posix.cc
-  - src/core/lib/support/tmpfile_windows.cc
-  - src/core/lib/support/wrap_memcpy.cc
   uses:
   - gpr_base_headers
 - name: gpr_base_headers
@@ -101,24 +101,24 @@ filegroups:
   - include/grpc/support/tls_pthread.h
   - include/grpc/support/useful.h
   headers:
+  - src/core/lib/gpr++/abstract.h
+  - src/core/lib/gpr++/atomic.h
+  - src/core/lib/gpr++/atomic_with_atm.h
+  - src/core/lib/gpr++/atomic_with_std.h
+  - src/core/lib/gpr++/manual_constructor.h
+  - src/core/lib/gpr++/memory.h
+  - src/core/lib/gpr/arena.h
+  - src/core/lib/gpr/env.h
+  - src/core/lib/gpr/fork.h
+  - src/core/lib/gpr/mpscq.h
+  - src/core/lib/gpr/murmur_hash.h
+  - src/core/lib/gpr/spinlock.h
+  - src/core/lib/gpr/string.h
+  - src/core/lib/gpr/string_windows.h
+  - src/core/lib/gpr/thd_internal.h
+  - src/core/lib/gpr/time_precise.h
+  - src/core/lib/gpr/tmpfile.h
   - src/core/lib/profiling/timers.h
-  - src/core/lib/support/abstract.h
-  - src/core/lib/support/arena.h
-  - src/core/lib/support/atomic.h
-  - src/core/lib/support/atomic_with_atm.h
-  - src/core/lib/support/atomic_with_std.h
-  - src/core/lib/support/env.h
-  - src/core/lib/support/fork.h
-  - src/core/lib/support/manual_constructor.h
-  - src/core/lib/support/memory.h
-  - src/core/lib/support/mpscq.h
-  - src/core/lib/support/murmur_hash.h
-  - src/core/lib/support/spinlock.h
-  - src/core/lib/support/string.h
-  - src/core/lib/support/string_windows.h
-  - src/core/lib/support/thd_internal.h
-  - src/core/lib/support/time_precise.h
-  - src/core/lib/support/tmpfile.h
   uses:
   - gpr_codegen
 - name: gpr_codegen
@@ -321,6 +321,11 @@ filegroups:
   - src/core/lib/compression/stream_compression_identity.h
   - src/core/lib/debug/stats.h
   - src/core/lib/debug/stats_data.h
+  - src/core/lib/gpr++/debug_location.h
+  - src/core/lib/gpr++/inlined_vector.h
+  - src/core/lib/gpr++/orphanable.h
+  - src/core/lib/gpr++/ref_counted.h
+  - src/core/lib/gpr++/ref_counted_ptr.h
   - src/core/lib/http/format_request.h
   - src/core/lib/http/httpcli.h
   - src/core/lib/http/parser.h
@@ -396,11 +401,6 @@ filegroups:
   - src/core/lib/slice/slice_hash_table.h
   - src/core/lib/slice/slice_internal.h
   - src/core/lib/slice/slice_string_helpers.h
-  - src/core/lib/support/debug_location.h
-  - src/core/lib/support/orphanable.h
-  - src/core/lib/support/ref_counted.h
-  - src/core/lib/support/ref_counted_ptr.h
-  - src/core/lib/support/vector.h
   - src/core/lib/surface/alarm_internal.h
   - src/core/lib/surface/api_trace.h
   - src/core/lib/surface/call.h
@@ -1732,7 +1732,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/alloc_test.cc
+  - test/core/gpr/alloc_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -1765,7 +1765,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/arena_test.cc
+  - test/core/gpr/arena_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2132,7 +2132,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/avl_test.cc
+  - test/core/gpr/avl_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2141,7 +2141,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/cmdline_test.cc
+  - test/core/gpr/cmdline_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2151,7 +2151,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/cpu_test.cc
+  - test/core/gpr/cpu_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2160,7 +2160,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/env_test.cc
+  - test/core/gpr/env_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2169,7 +2169,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/host_port_test.cc
+  - test/core/gpr/host_port_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2178,7 +2178,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/log_test.cc
+  - test/core/gpr/log_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2188,7 +2188,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/manual_constructor_test.cc
+  - test/core/gpr++/manual_constructor_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2198,7 +2198,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/mpscq_test.cc
+  - test/core/gpr/mpscq_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2208,7 +2208,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/spinlock_test.cc
+  - test/core/gpr/spinlock_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2217,7 +2217,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/string_test.cc
+  - test/core/gpr/string_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2227,7 +2227,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/sync_test.cc
+  - test/core/gpr/sync_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2237,7 +2237,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/thd_test.cc
+  - test/core/gpr/thd_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2246,7 +2246,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/time_test.cc
+  - test/core/gpr/time_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2255,7 +2255,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/tls_test.cc
+  - test/core/gpr/tls_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2264,7 +2264,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/useful_test.cc
+  - test/core/gpr/useful_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -2833,7 +2833,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/support/murmur_hash_test.cc
+  - test/core/gpr/murmur_hash_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -4260,6 +4260,20 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
+- name: inlined_vector_test
+  gtest: true
+  build: test
+  language: c++
+  src:
+  - test/core/gpr++/inlined_vector_test.cc
+  deps:
+  - grpc_test_util
+  - grpc++
+  - grpc
+  - gpr_test_util
+  - gpr
+  uses:
+  - grpc++_test
 - name: inproc_sync_unary_ping_pong_test
   build: test
   language: c++
@@ -4357,7 +4371,7 @@ targets:
   build: test
   language: c++
   src:
-  - test/core/support/memory_test.cc
+  - test/core/gpr++/memory_test.cc
   deps:
   - grpc_test_util
   - grpc++
@@ -4409,7 +4423,7 @@ targets:
   build: test
   language: c++
   src:
-  - test/core/support/orphanable_test.cc
+  - test/core/gpr++/orphanable_test.cc
   deps:
   - grpc_test_util
   - grpc++
@@ -4562,7 +4576,7 @@ targets:
   build: test
   language: c++
   src:
-  - test/core/support/ref_counted_ptr_test.cc
+  - test/core/gpr++/ref_counted_ptr_test.cc
   deps:
   - grpc_test_util
   - grpc++
@@ -4576,7 +4590,7 @@ targets:
   build: test
   language: c++
   src:
-  - test/core/support/ref_counted_test.cc
+  - test/core/gpr++/ref_counted_test.cc
   deps:
   - grpc_test_util
   - grpc++
@@ -4826,20 +4840,6 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
-- name: vector_test
-  gtest: true
-  build: test
-  language: c++
-  src:
-  - test/core/support/vector_test.cc
-  deps:
-  - grpc_test_util
-  - grpc++
-  - grpc
-  - gpr_test_util
-  - gpr
-  uses:
-  - grpc++_test
 - name: writes_per_rpc_test
   gtest: true
   cpu_cost: 0.5
diff --git a/config.m4 b/config.m4
index c026b83f359..54e29cc41de 100644
--- a/config.m4
+++ b/config.m4
@@ -39,50 +39,50 @@ if test "$PHP_GRPC" != "no"; then
     src/php/ext/grpc/server.c \
     src/php/ext/grpc/server_credentials.c \
     src/php/ext/grpc/timeval.c \
+    src/core/lib/gpr/alloc.cc \
+    src/core/lib/gpr/arena.cc \
+    src/core/lib/gpr/atm.cc \
+    src/core/lib/gpr/avl.cc \
+    src/core/lib/gpr/cmdline.cc \
+    src/core/lib/gpr/cpu_iphone.cc \
+    src/core/lib/gpr/cpu_linux.cc \
+    src/core/lib/gpr/cpu_posix.cc \
+    src/core/lib/gpr/cpu_windows.cc \
+    src/core/lib/gpr/env_linux.cc \
+    src/core/lib/gpr/env_posix.cc \
+    src/core/lib/gpr/env_windows.cc \
+    src/core/lib/gpr/fork.cc \
+    src/core/lib/gpr/host_port.cc \
+    src/core/lib/gpr/log.cc \
+    src/core/lib/gpr/log_android.cc \
+    src/core/lib/gpr/log_linux.cc \
+    src/core/lib/gpr/log_posix.cc \
+    src/core/lib/gpr/log_windows.cc \
+    src/core/lib/gpr/mpscq.cc \
+    src/core/lib/gpr/murmur_hash.cc \
+    src/core/lib/gpr/string.cc \
+    src/core/lib/gpr/string_posix.cc \
+    src/core/lib/gpr/string_util_windows.cc \
+    src/core/lib/gpr/string_windows.cc \
+    src/core/lib/gpr/subprocess_posix.cc \
+    src/core/lib/gpr/subprocess_windows.cc \
+    src/core/lib/gpr/sync.cc \
+    src/core/lib/gpr/sync_posix.cc \
+    src/core/lib/gpr/sync_windows.cc \
+    src/core/lib/gpr/thd.cc \
+    src/core/lib/gpr/thd_posix.cc \
+    src/core/lib/gpr/thd_windows.cc \
+    src/core/lib/gpr/time.cc \
+    src/core/lib/gpr/time_posix.cc \
+    src/core/lib/gpr/time_precise.cc \
+    src/core/lib/gpr/time_windows.cc \
+    src/core/lib/gpr/tls_pthread.cc \
+    src/core/lib/gpr/tmpfile_msys.cc \
+    src/core/lib/gpr/tmpfile_posix.cc \
+    src/core/lib/gpr/tmpfile_windows.cc \
+    src/core/lib/gpr/wrap_memcpy.cc \
     src/core/lib/profiling/basic_timers.cc \
     src/core/lib/profiling/stap_timers.cc \
-    src/core/lib/support/alloc.cc \
-    src/core/lib/support/arena.cc \
-    src/core/lib/support/atm.cc \
-    src/core/lib/support/avl.cc \
-    src/core/lib/support/cmdline.cc \
-    src/core/lib/support/cpu_iphone.cc \
-    src/core/lib/support/cpu_linux.cc \
-    src/core/lib/support/cpu_posix.cc \
-    src/core/lib/support/cpu_windows.cc \
-    src/core/lib/support/env_linux.cc \
-    src/core/lib/support/env_posix.cc \
-    src/core/lib/support/env_windows.cc \
-    src/core/lib/support/fork.cc \
-    src/core/lib/support/host_port.cc \
-    src/core/lib/support/log.cc \
-    src/core/lib/support/log_android.cc \
-    src/core/lib/support/log_linux.cc \
-    src/core/lib/support/log_posix.cc \
-    src/core/lib/support/log_windows.cc \
-    src/core/lib/support/mpscq.cc \
-    src/core/lib/support/murmur_hash.cc \
-    src/core/lib/support/string.cc \
-    src/core/lib/support/string_posix.cc \
-    src/core/lib/support/string_util_windows.cc \
-    src/core/lib/support/string_windows.cc \
-    src/core/lib/support/subprocess_posix.cc \
-    src/core/lib/support/subprocess_windows.cc \
-    src/core/lib/support/sync.cc \
-    src/core/lib/support/sync_posix.cc \
-    src/core/lib/support/sync_windows.cc \
-    src/core/lib/support/thd.cc \
-    src/core/lib/support/thd_posix.cc \
-    src/core/lib/support/thd_windows.cc \
-    src/core/lib/support/time.cc \
-    src/core/lib/support/time_posix.cc \
-    src/core/lib/support/time_precise.cc \
-    src/core/lib/support/time_windows.cc \
-    src/core/lib/support/tls_pthread.cc \
-    src/core/lib/support/tmpfile_msys.cc \
-    src/core/lib/support/tmpfile_posix.cc \
-    src/core/lib/support/tmpfile_windows.cc \
-    src/core/lib/support/wrap_memcpy.cc \
     src/core/lib/surface/init.cc \
     src/core/lib/backoff/backoff.cc \
     src/core/lib/channel/channel_args.cc \
@@ -678,6 +678,7 @@ if test "$PHP_GRPC" != "no"; then
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/channel)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/compression)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/debug)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/gpr)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/http)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/iomgr)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/json)
@@ -695,7 +696,6 @@ if test "$PHP_GRPC" != "no"; then
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/transport)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/util)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/slice)
-  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/support)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/surface)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/transport)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/plugin_registry)
diff --git a/config.w32 b/config.w32
index cd3a16a4653..c8003140d80 100644
--- a/config.w32
+++ b/config.w32
@@ -16,50 +16,50 @@ if (PHP_GRPC != "no") {
     "src\\php\\ext\\grpc\\server.c " +
     "src\\php\\ext\\grpc\\server_credentials.c " +
     "src\\php\\ext\\grpc\\timeval.c " +
+    "src\\core\\lib\\gpr\\alloc.cc " +
+    "src\\core\\lib\\gpr\\arena.cc " +
+    "src\\core\\lib\\gpr\\atm.cc " +
+    "src\\core\\lib\\gpr\\avl.cc " +
+    "src\\core\\lib\\gpr\\cmdline.cc " +
+    "src\\core\\lib\\gpr\\cpu_iphone.cc " +
+    "src\\core\\lib\\gpr\\cpu_linux.cc " +
+    "src\\core\\lib\\gpr\\cpu_posix.cc " +
+    "src\\core\\lib\\gpr\\cpu_windows.cc " +
+    "src\\core\\lib\\gpr\\env_linux.cc " +
+    "src\\core\\lib\\gpr\\env_posix.cc " +
+    "src\\core\\lib\\gpr\\env_windows.cc " +
+    "src\\core\\lib\\gpr\\fork.cc " +
+    "src\\core\\lib\\gpr\\host_port.cc " +
+    "src\\core\\lib\\gpr\\log.cc " +
+    "src\\core\\lib\\gpr\\log_android.cc " +
+    "src\\core\\lib\\gpr\\log_linux.cc " +
+    "src\\core\\lib\\gpr\\log_posix.cc " +
+    "src\\core\\lib\\gpr\\log_windows.cc " +
+    "src\\core\\lib\\gpr\\mpscq.cc " +
+    "src\\core\\lib\\gpr\\murmur_hash.cc " +
+    "src\\core\\lib\\gpr\\string.cc " +
+    "src\\core\\lib\\gpr\\string_posix.cc " +
+    "src\\core\\lib\\gpr\\string_util_windows.cc " +
+    "src\\core\\lib\\gpr\\string_windows.cc " +
+    "src\\core\\lib\\gpr\\subprocess_posix.cc " +
+    "src\\core\\lib\\gpr\\subprocess_windows.cc " +
+    "src\\core\\lib\\gpr\\sync.cc " +
+    "src\\core\\lib\\gpr\\sync_posix.cc " +
+    "src\\core\\lib\\gpr\\sync_windows.cc " +
+    "src\\core\\lib\\gpr\\thd.cc " +
+    "src\\core\\lib\\gpr\\thd_posix.cc " +
+    "src\\core\\lib\\gpr\\thd_windows.cc " +
+    "src\\core\\lib\\gpr\\time.cc " +
+    "src\\core\\lib\\gpr\\time_posix.cc " +
+    "src\\core\\lib\\gpr\\time_precise.cc " +
+    "src\\core\\lib\\gpr\\time_windows.cc " +
+    "src\\core\\lib\\gpr\\tls_pthread.cc " +
+    "src\\core\\lib\\gpr\\tmpfile_msys.cc " +
+    "src\\core\\lib\\gpr\\tmpfile_posix.cc " +
+    "src\\core\\lib\\gpr\\tmpfile_windows.cc " +
+    "src\\core\\lib\\gpr\\wrap_memcpy.cc " +
     "src\\core\\lib\\profiling\\basic_timers.cc " +
     "src\\core\\lib\\profiling\\stap_timers.cc " +
-    "src\\core\\lib\\support\\alloc.cc " +
-    "src\\core\\lib\\support\\arena.cc " +
-    "src\\core\\lib\\support\\atm.cc " +
-    "src\\core\\lib\\support\\avl.cc " +
-    "src\\core\\lib\\support\\cmdline.cc " +
-    "src\\core\\lib\\support\\cpu_iphone.cc " +
-    "src\\core\\lib\\support\\cpu_linux.cc " +
-    "src\\core\\lib\\support\\cpu_posix.cc " +
-    "src\\core\\lib\\support\\cpu_windows.cc " +
-    "src\\core\\lib\\support\\env_linux.cc " +
-    "src\\core\\lib\\support\\env_posix.cc " +
-    "src\\core\\lib\\support\\env_windows.cc " +
-    "src\\core\\lib\\support\\fork.cc " +
-    "src\\core\\lib\\support\\host_port.cc " +
-    "src\\core\\lib\\support\\log.cc " +
-    "src\\core\\lib\\support\\log_android.cc " +
-    "src\\core\\lib\\support\\log_linux.cc " +
-    "src\\core\\lib\\support\\log_posix.cc " +
-    "src\\core\\lib\\support\\log_windows.cc " +
-    "src\\core\\lib\\support\\mpscq.cc " +
-    "src\\core\\lib\\support\\murmur_hash.cc " +
-    "src\\core\\lib\\support\\string.cc " +
-    "src\\core\\lib\\support\\string_posix.cc " +
-    "src\\core\\lib\\support\\string_util_windows.cc " +
-    "src\\core\\lib\\support\\string_windows.cc " +
-    "src\\core\\lib\\support\\subprocess_posix.cc " +
-    "src\\core\\lib\\support\\subprocess_windows.cc " +
-    "src\\core\\lib\\support\\sync.cc " +
-    "src\\core\\lib\\support\\sync_posix.cc " +
-    "src\\core\\lib\\support\\sync_windows.cc " +
-    "src\\core\\lib\\support\\thd.cc " +
-    "src\\core\\lib\\support\\thd_posix.cc " +
-    "src\\core\\lib\\support\\thd_windows.cc " +
-    "src\\core\\lib\\support\\time.cc " +
-    "src\\core\\lib\\support\\time_posix.cc " +
-    "src\\core\\lib\\support\\time_precise.cc " +
-    "src\\core\\lib\\support\\time_windows.cc " +
-    "src\\core\\lib\\support\\tls_pthread.cc " +
-    "src\\core\\lib\\support\\tmpfile_msys.cc " +
-    "src\\core\\lib\\support\\tmpfile_posix.cc " +
-    "src\\core\\lib\\support\\tmpfile_windows.cc " +
-    "src\\core\\lib\\support\\wrap_memcpy.cc " +
     "src\\core\\lib\\surface\\init.cc " +
     "src\\core\\lib\\backoff\\backoff.cc " +
     "src\\core\\lib\\channel\\channel_args.cc " +
@@ -690,6 +690,7 @@ if (PHP_GRPC != "no") {
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\channel");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\compression");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\debug");
+  FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\gpr");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\http");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\iomgr");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\json");
@@ -708,7 +709,6 @@ if (PHP_GRPC != "no") {
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\security\\transport");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\security\\util");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\slice");
-  FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\support");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\surface");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\lib\\transport");
   FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\plugin_registry");
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index df5a4112a75..ccaa21194b8 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -84,7 +84,7 @@ Pod::Spec.new do |s|
     'HEADER_SEARCH_PATHS' => '"$(inherited)" "$(GRPC_SRC_ROOT)/include"',
     'USER_HEADER_SEARCH_PATHS' => '"$(GRPC_SRC_ROOT)"',
     # If we don't set these two settings, `include/grpc/support/time.h` and
-    # `src/core/lib/support/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
+    # `src/core/lib/gpr/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
     # build.
     'USE_HEADERMAP' => 'NO',
     'ALWAYS_SEARCH_USER_PATHS' => 'NO',
@@ -192,68 +192,68 @@ Pod::Spec.new do |s|
     ss.dependency 'nanopb', '~> 0.3'
 
     # To save you from scrolling, this is the last part of the podspec.
-    ss.source_files = 'src/core/lib/profiling/timers.h',
-                      'src/core/lib/support/abstract.h',
-                      'src/core/lib/support/arena.h',
-                      'src/core/lib/support/atomic.h',
-                      'src/core/lib/support/atomic_with_atm.h',
-                      'src/core/lib/support/atomic_with_std.h',
-                      'src/core/lib/support/env.h',
-                      'src/core/lib/support/fork.h',
-                      'src/core/lib/support/manual_constructor.h',
-                      'src/core/lib/support/memory.h',
-                      'src/core/lib/support/mpscq.h',
-                      'src/core/lib/support/murmur_hash.h',
-                      'src/core/lib/support/spinlock.h',
-                      'src/core/lib/support/string.h',
-                      'src/core/lib/support/string_windows.h',
-                      'src/core/lib/support/thd_internal.h',
-                      'src/core/lib/support/time_precise.h',
-                      'src/core/lib/support/tmpfile.h',
+    ss.source_files = 'src/core/lib/gpr++/abstract.h',
+                      'src/core/lib/gpr++/atomic.h',
+                      'src/core/lib/gpr++/atomic_with_atm.h',
+                      'src/core/lib/gpr++/atomic_with_std.h',
+                      'src/core/lib/gpr++/manual_constructor.h',
+                      'src/core/lib/gpr++/memory.h',
+                      'src/core/lib/gpr/arena.h',
+                      'src/core/lib/gpr/env.h',
+                      'src/core/lib/gpr/fork.h',
+                      'src/core/lib/gpr/mpscq.h',
+                      'src/core/lib/gpr/murmur_hash.h',
+                      'src/core/lib/gpr/spinlock.h',
+                      'src/core/lib/gpr/string.h',
+                      'src/core/lib/gpr/string_windows.h',
+                      'src/core/lib/gpr/thd_internal.h',
+                      'src/core/lib/gpr/time_precise.h',
+                      'src/core/lib/gpr/tmpfile.h',
+                      'src/core/lib/profiling/timers.h',
+                      'src/core/lib/gpr/alloc.cc',
+                      'src/core/lib/gpr/arena.cc',
+                      'src/core/lib/gpr/atm.cc',
+                      'src/core/lib/gpr/avl.cc',
+                      'src/core/lib/gpr/cmdline.cc',
+                      'src/core/lib/gpr/cpu_iphone.cc',
+                      'src/core/lib/gpr/cpu_linux.cc',
+                      'src/core/lib/gpr/cpu_posix.cc',
+                      'src/core/lib/gpr/cpu_windows.cc',
+                      'src/core/lib/gpr/env_linux.cc',
+                      'src/core/lib/gpr/env_posix.cc',
+                      'src/core/lib/gpr/env_windows.cc',
+                      'src/core/lib/gpr/fork.cc',
+                      'src/core/lib/gpr/host_port.cc',
+                      'src/core/lib/gpr/log.cc',
+                      'src/core/lib/gpr/log_android.cc',
+                      'src/core/lib/gpr/log_linux.cc',
+                      'src/core/lib/gpr/log_posix.cc',
+                      'src/core/lib/gpr/log_windows.cc',
+                      'src/core/lib/gpr/mpscq.cc',
+                      'src/core/lib/gpr/murmur_hash.cc',
+                      'src/core/lib/gpr/string.cc',
+                      'src/core/lib/gpr/string_posix.cc',
+                      'src/core/lib/gpr/string_util_windows.cc',
+                      'src/core/lib/gpr/string_windows.cc',
+                      'src/core/lib/gpr/subprocess_posix.cc',
+                      'src/core/lib/gpr/subprocess_windows.cc',
+                      'src/core/lib/gpr/sync.cc',
+                      'src/core/lib/gpr/sync_posix.cc',
+                      'src/core/lib/gpr/sync_windows.cc',
+                      'src/core/lib/gpr/thd.cc',
+                      'src/core/lib/gpr/thd_posix.cc',
+                      'src/core/lib/gpr/thd_windows.cc',
+                      'src/core/lib/gpr/time.cc',
+                      'src/core/lib/gpr/time_posix.cc',
+                      'src/core/lib/gpr/time_precise.cc',
+                      'src/core/lib/gpr/time_windows.cc',
+                      'src/core/lib/gpr/tls_pthread.cc',
+                      'src/core/lib/gpr/tmpfile_msys.cc',
+                      'src/core/lib/gpr/tmpfile_posix.cc',
+                      'src/core/lib/gpr/tmpfile_windows.cc',
+                      'src/core/lib/gpr/wrap_memcpy.cc',
                       'src/core/lib/profiling/basic_timers.cc',
                       'src/core/lib/profiling/stap_timers.cc',
-                      'src/core/lib/support/alloc.cc',
-                      'src/core/lib/support/arena.cc',
-                      'src/core/lib/support/atm.cc',
-                      'src/core/lib/support/avl.cc',
-                      'src/core/lib/support/cmdline.cc',
-                      'src/core/lib/support/cpu_iphone.cc',
-                      'src/core/lib/support/cpu_linux.cc',
-                      'src/core/lib/support/cpu_posix.cc',
-                      'src/core/lib/support/cpu_windows.cc',
-                      'src/core/lib/support/env_linux.cc',
-                      'src/core/lib/support/env_posix.cc',
-                      'src/core/lib/support/env_windows.cc',
-                      'src/core/lib/support/fork.cc',
-                      'src/core/lib/support/host_port.cc',
-                      'src/core/lib/support/log.cc',
-                      'src/core/lib/support/log_android.cc',
-                      'src/core/lib/support/log_linux.cc',
-                      'src/core/lib/support/log_posix.cc',
-                      'src/core/lib/support/log_windows.cc',
-                      'src/core/lib/support/mpscq.cc',
-                      'src/core/lib/support/murmur_hash.cc',
-                      'src/core/lib/support/string.cc',
-                      'src/core/lib/support/string_posix.cc',
-                      'src/core/lib/support/string_util_windows.cc',
-                      'src/core/lib/support/string_windows.cc',
-                      'src/core/lib/support/subprocess_posix.cc',
-                      'src/core/lib/support/subprocess_windows.cc',
-                      'src/core/lib/support/sync.cc',
-                      'src/core/lib/support/sync_posix.cc',
-                      'src/core/lib/support/sync_windows.cc',
-                      'src/core/lib/support/thd.cc',
-                      'src/core/lib/support/thd_posix.cc',
-                      'src/core/lib/support/thd_windows.cc',
-                      'src/core/lib/support/time.cc',
-                      'src/core/lib/support/time_posix.cc',
-                      'src/core/lib/support/time_precise.cc',
-                      'src/core/lib/support/time_windows.cc',
-                      'src/core/lib/support/tls_pthread.cc',
-                      'src/core/lib/support/tmpfile_msys.cc',
-                      'src/core/lib/support/tmpfile_posix.cc',
-                      'src/core/lib/support/tmpfile_windows.cc',
-                      'src/core/lib/support/wrap_memcpy.cc',
                       'src/core/ext/transport/chttp2/transport/bin_decoder.h',
                       'src/core/ext/transport/chttp2/transport/bin_encoder.h',
                       'src/core/ext/transport/chttp2/transport/chttp2_transport.h',
@@ -344,6 +344,11 @@ Pod::Spec.new do |s|
                       'src/core/lib/compression/stream_compression_identity.h',
                       'src/core/lib/debug/stats.h',
                       'src/core/lib/debug/stats_data.h',
+                      'src/core/lib/gpr++/debug_location.h',
+                      'src/core/lib/gpr++/inlined_vector.h',
+                      'src/core/lib/gpr++/orphanable.h',
+                      'src/core/lib/gpr++/ref_counted.h',
+                      'src/core/lib/gpr++/ref_counted_ptr.h',
                       'src/core/lib/http/format_request.h',
                       'src/core/lib/http/httpcli.h',
                       'src/core/lib/http/parser.h',
@@ -419,11 +424,6 @@ Pod::Spec.new do |s|
                       'src/core/lib/slice/slice_hash_table.h',
                       'src/core/lib/slice/slice_internal.h',
                       'src/core/lib/slice/slice_string_helpers.h',
-                      'src/core/lib/support/debug_location.h',
-                      'src/core/lib/support/orphanable.h',
-                      'src/core/lib/support/ref_counted.h',
-                      'src/core/lib/support/ref_counted_ptr.h',
-                      'src/core/lib/support/vector.h',
                       'src/core/lib/surface/alarm_internal.h',
                       'src/core/lib/surface/api_trace.h',
                       'src/core/lib/surface/call.h',
@@ -718,24 +718,24 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/workarounds/workaround_utils.cc',
                       'src/core/plugin_registry/grpc_plugin_registry.cc'
 
-    ss.private_header_files = 'src/core/lib/profiling/timers.h',
-                              'src/core/lib/support/abstract.h',
-                              'src/core/lib/support/arena.h',
-                              'src/core/lib/support/atomic.h',
-                              'src/core/lib/support/atomic_with_atm.h',
-                              'src/core/lib/support/atomic_with_std.h',
-                              'src/core/lib/support/env.h',
-                              'src/core/lib/support/fork.h',
-                              'src/core/lib/support/manual_constructor.h',
-                              'src/core/lib/support/memory.h',
-                              'src/core/lib/support/mpscq.h',
-                              'src/core/lib/support/murmur_hash.h',
-                              'src/core/lib/support/spinlock.h',
-                              'src/core/lib/support/string.h',
-                              'src/core/lib/support/string_windows.h',
-                              'src/core/lib/support/thd_internal.h',
-                              'src/core/lib/support/time_precise.h',
-                              'src/core/lib/support/tmpfile.h',
+    ss.private_header_files = 'src/core/lib/gpr++/abstract.h',
+                              'src/core/lib/gpr++/atomic.h',
+                              'src/core/lib/gpr++/atomic_with_atm.h',
+                              'src/core/lib/gpr++/atomic_with_std.h',
+                              'src/core/lib/gpr++/manual_constructor.h',
+                              'src/core/lib/gpr++/memory.h',
+                              'src/core/lib/gpr/arena.h',
+                              'src/core/lib/gpr/env.h',
+                              'src/core/lib/gpr/fork.h',
+                              'src/core/lib/gpr/mpscq.h',
+                              'src/core/lib/gpr/murmur_hash.h',
+                              'src/core/lib/gpr/spinlock.h',
+                              'src/core/lib/gpr/string.h',
+                              'src/core/lib/gpr/string_windows.h',
+                              'src/core/lib/gpr/thd_internal.h',
+                              'src/core/lib/gpr/time_precise.h',
+                              'src/core/lib/gpr/tmpfile.h',
+                              'src/core/lib/profiling/timers.h',
                               'src/core/ext/transport/chttp2/transport/bin_decoder.h',
                               'src/core/ext/transport/chttp2/transport/bin_encoder.h',
                               'src/core/ext/transport/chttp2/transport/chttp2_transport.h',
@@ -826,6 +826,11 @@ Pod::Spec.new do |s|
                               'src/core/lib/compression/stream_compression_identity.h',
                               'src/core/lib/debug/stats.h',
                               'src/core/lib/debug/stats_data.h',
+                              'src/core/lib/gpr++/debug_location.h',
+                              'src/core/lib/gpr++/inlined_vector.h',
+                              'src/core/lib/gpr++/orphanable.h',
+                              'src/core/lib/gpr++/ref_counted.h',
+                              'src/core/lib/gpr++/ref_counted_ptr.h',
                               'src/core/lib/http/format_request.h',
                               'src/core/lib/http/httpcli.h',
                               'src/core/lib/http/parser.h',
@@ -901,11 +906,6 @@ Pod::Spec.new do |s|
                               'src/core/lib/slice/slice_hash_table.h',
                               'src/core/lib/slice/slice_internal.h',
                               'src/core/lib/slice/slice_string_helpers.h',
-                              'src/core/lib/support/debug_location.h',
-                              'src/core/lib/support/orphanable.h',
-                              'src/core/lib/support/ref_counted.h',
-                              'src/core/lib/support/ref_counted_ptr.h',
-                              'src/core/lib/support/vector.h',
                               'src/core/lib/surface/alarm_internal.h',
                               'src/core/lib/surface/api_trace.h',
                               'src/core/lib/surface/call.h',
diff --git a/grpc.gemspec b/grpc.gemspec
index f8afaa5803c..5b1aebeaaa4 100644
--- a/grpc.gemspec
+++ b/grpc.gemspec
@@ -83,68 +83,68 @@ Gem::Specification.new do |s|
   s.files += %w( include/grpc/impl/codegen/sync_generic.h )
   s.files += %w( include/grpc/impl/codegen/sync_posix.h )
   s.files += %w( include/grpc/impl/codegen/sync_windows.h )
+  s.files += %w( src/core/lib/gpr++/abstract.h )
+  s.files += %w( src/core/lib/gpr++/atomic.h )
+  s.files += %w( src/core/lib/gpr++/atomic_with_atm.h )
+  s.files += %w( src/core/lib/gpr++/atomic_with_std.h )
+  s.files += %w( src/core/lib/gpr++/manual_constructor.h )
+  s.files += %w( src/core/lib/gpr++/memory.h )
+  s.files += %w( src/core/lib/gpr/arena.h )
+  s.files += %w( src/core/lib/gpr/env.h )
+  s.files += %w( src/core/lib/gpr/fork.h )
+  s.files += %w( src/core/lib/gpr/mpscq.h )
+  s.files += %w( src/core/lib/gpr/murmur_hash.h )
+  s.files += %w( src/core/lib/gpr/spinlock.h )
+  s.files += %w( src/core/lib/gpr/string.h )
+  s.files += %w( src/core/lib/gpr/string_windows.h )
+  s.files += %w( src/core/lib/gpr/thd_internal.h )
+  s.files += %w( src/core/lib/gpr/time_precise.h )
+  s.files += %w( src/core/lib/gpr/tmpfile.h )
   s.files += %w( src/core/lib/profiling/timers.h )
-  s.files += %w( src/core/lib/support/abstract.h )
-  s.files += %w( src/core/lib/support/arena.h )
-  s.files += %w( src/core/lib/support/atomic.h )
-  s.files += %w( src/core/lib/support/atomic_with_atm.h )
-  s.files += %w( src/core/lib/support/atomic_with_std.h )
-  s.files += %w( src/core/lib/support/env.h )
-  s.files += %w( src/core/lib/support/fork.h )
-  s.files += %w( src/core/lib/support/manual_constructor.h )
-  s.files += %w( src/core/lib/support/memory.h )
-  s.files += %w( src/core/lib/support/mpscq.h )
-  s.files += %w( src/core/lib/support/murmur_hash.h )
-  s.files += %w( src/core/lib/support/spinlock.h )
-  s.files += %w( src/core/lib/support/string.h )
-  s.files += %w( src/core/lib/support/string_windows.h )
-  s.files += %w( src/core/lib/support/thd_internal.h )
-  s.files += %w( src/core/lib/support/time_precise.h )
-  s.files += %w( src/core/lib/support/tmpfile.h )
+  s.files += %w( src/core/lib/gpr/alloc.cc )
+  s.files += %w( src/core/lib/gpr/arena.cc )
+  s.files += %w( src/core/lib/gpr/atm.cc )
+  s.files += %w( src/core/lib/gpr/avl.cc )
+  s.files += %w( src/core/lib/gpr/cmdline.cc )
+  s.files += %w( src/core/lib/gpr/cpu_iphone.cc )
+  s.files += %w( src/core/lib/gpr/cpu_linux.cc )
+  s.files += %w( src/core/lib/gpr/cpu_posix.cc )
+  s.files += %w( src/core/lib/gpr/cpu_windows.cc )
+  s.files += %w( src/core/lib/gpr/env_linux.cc )
+  s.files += %w( src/core/lib/gpr/env_posix.cc )
+  s.files += %w( src/core/lib/gpr/env_windows.cc )
+  s.files += %w( src/core/lib/gpr/fork.cc )
+  s.files += %w( src/core/lib/gpr/host_port.cc )
+  s.files += %w( src/core/lib/gpr/log.cc )
+  s.files += %w( src/core/lib/gpr/log_android.cc )
+  s.files += %w( src/core/lib/gpr/log_linux.cc )
+  s.files += %w( src/core/lib/gpr/log_posix.cc )
+  s.files += %w( src/core/lib/gpr/log_windows.cc )
+  s.files += %w( src/core/lib/gpr/mpscq.cc )
+  s.files += %w( src/core/lib/gpr/murmur_hash.cc )
+  s.files += %w( src/core/lib/gpr/string.cc )
+  s.files += %w( src/core/lib/gpr/string_posix.cc )
+  s.files += %w( src/core/lib/gpr/string_util_windows.cc )
+  s.files += %w( src/core/lib/gpr/string_windows.cc )
+  s.files += %w( src/core/lib/gpr/subprocess_posix.cc )
+  s.files += %w( src/core/lib/gpr/subprocess_windows.cc )
+  s.files += %w( src/core/lib/gpr/sync.cc )
+  s.files += %w( src/core/lib/gpr/sync_posix.cc )
+  s.files += %w( src/core/lib/gpr/sync_windows.cc )
+  s.files += %w( src/core/lib/gpr/thd.cc )
+  s.files += %w( src/core/lib/gpr/thd_posix.cc )
+  s.files += %w( src/core/lib/gpr/thd_windows.cc )
+  s.files += %w( src/core/lib/gpr/time.cc )
+  s.files += %w( src/core/lib/gpr/time_posix.cc )
+  s.files += %w( src/core/lib/gpr/time_precise.cc )
+  s.files += %w( src/core/lib/gpr/time_windows.cc )
+  s.files += %w( src/core/lib/gpr/tls_pthread.cc )
+  s.files += %w( src/core/lib/gpr/tmpfile_msys.cc )
+  s.files += %w( src/core/lib/gpr/tmpfile_posix.cc )
+  s.files += %w( src/core/lib/gpr/tmpfile_windows.cc )
+  s.files += %w( src/core/lib/gpr/wrap_memcpy.cc )
   s.files += %w( src/core/lib/profiling/basic_timers.cc )
   s.files += %w( src/core/lib/profiling/stap_timers.cc )
-  s.files += %w( src/core/lib/support/alloc.cc )
-  s.files += %w( src/core/lib/support/arena.cc )
-  s.files += %w( src/core/lib/support/atm.cc )
-  s.files += %w( src/core/lib/support/avl.cc )
-  s.files += %w( src/core/lib/support/cmdline.cc )
-  s.files += %w( src/core/lib/support/cpu_iphone.cc )
-  s.files += %w( src/core/lib/support/cpu_linux.cc )
-  s.files += %w( src/core/lib/support/cpu_posix.cc )
-  s.files += %w( src/core/lib/support/cpu_windows.cc )
-  s.files += %w( src/core/lib/support/env_linux.cc )
-  s.files += %w( src/core/lib/support/env_posix.cc )
-  s.files += %w( src/core/lib/support/env_windows.cc )
-  s.files += %w( src/core/lib/support/fork.cc )
-  s.files += %w( src/core/lib/support/host_port.cc )
-  s.files += %w( src/core/lib/support/log.cc )
-  s.files += %w( src/core/lib/support/log_android.cc )
-  s.files += %w( src/core/lib/support/log_linux.cc )
-  s.files += %w( src/core/lib/support/log_posix.cc )
-  s.files += %w( src/core/lib/support/log_windows.cc )
-  s.files += %w( src/core/lib/support/mpscq.cc )
-  s.files += %w( src/core/lib/support/murmur_hash.cc )
-  s.files += %w( src/core/lib/support/string.cc )
-  s.files += %w( src/core/lib/support/string_posix.cc )
-  s.files += %w( src/core/lib/support/string_util_windows.cc )
-  s.files += %w( src/core/lib/support/string_windows.cc )
-  s.files += %w( src/core/lib/support/subprocess_posix.cc )
-  s.files += %w( src/core/lib/support/subprocess_windows.cc )
-  s.files += %w( src/core/lib/support/sync.cc )
-  s.files += %w( src/core/lib/support/sync_posix.cc )
-  s.files += %w( src/core/lib/support/sync_windows.cc )
-  s.files += %w( src/core/lib/support/thd.cc )
-  s.files += %w( src/core/lib/support/thd_posix.cc )
-  s.files += %w( src/core/lib/support/thd_windows.cc )
-  s.files += %w( src/core/lib/support/time.cc )
-  s.files += %w( src/core/lib/support/time_posix.cc )
-  s.files += %w( src/core/lib/support/time_precise.cc )
-  s.files += %w( src/core/lib/support/time_windows.cc )
-  s.files += %w( src/core/lib/support/tls_pthread.cc )
-  s.files += %w( src/core/lib/support/tmpfile_msys.cc )
-  s.files += %w( src/core/lib/support/tmpfile_posix.cc )
-  s.files += %w( src/core/lib/support/tmpfile_windows.cc )
-  s.files += %w( src/core/lib/support/wrap_memcpy.cc )
   s.files += %w( include/grpc/impl/codegen/byte_buffer.h )
   s.files += %w( include/grpc/impl/codegen/byte_buffer_reader.h )
   s.files += %w( include/grpc/impl/codegen/compression_types.h )
@@ -270,6 +270,11 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/compression/stream_compression_identity.h )
   s.files += %w( src/core/lib/debug/stats.h )
   s.files += %w( src/core/lib/debug/stats_data.h )
+  s.files += %w( src/core/lib/gpr++/debug_location.h )
+  s.files += %w( src/core/lib/gpr++/inlined_vector.h )
+  s.files += %w( src/core/lib/gpr++/orphanable.h )
+  s.files += %w( src/core/lib/gpr++/ref_counted.h )
+  s.files += %w( src/core/lib/gpr++/ref_counted_ptr.h )
   s.files += %w( src/core/lib/http/format_request.h )
   s.files += %w( src/core/lib/http/httpcli.h )
   s.files += %w( src/core/lib/http/parser.h )
@@ -345,11 +350,6 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/slice/slice_hash_table.h )
   s.files += %w( src/core/lib/slice/slice_internal.h )
   s.files += %w( src/core/lib/slice/slice_string_helpers.h )
-  s.files += %w( src/core/lib/support/debug_location.h )
-  s.files += %w( src/core/lib/support/orphanable.h )
-  s.files += %w( src/core/lib/support/ref_counted.h )
-  s.files += %w( src/core/lib/support/ref_counted_ptr.h )
-  s.files += %w( src/core/lib/support/vector.h )
   s.files += %w( src/core/lib/surface/alarm_internal.h )
   s.files += %w( src/core/lib/surface/api_trace.h )
   s.files += %w( src/core/lib/surface/call.h )
diff --git a/grpc.gyp b/grpc.gyp
index 281fbfa8a6e..49dcdf93a2c 100644
--- a/grpc.gyp
+++ b/grpc.gyp
@@ -161,50 +161,50 @@
       'dependencies': [
       ],
       'sources': [
+        'src/core/lib/gpr/alloc.cc',
+        'src/core/lib/gpr/arena.cc',
+        'src/core/lib/gpr/atm.cc',
+        'src/core/lib/gpr/avl.cc',
+        'src/core/lib/gpr/cmdline.cc',
+        'src/core/lib/gpr/cpu_iphone.cc',
+        'src/core/lib/gpr/cpu_linux.cc',
+        'src/core/lib/gpr/cpu_posix.cc',
+        'src/core/lib/gpr/cpu_windows.cc',
+        'src/core/lib/gpr/env_linux.cc',
+        'src/core/lib/gpr/env_posix.cc',
+        'src/core/lib/gpr/env_windows.cc',
+        'src/core/lib/gpr/fork.cc',
+        'src/core/lib/gpr/host_port.cc',
+        'src/core/lib/gpr/log.cc',
+        'src/core/lib/gpr/log_android.cc',
+        'src/core/lib/gpr/log_linux.cc',
+        'src/core/lib/gpr/log_posix.cc',
+        'src/core/lib/gpr/log_windows.cc',
+        'src/core/lib/gpr/mpscq.cc',
+        'src/core/lib/gpr/murmur_hash.cc',
+        'src/core/lib/gpr/string.cc',
+        'src/core/lib/gpr/string_posix.cc',
+        'src/core/lib/gpr/string_util_windows.cc',
+        'src/core/lib/gpr/string_windows.cc',
+        'src/core/lib/gpr/subprocess_posix.cc',
+        'src/core/lib/gpr/subprocess_windows.cc',
+        'src/core/lib/gpr/sync.cc',
+        'src/core/lib/gpr/sync_posix.cc',
+        'src/core/lib/gpr/sync_windows.cc',
+        'src/core/lib/gpr/thd.cc',
+        'src/core/lib/gpr/thd_posix.cc',
+        'src/core/lib/gpr/thd_windows.cc',
+        'src/core/lib/gpr/time.cc',
+        'src/core/lib/gpr/time_posix.cc',
+        'src/core/lib/gpr/time_precise.cc',
+        'src/core/lib/gpr/time_windows.cc',
+        'src/core/lib/gpr/tls_pthread.cc',
+        'src/core/lib/gpr/tmpfile_msys.cc',
+        'src/core/lib/gpr/tmpfile_posix.cc',
+        'src/core/lib/gpr/tmpfile_windows.cc',
+        'src/core/lib/gpr/wrap_memcpy.cc',
         'src/core/lib/profiling/basic_timers.cc',
         'src/core/lib/profiling/stap_timers.cc',
-        'src/core/lib/support/alloc.cc',
-        'src/core/lib/support/arena.cc',
-        'src/core/lib/support/atm.cc',
-        'src/core/lib/support/avl.cc',
-        'src/core/lib/support/cmdline.cc',
-        'src/core/lib/support/cpu_iphone.cc',
-        'src/core/lib/support/cpu_linux.cc',
-        'src/core/lib/support/cpu_posix.cc',
-        'src/core/lib/support/cpu_windows.cc',
-        'src/core/lib/support/env_linux.cc',
-        'src/core/lib/support/env_posix.cc',
-        'src/core/lib/support/env_windows.cc',
-        'src/core/lib/support/fork.cc',
-        'src/core/lib/support/host_port.cc',
-        'src/core/lib/support/log.cc',
-        'src/core/lib/support/log_android.cc',
-        'src/core/lib/support/log_linux.cc',
-        'src/core/lib/support/log_posix.cc',
-        'src/core/lib/support/log_windows.cc',
-        'src/core/lib/support/mpscq.cc',
-        'src/core/lib/support/murmur_hash.cc',
-        'src/core/lib/support/string.cc',
-        'src/core/lib/support/string_posix.cc',
-        'src/core/lib/support/string_util_windows.cc',
-        'src/core/lib/support/string_windows.cc',
-        'src/core/lib/support/subprocess_posix.cc',
-        'src/core/lib/support/subprocess_windows.cc',
-        'src/core/lib/support/sync.cc',
-        'src/core/lib/support/sync_posix.cc',
-        'src/core/lib/support/sync_windows.cc',
-        'src/core/lib/support/thd.cc',
-        'src/core/lib/support/thd_posix.cc',
-        'src/core/lib/support/thd_windows.cc',
-        'src/core/lib/support/time.cc',
-        'src/core/lib/support/time_posix.cc',
-        'src/core/lib/support/time_precise.cc',
-        'src/core/lib/support/time_windows.cc',
-        'src/core/lib/support/tls_pthread.cc',
-        'src/core/lib/support/tmpfile_msys.cc',
-        'src/core/lib/support/tmpfile_posix.cc',
-        'src/core/lib/support/tmpfile_windows.cc',
-        'src/core/lib/support/wrap_memcpy.cc',
       ],
     },
     {
diff --git a/package.xml b/package.xml
index 8c590348b6c..f1282964c2b 100644
--- a/package.xml
+++ b/package.xml
@@ -95,68 +95,68 @@
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_generic.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_posix.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_windows.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr++/abstract.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr++/atomic.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr++/atomic_with_atm.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr++/atomic_with_std.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr++/manual_constructor.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr++/memory.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/arena.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/env.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/fork.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/mpscq.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/murmur_hash.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/spinlock.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/string.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/string_windows.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/thd_internal.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/time_precise.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/tmpfile.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/profiling/timers.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/abstract.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/arena.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/atomic.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/atomic_with_atm.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/atomic_with_std.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/env.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/fork.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/manual_constructor.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/memory.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/mpscq.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/murmur_hash.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/spinlock.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/string.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/string_windows.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/thd_internal.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/time_precise.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/tmpfile.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/alloc.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/arena.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/atm.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/avl.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/cmdline.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/cpu_iphone.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/cpu_linux.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/cpu_posix.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/cpu_windows.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/env_linux.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/env_posix.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/env_windows.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/fork.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/host_port.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/log.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/log_android.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/log_linux.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/log_posix.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/log_windows.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/mpscq.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/murmur_hash.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/string.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/string_posix.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/string_util_windows.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/string_windows.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/subprocess_posix.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/subprocess_windows.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/sync.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/sync_posix.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/sync_windows.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/thd.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/thd_posix.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/thd_windows.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/time.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/time_posix.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/time_precise.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/time_windows.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/tls_pthread.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/tmpfile_msys.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/tmpfile_posix.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/tmpfile_windows.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr/wrap_memcpy.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/profiling/basic_timers.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/profiling/stap_timers.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/alloc.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/arena.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/atm.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/avl.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/cmdline.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/cpu_iphone.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/cpu_linux.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/cpu_posix.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/cpu_windows.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/env_linux.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/env_posix.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/env_windows.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/fork.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/host_port.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/log.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/log_android.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/log_linux.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/log_posix.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/log_windows.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/mpscq.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/murmur_hash.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/string.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/string_posix.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/string_util_windows.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/string_windows.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/subprocess_posix.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/subprocess_windows.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/sync.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/sync_posix.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/sync_windows.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/thd.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/thd_posix.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/thd_windows.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/time.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/time_posix.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/time_precise.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/time_windows.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/tls_pthread.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/tmpfile_msys.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/tmpfile_posix.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/tmpfile_windows.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/wrap_memcpy.cc" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/byte_buffer.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/byte_buffer_reader.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/compression_types.h" role="src" />
@@ -282,6 +282,11 @@
     <file baseinstalldir="/" name="src/core/lib/compression/stream_compression_identity.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/debug/stats.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/debug/stats_data.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr++/debug_location.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr++/inlined_vector.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr++/orphanable.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr++/ref_counted.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gpr++/ref_counted_ptr.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/format_request.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/httpcli.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/parser.h" role="src" />
@@ -357,11 +362,6 @@
     <file baseinstalldir="/" name="src/core/lib/slice/slice_hash_table.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/slice/slice_internal.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/slice/slice_string_helpers.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/debug_location.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/orphanable.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/ref_counted.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/ref_counted_ptr.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/support/vector.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/surface/alarm_internal.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/surface/api_trace.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/surface/call.h" role="src" />
diff --git a/src/core/ext/filters/client_channel/backup_poller.cc b/src/core/ext/filters/client_channel/backup_poller.cc
index 4ee5e9c1093..906a72b6627 100644
--- a/src/core/ext/filters/client_channel/backup_poller.cc
+++ b/src/core/ext/filters/client_channel/backup_poller.cc
@@ -23,11 +23,11 @@
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include "src/core/ext/filters/client_channel/client_channel.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/timer.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/completion_queue.h"
 
diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc
index bb29f65af97..a8a7a37be01 100644
--- a/src/core/ext/filters/client_channel/client_channel.cc
+++ b/src/core/ext/filters/client_channel/client_channel.cc
@@ -41,12 +41,12 @@
 #include "src/core/ext/filters/deadline/deadline_filter.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/connected_channel.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/metadata.h"
diff --git a/src/core/ext/filters/client_channel/http_connect_handshaker.cc b/src/core/ext/filters/client_channel/http_connect_handshaker.cc
index 556a3bc6a18..6bfd0388874 100644
--- a/src/core/ext/filters/client_channel/http_connect_handshaker.cc
+++ b/src/core/ext/filters/client_channel/http_connect_handshaker.cc
@@ -30,11 +30,11 @@
 #include "src/core/ext/filters/client_channel/uri_parser.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/handshaker_registry.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/http/format_request.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
 
 typedef struct http_connect_handshaker {
   // Base class.  Must be first.
diff --git a/src/core/ext/filters/client_channel/http_proxy.cc b/src/core/ext/filters/client_channel/http_proxy.cc
index 2eafeee7026..037c65822aa 100644
--- a/src/core/ext/filters/client_channel/http_proxy.cc
+++ b/src/core/ext/filters/client_channel/http_proxy.cc
@@ -30,9 +30,9 @@
 #include "src/core/ext/filters/client_channel/proxy_mapper_registry.h"
 #include "src/core/ext/filters/client_channel/uri_parser.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/b64.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
 
 /**
  * Parses the 'http_proxy' env var and returns the proxy hostname to resolve or
diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h
index 0cc0cb59c3e..e19726efb3b 100644
--- a/src/core/ext/filters/client_channel/lb_policy.h
+++ b/src/core/ext/filters/client_channel/lb_policy.h
@@ -20,8 +20,8 @@
 #define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_H
 
 #include "src/core/ext/filters/client_channel/subchannel.h"
+#include "src/core/lib/gpr++/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/polling_entity.h"
-#include "src/core/lib/support/ref_counted_ptr.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
 /** A load balancing policy: specified by a vtable and a struct (which
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index a8c5fb9c1bc..eb5ced4c201 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -106,6 +106,8 @@
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/gpr++/manual_constructor.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
@@ -113,8 +115,6 @@
 #include "src/core/lib/slice/slice_hash_table.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/manual_constructor.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/channel_init.h"
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.cc
index a8ecea42125..1e7f34bdc7e 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.cc
@@ -22,8 +22,8 @@
 #include "src/core/ext/filters/client_channel/client_channel.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
-#include "src/core/lib/support/string.h"
 
 grpc_channel* grpc_lb_policy_grpclb_create_lb_channel(
     const char* lb_service_target_addresses,
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc
index 76bcddf9453..15233d371ce 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc
@@ -22,11 +22,11 @@
 #include "src/core/ext/filters/client_channel/client_channel.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/security/transport/lb_targets_info.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/string.h"
 
 grpc_channel* grpc_lb_policy_grpclb_create_lb_channel(
     const char* lb_service_target_addresses,
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
index 5a36acaa572..e217a0b0c00 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
@@ -34,9 +34,9 @@
 #include "src/core/ext/filters/client_channel/subchannel_index.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gpr++/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
-#include "src/core/lib/support/ref_counted_ptr.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/static_metadata.h"
 
diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
index f146c724e07..f4e345def6b 100644
--- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
+++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
@@ -22,7 +22,7 @@
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/ext/filters/client_channel/subchannel.h"
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/support/ref_counted_ptr.h"
+#include "src/core/lib/gpr++/ref_counted_ptr.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
 // TODO(roth): This code is intended to be shared between pick_first and
diff --git a/src/core/ext/filters/client_channel/lb_policy_registry.cc b/src/core/ext/filters/client_channel/lb_policy_registry.cc
index edd0330c6a7..8414504e8fb 100644
--- a/src/core/ext/filters/client_channel/lb_policy_registry.cc
+++ b/src/core/ext/filters/client_channel/lb_policy_registry.cc
@@ -20,7 +20,7 @@
 
 #include <string.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 #define MAX_POLICIES 10
 
diff --git a/src/core/ext/filters/client_channel/parse_address.cc b/src/core/ext/filters/client_channel/parse_address.cc
index 39b1237c770..c3309e36a3c 100644
--- a/src/core/ext/filters/client_channel/parse_address.cc
+++ b/src/core/ext/filters/client_channel/parse_address.cc
@@ -29,7 +29,7 @@
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 #ifdef GRPC_HAVE_UNIX_SOCKET
 
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
index 4659a5f3ede..7ea3cdd6e19 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
@@ -34,14 +34,14 @@
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr++/manual_constructor.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/gethostname.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/json/json.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/manual_constructor.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/service_config.h"
 
 #define GRPC_DNS_INITIAL_CONNECT_BACKOFF_SECONDS 1
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc
index 40e264504cf..2eb2a9b59d8 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc
@@ -30,10 +30,10 @@
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
-#include "src/core/lib/support/string.h"
 
 typedef struct fd_node {
   /** the owner of this fd node */
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc
index 3a870b2d061..2b35bdb6059 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc
@@ -36,12 +36,12 @@
 
 #include "src/core/ext/filters/client_channel/parse_address.h"
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/nameser.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
-#include "src/core/lib/support/string.h"
 
 static gpr_once g_basic_init = GPR_ONCE_INIT;
 static gpr_mu g_init_mu;
diff --git a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
index 1c2cfc08e7b..93a1fe87a28 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
+++ b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
@@ -29,12 +29,12 @@
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr++/manual_constructor.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/timer.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/manual_constructor.h"
-#include "src/core/lib/support/string.h"
 
 #define GRPC_DNS_INITIAL_CONNECT_BACKOFF_SECONDS 1
 #define GRPC_DNS_RECONNECT_BACKOFF_MULTIPLIER 1.6
diff --git a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc
index fe3ad1403c2..eaa5e6ac490 100644
--- a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc
+++ b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc
@@ -32,13 +32,13 @@
 #include "src/core/ext/filters/client_channel/parse_address.h"
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 
 #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
 
diff --git a/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc b/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc
index 7d1e283fa3a..99ad78e23ca 100644
--- a/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc
+++ b/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc
@@ -30,12 +30,12 @@
 #include "src/core/ext/filters/client_channel/parse_address.h"
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 
 typedef struct {
   /** base class: must be first */
diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc
index 5b8a14b9e7b..fe4fcbbb7d3 100644
--- a/src/core/ext/filters/client_channel/subchannel.cc
+++ b/src/core/ext/filters/client_channel/subchannel.cc
@@ -37,12 +37,12 @@
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/connected_channel.h"
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr++/debug_location.h"
+#include "src/core/lib/gpr++/manual_constructor.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/debug_location.h"
-#include "src/core/lib/support/manual_constructor.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/channel_init.h"
 #include "src/core/lib/transport/connectivity_state.h"
diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h
index 3bcc5c2432e..f2a5c1e2732 100644
--- a/src/core/ext/filters/client_channel/subchannel.h
+++ b/src/core/ext/filters/client_channel/subchannel.h
@@ -21,10 +21,10 @@
 
 #include "src/core/ext/filters/client_channel/connector.h"
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/gpr++/ref_counted.h"
+#include "src/core/lib/gpr++/ref_counted_ptr.h"
+#include "src/core/lib/gpr/arena.h"
 #include "src/core/lib/iomgr/polling_entity.h"
-#include "src/core/lib/support/arena.h"
-#include "src/core/lib/support/ref_counted.h"
-#include "src/core/lib/support/ref_counted_ptr.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/metadata.h"
 
diff --git a/src/core/ext/filters/client_channel/uri_parser.cc b/src/core/ext/filters/client_channel/uri_parser.cc
index 3428f4b54c5..c5f2d6822cd 100644
--- a/src/core/ext/filters/client_channel/uri_parser.cc
+++ b/src/core/ext/filters/client_channel/uri_parser.cc
@@ -26,10 +26,10 @@
 #include <grpc/support/port_platform.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/percent_encoding.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 
 /** a size_t default value... maps to all 1's */
 #define NOT_SET (~(size_t)0)
diff --git a/src/core/ext/filters/http/client/http_client_filter.cc b/src/core/ext/filters/http/client/http_client_filter.cc
index 6dbd8c2a6d5..5584d50018e 100644
--- a/src/core/ext/filters/http/client/http_client_filter.cc
+++ b/src/core/ext/filters/http/client/http_client_filter.cc
@@ -20,12 +20,12 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include <string.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/b64.h"
 #include "src/core/lib/slice/percent_encoding.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/static_metadata.h"
 #include "src/core/lib/transport/transport_impl.h"
 
diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.cc b/src/core/ext/filters/http/message_compress/message_compress_filter.cc
index 92d17162001..d0b97504973 100644
--- a/src/core/ext/filters/http/message_compress/message_compress_filter.cc
+++ b/src/core/ext/filters/http/message_compress/message_compress_filter.cc
@@ -28,10 +28,10 @@
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/compression/algorithm_metadata.h"
 #include "src/core/lib/compression/message_compress.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/transport/static_metadata.h"
 
diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc
index 3cb7b136c00..8d76c4a837b 100644
--- a/src/core/ext/filters/message_size/message_size_filter.cc
+++ b/src/core/ext/filters/message_size/message_size_filter.cc
@@ -26,7 +26,7 @@
 
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack_builder.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/surface/channel_init.h"
 #include "src/core/lib/transport/service_config.h"
 
diff --git a/src/core/ext/transport/chttp2/transport/bin_decoder.cc b/src/core/ext/transport/chttp2/transport/bin_decoder.cc
index 984cd4ca78c..74778ac0468 100644
--- a/src/core/ext/transport/chttp2/transport/bin_decoder.cc
+++ b/src/core/ext/transport/chttp2/transport/bin_decoder.cc
@@ -19,9 +19,9 @@
 #include "src/core/ext/transport/chttp2/transport/bin_decoder.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 
 static uint8_t decode_table[] = {
     0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
diff --git a/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc b/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc
index 3aca61fdac7..a69908116ab 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc
@@ -18,7 +18,7 @@
 
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/transport/metadata.h"
 
 void grpc_chttp2_plugin_init(void) {
diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 835de6aa0f4..e067b696a16 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -38,14 +38,14 @@
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/compression/stream_compression.h"
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/error_utils.h"
 #include "src/core/lib/transport/http2_errors.h"
 #include "src/core/lib/transport/static_metadata.h"
diff --git a/src/core/ext/transport/chttp2/transport/flow_control.cc b/src/core/ext/transport/chttp2/transport/flow_control.cc
index 3013db23f70..581241d392d 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.cc
+++ b/src/core/ext/transport/chttp2/transport/flow_control.cc
@@ -29,7 +29,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/ext/transport/chttp2/transport/internal.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 grpc_core::TraceFlag grpc_flowctl_trace(false, "flowctl");
 
diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h
index 38b9f50c8d4..7e83ea05cd5 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.h
+++ b/src/core/ext/transport/chttp2/transport/flow_control.h
@@ -24,8 +24,8 @@
 
 #include <grpc/support/useful.h>
 #include "src/core/ext/transport/chttp2/transport/http2_settings.h"
-#include "src/core/lib/support/abstract.h"
-#include "src/core/lib/support/manual_constructor.h"
+#include "src/core/lib/gpr++/abstract.h"
+#include "src/core/lib/gpr++/manual_constructor.h"
 #include "src/core/lib/transport/bdp_estimator.h"
 #include "src/core/lib/transport/pid_controller.h"
 
diff --git a/src/core/ext/transport/chttp2/transport/frame_data.cc b/src/core/ext/transport/chttp2/transport/frame_data.cc
index 9b3a6acc9e4..043b80a3cb8 100644
--- a/src/core/ext/transport/chttp2/transport/frame_data.cc
+++ b/src/core/ext/transport/chttp2/transport/frame_data.cc
@@ -25,9 +25,9 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 #include "src/core/ext/transport/chttp2/transport/internal.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/transport.h"
 
 grpc_error* grpc_chttp2_data_parser_init(grpc_chttp2_data_parser* parser) {
diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.cc b/src/core/ext/transport/chttp2/transport/hpack_parser.cc
index a395ab234c0..ebee5913cb8 100644
--- a/src/core/ext/transport/chttp2/transport/hpack_parser.cc
+++ b/src/core/ext/transport/chttp2/transport/hpack_parser.cc
@@ -31,10 +31,10 @@
 
 #include "src/core/ext/transport/chttp2/transport/bin_encoder.h"
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/http2_errors.h"
 
 typedef enum {
diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.cc b/src/core/ext/transport/chttp2/transport/hpack_table.cc
index c325465daab..9fad158d275 100644
--- a/src/core/ext/transport/chttp2/transport/hpack_table.cc
+++ b/src/core/ext/transport/chttp2/transport/hpack_table.cc
@@ -26,7 +26,7 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/support/murmur_hash.h"
+#include "src/core/lib/gpr/murmur_hash.h"
 
 extern grpc_core::TraceFlag grpc_http_trace;
 
diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h
index 8852d9ee2b0..de901f0ce89 100644
--- a/src/core/ext/transport/chttp2/transport/internal.h
+++ b/src/core/ext/transport/chttp2/transport/internal.h
@@ -35,10 +35,10 @@
 #include "src/core/ext/transport/chttp2/transport/incoming_metadata.h"
 #include "src/core/ext/transport/chttp2/transport/stream_map.h"
 #include "src/core/lib/compression/stream_compression.h"
+#include "src/core/lib/gpr++/manual_constructor.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/timer.h"
-#include "src/core/lib/support/manual_constructor.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/transport_impl.h"
 
diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.cc b/src/core/ext/transport/cronet/transport/cronet_transport.cc
index 5723da5f9d6..5b1c6ab3f9a 100644
--- a/src/core/ext/transport/cronet/transport/cronet_transport.cc
+++ b/src/core/ext/transport/cronet/transport/cronet_transport.cc
@@ -28,11 +28,11 @@
 
 #include "src/core/ext/transport/chttp2/transport/incoming_metadata.h"
 #include "src/core/ext/transport/cronet/transport/cronet_transport.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/transport/metadata_batch.h"
 #include "src/core/lib/transport/static_metadata.h"
diff --git a/src/core/lib/channel/channel_args.cc b/src/core/lib/channel/channel_args.cc
index 578475b2484..634286d403f 100644
--- a/src/core/lib/channel/channel_args.cc
+++ b/src/core/lib/channel/channel_args.cc
@@ -29,7 +29,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 static grpc_arg copy_arg(const grpc_arg* src) {
   grpc_arg dst;
diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h
index 716866be26a..b9f97480012 100644
--- a/src/core/lib/channel/channel_stack.h
+++ b/src/core/lib/channel/channel_stack.h
@@ -40,9 +40,9 @@
 #include <grpc/support/time.h>
 
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gpr/arena.h"
 #include "src/core/lib/iomgr/call_combiner.h"
 #include "src/core/lib/iomgr/polling_entity.h"
-#include "src/core/lib/support/arena.h"
 #include "src/core/lib/transport/transport.h"
 
 typedef struct grpc_channel_element grpc_channel_element;
diff --git a/src/core/lib/channel/connected_channel.cc b/src/core/lib/channel/connected_channel.cc
index 9d07cfff4e5..fb26bdf586b 100644
--- a/src/core/lib/channel/connected_channel.cc
+++ b/src/core/lib/channel/connected_channel.cc
@@ -26,8 +26,8 @@
 #include <grpc/slice_buffer.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/transport.h"
 
 #define MAX_BUFFER_LENGTH 8192
diff --git a/src/core/lib/debug/stats.cc b/src/core/lib/debug/stats.cc
index 0b39b2b1e71..465c76114d4 100644
--- a/src/core/lib/debug/stats.cc
+++ b/src/core/lib/debug/stats.cc
@@ -25,7 +25,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 grpc_stats_data* grpc_stats_per_cpu_storage = nullptr;
 static size_t g_num_cores;
diff --git a/src/core/lib/debug/trace.cc b/src/core/lib/debug/trace.cc
index a76c1afb4c5..d870bbc5e7c 100644
--- a/src/core/lib/debug/trace.cc
+++ b/src/core/lib/debug/trace.cc
@@ -23,7 +23,7 @@
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 
 int grpc_tracer_set_enabled(const char* name, int enabled);
 
diff --git a/src/core/lib/gpr++/README.md b/src/core/lib/gpr++/README.md
new file mode 100644
index 00000000000..eab018bb312
--- /dev/null
+++ b/src/core/lib/gpr++/README.md
@@ -0,0 +1,16 @@
+# GPR++ - Google Portable Runtime for C++
+
+The files in this directory contain various utility code for C++ code.
+None of this code is gRPC-specific; anything here may also be useful
+for other open source projects written in C++.
+
+Note that this is one of the few places in src/core where we allow
+the use of portability macros.
+
+Note that this is the only place in src/core where we allow
+use of the C++ standard library (i.e., anything in the `std::`
+namespace).  And for now, we require that any use of the
+standard library is build-time-only -- i.e., we do not allow
+run-time dependencies on libstdc++.  For more details, see
+[gRFC L6](/grpc/proposal/blob/master/L6-allow-c%2B%2B-in-grpc-core.md) and
+[Moving gRPC core to C++](/grpc/grpc/blob/master/doc/core/moving-to-c%2B%2B.md).
diff --git a/src/core/lib/support/abstract.h b/src/core/lib/gpr++/abstract.h
similarity index 90%
rename from src/core/lib/support/abstract.h
rename to src/core/lib/gpr++/abstract.h
index 1dffa301282..51d75723026 100644
--- a/src/core/lib/support/abstract.h
+++ b/src/core/lib/gpr++/abstract.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_ABSTRACT_H
-#define GRPC_CORE_LIB_SUPPORT_ABSTRACT_H
+#ifndef GRPC_CORE_LIB_GPRXX_ABSTRACT_H
+#define GRPC_CORE_LIB_GPRXX_ABSTRACT_H
 
 // This is needed to support abstract base classes in the c core. Since gRPC
 // doesn't have a c++ runtime, it will hit a linker error on delete unless
@@ -31,4 +31,4 @@
 #define GRPC_ABSTRACT \
   { GPR_ASSERT(false); }
 
-#endif /* GRPC_CORE_LIB_SUPPORT_ABSTRACT_H */
+#endif /* GRPC_CORE_LIB_GPRXX_ABSTRACT_H */
diff --git a/src/core/lib/support/atomic.h b/src/core/lib/gpr++/atomic.h
similarity index 75%
rename from src/core/lib/support/atomic.h
rename to src/core/lib/gpr++/atomic.h
index 73c59ae3cd0..d68ccea864f 100644
--- a/src/core/lib/support/atomic.h
+++ b/src/core/lib/gpr++/atomic.h
@@ -16,15 +16,15 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_H
-#define GRPC_CORE_LIB_SUPPORT_ATOMIC_H
+#ifndef GRPC_CORE_LIB_GPRXX_ATOMIC_H
+#define GRPC_CORE_LIB_GPRXX_ATOMIC_H
 
 #include <grpc/support/port_platform.h>
 
 #ifdef GPR_HAS_CXX11_ATOMIC
-#include "src/core/lib/support/atomic_with_std.h"
+#include "src/core/lib/gpr++/atomic_with_std.h"
 #else
-#include "src/core/lib/support/atomic_with_atm.h"
+#include "src/core/lib/gpr++/atomic_with_atm.h"
 #endif
 
-#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_H */
+#endif /* GRPC_CORE_LIB_GPRXX_ATOMIC_H */
diff --git a/src/core/lib/support/atomic_with_atm.h b/src/core/lib/gpr++/atomic_with_atm.h
similarity index 90%
rename from src/core/lib/support/atomic_with_atm.h
rename to src/core/lib/gpr++/atomic_with_atm.h
index fe00e9b5bc5..09490e8148e 100644
--- a/src/core/lib/support/atomic_with_atm.h
+++ b/src/core/lib/gpr++/atomic_with_atm.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H
-#define GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H
+#ifndef GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_ATM_H
+#define GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_ATM_H
 
 #include <grpc/support/atm.h>
 
@@ -52,4 +52,4 @@ class atomic<bool> {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H */
+#endif /* GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_ATM_H */
diff --git a/src/core/lib/support/atomic_with_std.h b/src/core/lib/gpr++/atomic_with_std.h
similarity index 83%
rename from src/core/lib/support/atomic_with_std.h
rename to src/core/lib/gpr++/atomic_with_std.h
index c7a92f70124..b6ff90655e9 100644
--- a/src/core/lib/support/atomic_with_std.h
+++ b/src/core/lib/gpr++/atomic_with_std.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H
-#define GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H
+#ifndef GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_STD_H
+#define GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_STD_H
 
 #include <atomic>
 
@@ -30,4 +30,4 @@ typedef std::memory_order memory_order;
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H */
+#endif /* GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_STD_H */
diff --git a/src/core/lib/support/debug_location.h b/src/core/lib/gpr++/debug_location.h
similarity index 90%
rename from src/core/lib/support/debug_location.h
rename to src/core/lib/gpr++/debug_location.h
index 9b3f9220fc9..5a8665ce193 100644
--- a/src/core/lib/support/debug_location.h
+++ b/src/core/lib/gpr++/debug_location.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_DEBUG_LOCATION_H
-#define GRPC_CORE_LIB_SUPPORT_DEBUG_LOCATION_H
+#ifndef GRPC_CORE_LIB_GPRXX_DEBUG_LOCATION_H
+#define GRPC_CORE_LIB_GPRXX_DEBUG_LOCATION_H
 
 namespace grpc_core {
 
@@ -49,4 +49,4 @@ class DebugLocation {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_SUPPORT_DEBUG_LOCATION_H */
+#endif /* GRPC_CORE_LIB_GPRXX_DEBUG_LOCATION_H */
diff --git a/src/core/lib/support/vector.h b/src/core/lib/gpr++/inlined_vector.h
similarity index 94%
rename from src/core/lib/support/vector.h
rename to src/core/lib/gpr++/inlined_vector.h
index 2f249a5b9e8..17ee9e16bbe 100644
--- a/src/core/lib/support/vector.h
+++ b/src/core/lib/gpr++/inlined_vector.h
@@ -16,12 +16,12 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_VECTOR_H
-#define GRPC_CORE_LIB_SUPPORT_VECTOR_H
+#ifndef GRPC_CORE_LIB_GPRXX_INLINED_VECTOR_H
+#define GRPC_CORE_LIB_GPRXX_INLINED_VECTOR_H
 
 #include <cassert>
 
-#include "src/core/lib/support/memory.h"
+#include "src/core/lib/gpr++/memory.h"
 
 namespace grpc_core {
 
@@ -109,4 +109,4 @@ class InlinedVector {
 
 }  // namespace grpc_core
 
-#endif
+#endif /* GRPC_CORE_LIB_GPRXX_INLINED_VECTOR_H */
diff --git a/src/core/lib/support/manual_constructor.h b/src/core/lib/gpr++/manual_constructor.h
similarity index 98%
rename from src/core/lib/support/manual_constructor.h
rename to src/core/lib/gpr++/manual_constructor.h
index fda7653dbce..a3f006da341 100644
--- a/src/core/lib/support/manual_constructor.h
+++ b/src/core/lib/gpr++/manual_constructor.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_MANUAL_CONSTRUCTOR_H
-#define GRPC_CORE_LIB_SUPPORT_MANUAL_CONSTRUCTOR_H
+#ifndef GRPC_CORE_LIB_GPRXX_MANUAL_CONSTRUCTOR_H
+#define GRPC_CORE_LIB_GPRXX_MANUAL_CONSTRUCTOR_H
 
 // manually construct a region of memory with some type
 
diff --git a/src/core/lib/support/memory.h b/src/core/lib/gpr++/memory.h
similarity index 95%
rename from src/core/lib/support/memory.h
rename to src/core/lib/gpr++/memory.h
index 695418e3e1e..75ed3d6ceae 100644
--- a/src/core/lib/support/memory.h
+++ b/src/core/lib/gpr++/memory.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_MEMORY_H
-#define GRPC_CORE_LIB_SUPPORT_MEMORY_H
+#ifndef GRPC_CORE_LIB_GPRXX_MEMORY_H
+#define GRPC_CORE_LIB_GPRXX_MEMORY_H
 
 #include <grpc/support/alloc.h>
 
@@ -97,4 +97,4 @@ class Allocator {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_SUPPORT_MEMORY_H */
+#endif /* GRPC_CORE_LIB_GPRXX_MEMORY_H */
diff --git a/src/core/lib/support/orphanable.h b/src/core/lib/gpr++/orphanable.h
similarity index 94%
rename from src/core/lib/support/orphanable.h
rename to src/core/lib/gpr++/orphanable.h
index 2f537573fd4..f106e74dde0 100644
--- a/src/core/lib/support/orphanable.h
+++ b/src/core/lib/gpr++/orphanable.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_ORPHANABLE_H
-#define GRPC_CORE_LIB_SUPPORT_ORPHANABLE_H
+#ifndef GRPC_CORE_LIB_GPRXX_ORPHANABLE_H
+#define GRPC_CORE_LIB_GPRXX_ORPHANABLE_H
 
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
@@ -25,9 +25,9 @@
 #include <memory>
 
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/support/abstract.h"
-#include "src/core/lib/support/debug_location.h"
-#include "src/core/lib/support/memory.h"
+#include "src/core/lib/gpr++/abstract.h"
+#include "src/core/lib/gpr++/debug_location.h"
+#include "src/core/lib/gpr++/memory.h"
 
 namespace grpc_core {
 
@@ -168,4 +168,4 @@ class InternallyRefCountedWithTracing : public Orphanable {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_SUPPORT_ORPHANABLE_H */
+#endif /* GRPC_CORE_LIB_GPRXX_ORPHANABLE_H */
diff --git a/src/core/lib/support/ref_counted.h b/src/core/lib/gpr++/ref_counted.h
similarity index 93%
rename from src/core/lib/support/ref_counted.h
rename to src/core/lib/gpr++/ref_counted.h
index 48c11f7bbfe..c2ae76c0aef 100644
--- a/src/core/lib/support/ref_counted.h
+++ b/src/core/lib/gpr++/ref_counted.h
@@ -16,16 +16,16 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_REF_COUNTED_H
-#define GRPC_CORE_LIB_SUPPORT_REF_COUNTED_H
+#ifndef GRPC_CORE_LIB_GPRXX_REF_COUNTED_H
+#define GRPC_CORE_LIB_GPRXX_REF_COUNTED_H
 
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/support/abstract.h"
-#include "src/core/lib/support/debug_location.h"
-#include "src/core/lib/support/memory.h"
+#include "src/core/lib/gpr++/abstract.h"
+#include "src/core/lib/gpr++/debug_location.h"
+#include "src/core/lib/gpr++/memory.h"
 
 namespace grpc_core {
 
@@ -130,4 +130,4 @@ class RefCountedWithTracing {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_SUPPORT_REF_COUNTED_H */
+#endif /* GRPC_CORE_LIB_GPRXX_REF_COUNTED_H */
diff --git a/src/core/lib/support/ref_counted_ptr.h b/src/core/lib/gpr++/ref_counted_ptr.h
similarity index 93%
rename from src/core/lib/support/ref_counted_ptr.h
rename to src/core/lib/gpr++/ref_counted_ptr.h
index 76ff0bba668..862294d1aa0 100644
--- a/src/core/lib/support/ref_counted_ptr.h
+++ b/src/core/lib/gpr++/ref_counted_ptr.h
@@ -16,12 +16,12 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_REF_COUNTED_PTR_H
-#define GRPC_CORE_LIB_SUPPORT_REF_COUNTED_PTR_H
+#ifndef GRPC_CORE_LIB_GPRXX_REF_COUNTED_PTR_H
+#define GRPC_CORE_LIB_GPRXX_REF_COUNTED_PTR_H
 
 #include <utility>
 
-#include "src/core/lib/support/memory.h"
+#include "src/core/lib/gpr++/memory.h"
 
 namespace grpc_core {
 
@@ -96,4 +96,4 @@ inline RefCountedPtr<T> MakeRefCounted(Args&&... args) {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_SUPPORT_REF_COUNTED_PTR_H */
+#endif /* GRPC_CORE_LIB_GPRXX_REF_COUNTED_PTR_H */
diff --git a/src/core/lib/gpr/README.md b/src/core/lib/gpr/README.md
new file mode 100644
index 00000000000..21fb0c796db
--- /dev/null
+++ b/src/core/lib/gpr/README.md
@@ -0,0 +1,8 @@
+# GPR - Google Portable Runtime for C
+
+The files in this directory contain basic utility code and platform
+abstractions for C code.  None of this code is gRPC-specific; anything
+here may also be useful for other open source projects written in C.
+
+Note that this is one of the few places in src/core where we allow
+the use of portability macros.
diff --git a/src/core/lib/support/alloc.cc b/src/core/lib/gpr/alloc.cc
similarity index 100%
rename from src/core/lib/support/alloc.cc
rename to src/core/lib/gpr/alloc.cc
diff --git a/src/core/lib/support/arena.cc b/src/core/lib/gpr/arena.cc
similarity index 98%
rename from src/core/lib/support/arena.cc
rename to src/core/lib/gpr/arena.cc
index 5b9dd370d84..177c1767325 100644
--- a/src/core/lib/support/arena.cc
+++ b/src/core/lib/gpr/arena.cc
@@ -16,7 +16,7 @@
  *
  */
 
-#include "src/core/lib/support/arena.h"
+#include "src/core/lib/gpr/arena.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/support/arena.h b/src/core/lib/gpr/arena.h
similarity index 91%
rename from src/core/lib/support/arena.h
rename to src/core/lib/gpr/arena.h
index cfe973a0368..339771c0e3c 100644
--- a/src/core/lib/support/arena.h
+++ b/src/core/lib/gpr/arena.h
@@ -22,8 +22,8 @@
 // Tracks the total memory allocated against it, so that future arenas can
 // pre-allocate the right amount of memory
 
-#ifndef GRPC_CORE_LIB_SUPPORT_ARENA_H
-#define GRPC_CORE_LIB_SUPPORT_ARENA_H
+#ifndef GRPC_CORE_LIB_GPR_ARENA_H
+#define GRPC_CORE_LIB_GPR_ARENA_H
 
 #include <stddef.h>
 
@@ -36,4 +36,4 @@ void* gpr_arena_alloc(gpr_arena* arena, size_t size);
 // Destroy an arena, returning the total number of bytes allocated
 size_t gpr_arena_destroy(gpr_arena* arena);
 
-#endif /* GRPC_CORE_LIB_SUPPORT_ARENA_H */
+#endif /* GRPC_CORE_LIB_GPR_ARENA_H */
diff --git a/src/core/lib/support/atm.cc b/src/core/lib/gpr/atm.cc
similarity index 100%
rename from src/core/lib/support/atm.cc
rename to src/core/lib/gpr/atm.cc
diff --git a/src/core/lib/support/avl.cc b/src/core/lib/gpr/avl.cc
similarity index 100%
rename from src/core/lib/support/avl.cc
rename to src/core/lib/gpr/avl.cc
diff --git a/src/core/lib/support/cmdline.cc b/src/core/lib/gpr/cmdline.cc
similarity index 99%
rename from src/core/lib/support/cmdline.cc
rename to src/core/lib/gpr/cmdline.cc
index da9f10a4963..4118f9a72a4 100644
--- a/src/core/lib/support/cmdline.cc
+++ b/src/core/lib/gpr/cmdline.cc
@@ -25,7 +25,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 typedef enum { ARGTYPE_INT, ARGTYPE_BOOL, ARGTYPE_STRING } argtype;
 
diff --git a/src/core/lib/support/cpu_iphone.cc b/src/core/lib/gpr/cpu_iphone.cc
similarity index 100%
rename from src/core/lib/support/cpu_iphone.cc
rename to src/core/lib/gpr/cpu_iphone.cc
diff --git a/src/core/lib/support/cpu_linux.cc b/src/core/lib/gpr/cpu_linux.cc
similarity index 100%
rename from src/core/lib/support/cpu_linux.cc
rename to src/core/lib/gpr/cpu_linux.cc
diff --git a/src/core/lib/support/cpu_posix.cc b/src/core/lib/gpr/cpu_posix.cc
similarity index 100%
rename from src/core/lib/support/cpu_posix.cc
rename to src/core/lib/gpr/cpu_posix.cc
diff --git a/src/core/lib/support/cpu_windows.cc b/src/core/lib/gpr/cpu_windows.cc
similarity index 100%
rename from src/core/lib/support/cpu_windows.cc
rename to src/core/lib/gpr/cpu_windows.cc
diff --git a/src/core/lib/support/env.h b/src/core/lib/gpr/env.h
similarity index 92%
rename from src/core/lib/support/env.h
rename to src/core/lib/gpr/env.h
index 2452fd330d9..7f35104be3e 100644
--- a/src/core/lib/support/env.h
+++ b/src/core/lib/gpr/env.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_ENV_H
-#define GRPC_CORE_LIB_SUPPORT_ENV_H
+#ifndef GRPC_CORE_LIB_GPR_ENV_H
+#define GRPC_CORE_LIB_GPR_ENV_H
 
 #include <stdio.h>
 
@@ -38,4 +38,4 @@ void gpr_setenv(const char* name, const char* value);
    level of logging. So DO NOT USE THIS. */
 const char* gpr_getenv_silent(const char* name, char** dst);
 
-#endif /* GRPC_CORE_LIB_SUPPORT_ENV_H */
+#endif /* GRPC_CORE_LIB_GPR_ENV_H */
diff --git a/src/core/lib/support/env_linux.cc b/src/core/lib/gpr/env_linux.cc
similarity index 96%
rename from src/core/lib/support/env_linux.cc
rename to src/core/lib/gpr/env_linux.cc
index 0af2de9f7e0..17902c3d0b9 100644
--- a/src/core/lib/support/env_linux.cc
+++ b/src/core/lib/gpr/env_linux.cc
@@ -25,7 +25,7 @@
 
 #ifdef GPR_LINUX_ENV
 
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 
 #include <dlfcn.h>
 #include <features.h>
@@ -36,7 +36,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 const char* gpr_getenv_silent(const char* name, char** dst) {
   const char* insecure_func_used = nullptr;
diff --git a/src/core/lib/support/env_posix.cc b/src/core/lib/gpr/env_posix.cc
similarity index 93%
rename from src/core/lib/support/env_posix.cc
rename to src/core/lib/gpr/env_posix.cc
index 8146330555c..599f85aa72b 100644
--- a/src/core/lib/support/env_posix.cc
+++ b/src/core/lib/gpr/env_posix.cc
@@ -20,14 +20,14 @@
 
 #ifdef GPR_POSIX_ENV
 
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 
 #include <stdlib.h>
 
 #include <grpc/support/log.h>
 
 #include <grpc/support/string_util.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 const char* gpr_getenv_silent(const char* name, char** dst) {
   *dst = gpr_getenv(name);
diff --git a/src/core/lib/support/env_windows.cc b/src/core/lib/gpr/env_windows.cc
similarity index 93%
rename from src/core/lib/support/env_windows.cc
rename to src/core/lib/gpr/env_windows.cc
index cdb1d58ccd9..9ca1e02d038 100644
--- a/src/core/lib/support/env_windows.cc
+++ b/src/core/lib/gpr/env_windows.cc
@@ -22,9 +22,9 @@
 
 #include <windows.h>
 
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/string_windows.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/string_windows.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/support/fork.cc b/src/core/lib/gpr/fork.cc
similarity index 95%
rename from src/core/lib/support/fork.cc
rename to src/core/lib/gpr/fork.cc
index dc291c40804..c47e686378e 100644
--- a/src/core/lib/support/fork.cc
+++ b/src/core/lib/gpr/fork.cc
@@ -16,14 +16,14 @@
  *
  */
 
-#include "src/core/lib/support/fork.h"
+#include "src/core/lib/gpr/fork.h"
 
 #include <string.h>
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 
 /*
  * NOTE: FORKING IS NOT GENERALLY SUPPORTED, THIS IS ONLY INTENDED TO WORK
diff --git a/src/core/lib/support/fork.h b/src/core/lib/gpr/fork.h
similarity index 89%
rename from src/core/lib/support/fork.h
rename to src/core/lib/gpr/fork.h
index 215d4214a61..94c61bb836b 100644
--- a/src/core/lib/support/fork.h
+++ b/src/core/lib/gpr/fork.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_FORK_H
-#define GRPC_CORE_LIB_SUPPORT_FORK_H
+#ifndef GRPC_CORE_LIB_GPR_FORK_H
+#define GRPC_CORE_LIB_GPR_FORK_H
 
 /*
  * NOTE: FORKING IS NOT GENERALLY SUPPORTED, THIS IS ONLY INTENDED TO WORK
@@ -32,4 +32,4 @@ int grpc_fork_support_enabled(void);
 // environment variables/compile flags
 void grpc_enable_fork_support(int enable);
 
-#endif /* GRPC_CORE_LIB_SUPPORT_FORK_H */
+#endif /* GRPC_CORE_LIB_GPR_FORK_H */
diff --git a/src/core/lib/support/host_port.cc b/src/core/lib/gpr/host_port.cc
similarity index 98%
rename from src/core/lib/support/host_port.cc
rename to src/core/lib/gpr/host_port.cc
index cb8e3d44797..29178279d3f 100644
--- a/src/core/lib/support/host_port.cc
+++ b/src/core/lib/gpr/host_port.cc
@@ -23,7 +23,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 int gpr_join_host_port(char** out, const char* host, int port) {
   if (host[0] != '[' && strchr(host, ':') != nullptr) {
diff --git a/src/core/lib/support/log.cc b/src/core/lib/gpr/log.cc
similarity index 97%
rename from src/core/lib/support/log.cc
rename to src/core/lib/gpr/log.cc
index 2a40745e97d..19c0f6c34d3 100644
--- a/src/core/lib/support/log.cc
+++ b/src/core/lib/gpr/log.cc
@@ -21,8 +21,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 
 #include <stdio.h>
 #include <string.h>
diff --git a/src/core/lib/support/log_android.cc b/src/core/lib/gpr/log_android.cc
similarity index 100%
rename from src/core/lib/support/log_android.cc
rename to src/core/lib/gpr/log_android.cc
diff --git a/src/core/lib/support/log_linux.cc b/src/core/lib/gpr/log_linux.cc
similarity index 100%
rename from src/core/lib/support/log_linux.cc
rename to src/core/lib/gpr/log_linux.cc
diff --git a/src/core/lib/support/log_posix.cc b/src/core/lib/gpr/log_posix.cc
similarity index 100%
rename from src/core/lib/support/log_posix.cc
rename to src/core/lib/gpr/log_posix.cc
diff --git a/src/core/lib/support/log_windows.cc b/src/core/lib/gpr/log_windows.cc
similarity index 96%
rename from src/core/lib/support/log_windows.cc
rename to src/core/lib/gpr/log_windows.cc
index 0013bf448fd..caaa973f5a8 100644
--- a/src/core/lib/support/log_windows.cc
+++ b/src/core/lib/gpr/log_windows.cc
@@ -29,8 +29,8 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/string_windows.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/string_windows.h"
 
 void gpr_log(const char* file, int line, gpr_log_severity severity,
              const char* format, ...) {
diff --git a/src/core/lib/support/mpscq.cc b/src/core/lib/gpr/mpscq.cc
similarity index 98%
rename from src/core/lib/support/mpscq.cc
rename to src/core/lib/gpr/mpscq.cc
index 47e896d2df6..34fc050a116 100644
--- a/src/core/lib/support/mpscq.cc
+++ b/src/core/lib/gpr/mpscq.cc
@@ -16,7 +16,7 @@
  *
  */
 
-#include "src/core/lib/support/mpscq.h"
+#include "src/core/lib/gpr/mpscq.h"
 
 #include <grpc/support/log.h>
 
diff --git a/src/core/lib/support/mpscq.h b/src/core/lib/gpr/mpscq.h
similarity index 96%
rename from src/core/lib/support/mpscq.h
rename to src/core/lib/gpr/mpscq.h
index 648ead1f5b5..4409c5c9f5e 100644
--- a/src/core/lib/support/mpscq.h
+++ b/src/core/lib/gpr/mpscq.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_MPSCQ_H
-#define GRPC_CORE_LIB_SUPPORT_MPSCQ_H
+#ifndef GRPC_CORE_LIB_GPR_MPSCQ_H
+#define GRPC_CORE_LIB_GPR_MPSCQ_H
 
 #include <grpc/support/atm.h>
 #include <grpc/support/sync.h>
@@ -81,4 +81,4 @@ gpr_mpscq_node* gpr_locked_mpscq_try_pop(gpr_locked_mpscq* q);
 // calling this function
 gpr_mpscq_node* gpr_locked_mpscq_pop(gpr_locked_mpscq* q);
 
-#endif /* GRPC_CORE_LIB_SUPPORT_MPSCQ_H */
+#endif /* GRPC_CORE_LIB_GPR_MPSCQ_H */
diff --git a/src/core/lib/support/murmur_hash.cc b/src/core/lib/gpr/murmur_hash.cc
similarity index 97%
rename from src/core/lib/support/murmur_hash.cc
rename to src/core/lib/gpr/murmur_hash.cc
index 2f0e71a53c9..3f5e04d2119 100644
--- a/src/core/lib/support/murmur_hash.cc
+++ b/src/core/lib/gpr/murmur_hash.cc
@@ -16,7 +16,7 @@
  *
  */
 
-#include "src/core/lib/support/murmur_hash.h"
+#include "src/core/lib/gpr/murmur_hash.h"
 
 #include <string.h>
 
diff --git a/src/core/lib/support/murmur_hash.h b/src/core/lib/gpr/murmur_hash.h
similarity index 85%
rename from src/core/lib/support/murmur_hash.h
rename to src/core/lib/gpr/murmur_hash.h
index 422770f1035..8004889a9a5 100644
--- a/src/core/lib/support/murmur_hash.h
+++ b/src/core/lib/gpr/murmur_hash.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_MURMUR_HASH_H
-#define GRPC_CORE_LIB_SUPPORT_MURMUR_HASH_H
+#ifndef GRPC_CORE_LIB_GPR_MURMUR_HASH_H
+#define GRPC_CORE_LIB_GPR_MURMUR_HASH_H
 
 #include <grpc/support/port_platform.h>
 
@@ -26,4 +26,4 @@
 /* compute the hash of key (length len) */
 uint32_t gpr_murmur_hash3(const void* key, size_t len, uint32_t seed);
 
-#endif /* GRPC_CORE_LIB_SUPPORT_MURMUR_HASH_H */
+#endif /* GRPC_CORE_LIB_GPR_MURMUR_HASH_H */
diff --git a/src/core/lib/support/spinlock.h b/src/core/lib/gpr/spinlock.h
similarity index 90%
rename from src/core/lib/support/spinlock.h
rename to src/core/lib/gpr/spinlock.h
index 8b439642e9d..f03be1d7919 100644
--- a/src/core/lib/support/spinlock.h
+++ b/src/core/lib/gpr/spinlock.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_SPINLOCK_H
-#define GRPC_CORE_LIB_SUPPORT_SPINLOCK_H
+#ifndef GRPC_CORE_LIB_GPR_SPINLOCK_H
+#define GRPC_CORE_LIB_GPR_SPINLOCK_H
 
 #include <grpc/support/atm.h>
 
@@ -41,4 +41,4 @@ typedef struct {
   do {                          \
   } while (!gpr_spinlock_trylock((lock)))
 
-#endif /* GRPC_CORE_LIB_SUPPORT_SPINLOCK_H */
+#endif /* GRPC_CORE_LIB_GPR_SPINLOCK_H */
diff --git a/src/core/lib/support/string.cc b/src/core/lib/gpr/string.cc
similarity index 99%
rename from src/core/lib/support/string.cc
rename to src/core/lib/gpr/string.cc
index e31ad72c68f..919d957d1b9 100644
--- a/src/core/lib/support/string.cc
+++ b/src/core/lib/gpr/string.cc
@@ -16,7 +16,7 @@
  *
  */
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 #include <ctype.h>
 #include <limits.h>
diff --git a/src/core/lib/support/string.h b/src/core/lib/gpr/string.h
similarity index 96%
rename from src/core/lib/support/string.h
rename to src/core/lib/gpr/string.h
index dd37f0b0e19..ef3a8c6086e 100644
--- a/src/core/lib/support/string.h
+++ b/src/core/lib/gpr/string.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_STRING_H
-#define GRPC_CORE_LIB_SUPPORT_STRING_H
+#ifndef GRPC_CORE_LIB_GPR_STRING_H
+#define GRPC_CORE_LIB_GPR_STRING_H
 
 #include <stdbool.h>
 #include <stddef.h>
@@ -106,4 +106,4 @@ void* gpr_memrchr(const void* s, int c, size_t n);
 /** Return true if lower(s) equals "true", "yes" or "1", otherwise false. */
 bool gpr_is_true(const char* s);
 
-#endif /* GRPC_CORE_LIB_SUPPORT_STRING_H */
+#endif /* GRPC_CORE_LIB_GPR_STRING_H */
diff --git a/src/core/lib/support/string_posix.cc b/src/core/lib/gpr/string_posix.cc
similarity index 100%
rename from src/core/lib/support/string_posix.cc
rename to src/core/lib/gpr/string_posix.cc
diff --git a/src/core/lib/support/string_util_windows.cc b/src/core/lib/gpr/string_util_windows.cc
similarity index 96%
rename from src/core/lib/support/string_util_windows.cc
rename to src/core/lib/gpr/string_util_windows.cc
index e2b386be55c..8c8c99cd211 100644
--- a/src/core/lib/support/string_util_windows.cc
+++ b/src/core/lib/gpr/string_util_windows.cc
@@ -36,8 +36,8 @@
 #include <grpc/support/log_windows.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/string_windows.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/string_windows.h"
 
 #if defined UNICODE || defined _UNICODE
 LPTSTR
diff --git a/src/core/lib/support/string_windows.cc b/src/core/lib/gpr/string_windows.cc
similarity index 97%
rename from src/core/lib/support/string_windows.cc
rename to src/core/lib/gpr/string_windows.cc
index ceb78f00543..25bfd412e44 100644
--- a/src/core/lib/support/string_windows.cc
+++ b/src/core/lib/gpr/string_windows.cc
@@ -29,7 +29,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 int gpr_asprintf(char** strp, const char* format, ...) {
   va_list args;
diff --git a/src/core/lib/support/string_windows.h b/src/core/lib/gpr/string_windows.h
similarity index 85%
rename from src/core/lib/support/string_windows.h
rename to src/core/lib/gpr/string_windows.h
index 7c7f31e7aaf..e370f802cf9 100644
--- a/src/core/lib/support/string_windows.h
+++ b/src/core/lib/gpr/string_windows.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H
-#define GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H
+#ifndef GRPC_CORE_LIB_GPR_STRING_WINDOWS_H
+#define GRPC_CORE_LIB_GPR_STRING_WINDOWS_H
 
 #include <grpc/support/port_platform.h>
 
@@ -29,4 +29,4 @@ LPSTR gpr_tchar_to_char(LPCTSTR input);
 
 #endif /* GPR_WINDOWS */
 
-#endif /* GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H */
+#endif /* GRPC_CORE_LIB_GPR_STRING_WINDOWS_H */
diff --git a/src/core/lib/support/subprocess_posix.cc b/src/core/lib/gpr/subprocess_posix.cc
similarity index 100%
rename from src/core/lib/support/subprocess_posix.cc
rename to src/core/lib/gpr/subprocess_posix.cc
diff --git a/src/core/lib/support/subprocess_windows.cc b/src/core/lib/gpr/subprocess_windows.cc
similarity index 97%
rename from src/core/lib/support/subprocess_windows.cc
rename to src/core/lib/gpr/subprocess_windows.cc
index dcdafb5a637..1947d475e34 100644
--- a/src/core/lib/support/subprocess_windows.cc
+++ b/src/core/lib/gpr/subprocess_windows.cc
@@ -27,8 +27,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/subprocess.h>
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/string_windows.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/string_windows.h"
 
 struct gpr_subprocess {
   PROCESS_INFORMATION pi;
diff --git a/src/core/lib/support/sync.cc b/src/core/lib/gpr/sync.cc
similarity index 100%
rename from src/core/lib/support/sync.cc
rename to src/core/lib/gpr/sync.cc
diff --git a/src/core/lib/support/sync_posix.cc b/src/core/lib/gpr/sync_posix.cc
similarity index 100%
rename from src/core/lib/support/sync_posix.cc
rename to src/core/lib/gpr/sync_posix.cc
diff --git a/src/core/lib/support/sync_windows.cc b/src/core/lib/gpr/sync_windows.cc
similarity index 100%
rename from src/core/lib/support/sync_windows.cc
rename to src/core/lib/gpr/sync_windows.cc
diff --git a/src/core/lib/support/thd.cc b/src/core/lib/gpr/thd.cc
similarity index 100%
rename from src/core/lib/support/thd.cc
rename to src/core/lib/gpr/thd.cc
diff --git a/src/core/lib/support/thd_internal.h b/src/core/lib/gpr/thd_internal.h
similarity index 85%
rename from src/core/lib/support/thd_internal.h
rename to src/core/lib/gpr/thd_internal.h
index 38bffc847d6..692c00c8e13 100644
--- a/src/core/lib/support/thd_internal.h
+++ b/src/core/lib/gpr/thd_internal.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H
-#define GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H
+#ifndef GRPC_CORE_LIB_GPR_THD_INTERNAL_H
+#define GRPC_CORE_LIB_GPR_THD_INTERNAL_H
 
 #include <grpc/support/time.h>
 
@@ -27,4 +27,4 @@ void gpr_thd_init();
 /* Wait for all outstanding threads to finish, up to deadline */
 int gpr_await_threads(gpr_timespec deadline);
 
-#endif /* GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H */
+#endif /* GRPC_CORE_LIB_GPR_THD_INTERNAL_H */
diff --git a/src/core/lib/support/thd_posix.cc b/src/core/lib/gpr/thd_posix.cc
similarity index 99%
rename from src/core/lib/support/thd_posix.cc
rename to src/core/lib/gpr/thd_posix.cc
index f0ed48dbfc2..cfff0df6dee 100644
--- a/src/core/lib/support/thd_posix.cc
+++ b/src/core/lib/gpr/thd_posix.cc
@@ -31,7 +31,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "src/core/lib/support/fork.h"
+#include "src/core/lib/gpr/fork.h"
 
 static gpr_mu g_mu;
 static gpr_cv g_cv;
diff --git a/src/core/lib/support/thd_windows.cc b/src/core/lib/gpr/thd_windows.cc
similarity index 100%
rename from src/core/lib/support/thd_windows.cc
rename to src/core/lib/gpr/thd_windows.cc
diff --git a/src/core/lib/support/time.cc b/src/core/lib/gpr/time.cc
similarity index 100%
rename from src/core/lib/support/time.cc
rename to src/core/lib/gpr/time.cc
diff --git a/src/core/lib/support/time_posix.cc b/src/core/lib/gpr/time_posix.cc
similarity index 98%
rename from src/core/lib/support/time_posix.cc
rename to src/core/lib/gpr/time_posix.cc
index b2087c93cfe..9c7e86b080b 100644
--- a/src/core/lib/support/time_posix.cc
+++ b/src/core/lib/gpr/time_posix.cc
@@ -17,7 +17,7 @@
  */
 
 #include <grpc/support/port_platform.h>
-#include "src/core/lib/support/time_precise.h"
+#include "src/core/lib/gpr/time_precise.h"
 
 #ifdef GPR_POSIX_TIME
 
diff --git a/src/core/lib/support/time_precise.cc b/src/core/lib/gpr/time_precise.cc
similarity index 97%
rename from src/core/lib/support/time_precise.cc
rename to src/core/lib/gpr/time_precise.cc
index b7372df1b81..3c7aaabc402 100644
--- a/src/core/lib/support/time_precise.cc
+++ b/src/core/lib/gpr/time_precise.cc
@@ -20,7 +20,7 @@
 #include <grpc/support/time.h>
 #include <stdio.h>
 
-#include "src/core/lib/support/time_precise.h"
+#include "src/core/lib/gpr/time_precise.h"
 
 #ifdef GRPC_TIMERS_RDTSC
 #if defined(__i386__)
diff --git a/src/core/lib/support/time_precise.h b/src/core/lib/gpr/time_precise.h
similarity index 83%
rename from src/core/lib/support/time_precise.h
rename to src/core/lib/gpr/time_precise.h
index 35cd154dbdf..acc4ee3d1b5 100644
--- a/src/core/lib/support/time_precise.h
+++ b/src/core/lib/gpr/time_precise.h
@@ -16,12 +16,12 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_TIME_PRECISE_H
-#define GRPC_CORE_LIB_SUPPORT_TIME_PRECISE_H
+#ifndef GRPC_CORE_LIB_GPR_TIME_PRECISE_H
+#define GRPC_CORE_LIB_GPR_TIME_PRECISE_H
 
 #include <grpc/support/time.h>
 
 void gpr_precise_clock_init(void);
 void gpr_precise_clock_now(gpr_timespec* clk);
 
-#endif /* GRPC_CORE_LIB_SUPPORT_TIME_PRECISE_H */
+#endif /* GRPC_CORE_LIB_GPR_TIME_PRECISE_H */
diff --git a/src/core/lib/support/time_windows.cc b/src/core/lib/gpr/time_windows.cc
similarity index 98%
rename from src/core/lib/support/time_windows.cc
rename to src/core/lib/gpr/time_windows.cc
index fb17e5c079f..247cc164682 100644
--- a/src/core/lib/support/time_windows.cc
+++ b/src/core/lib/gpr/time_windows.cc
@@ -28,7 +28,7 @@
 #include <process.h>
 #include <sys/timeb.h>
 
-#include "src/core/lib/support/time_precise.h"
+#include "src/core/lib/gpr/time_precise.h"
 
 static LARGE_INTEGER g_start_time;
 static double g_time_scale;
diff --git a/src/core/lib/support/tls_pthread.cc b/src/core/lib/gpr/tls_pthread.cc
similarity index 100%
rename from src/core/lib/support/tls_pthread.cc
rename to src/core/lib/gpr/tls_pthread.cc
diff --git a/src/core/lib/support/tmpfile.h b/src/core/lib/gpr/tmpfile.h
similarity index 88%
rename from src/core/lib/support/tmpfile.h
rename to src/core/lib/gpr/tmpfile.h
index c5ceda86751..f47ec7aa637 100644
--- a/src/core/lib/support/tmpfile.h
+++ b/src/core/lib/gpr/tmpfile.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SUPPORT_TMPFILE_H
-#define GRPC_CORE_LIB_SUPPORT_TMPFILE_H
+#ifndef GRPC_CORE_LIB_GPR_TMPFILE_H
+#define GRPC_CORE_LIB_GPR_TMPFILE_H
 
 #include <stdio.h>
 
@@ -27,4 +27,4 @@
    unless an error occurs in which case it will be set to NULL. */
 FILE* gpr_tmpfile(const char* prefix, char** tmp_filename);
 
-#endif /* GRPC_CORE_LIB_SUPPORT_TMPFILE_H */
+#endif /* GRPC_CORE_LIB_GPR_TMPFILE_H */
diff --git a/src/core/lib/support/tmpfile_msys.cc b/src/core/lib/gpr/tmpfile_msys.cc
similarity index 94%
rename from src/core/lib/support/tmpfile_msys.cc
rename to src/core/lib/gpr/tmpfile_msys.cc
index 430e866629f..76cd886f3a0 100644
--- a/src/core/lib/support/tmpfile_msys.cc
+++ b/src/core/lib/gpr/tmpfile_msys.cc
@@ -29,8 +29,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/support/string_windows.h"
-#include "src/core/lib/support/tmpfile.h"
+#include "src/core/lib/gpr/string_windows.h"
+#include "src/core/lib/gpr/tmpfile.h"
 
 FILE* gpr_tmpfile(const char* prefix, char** tmp_filename_out) {
   FILE* result = NULL;
diff --git a/src/core/lib/support/tmpfile_posix.cc b/src/core/lib/gpr/tmpfile_posix.cc
similarity index 95%
rename from src/core/lib/support/tmpfile_posix.cc
rename to src/core/lib/gpr/tmpfile_posix.cc
index 79c5c688742..ffdad335d67 100644
--- a/src/core/lib/support/tmpfile_posix.cc
+++ b/src/core/lib/gpr/tmpfile_posix.cc
@@ -20,7 +20,7 @@
 
 #ifdef GPR_POSIX_TMPFILE
 
-#include "src/core/lib/support/tmpfile.h"
+#include "src/core/lib/gpr/tmpfile.h"
 
 #include <errno.h>
 #include <stdlib.h>
@@ -31,7 +31,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 FILE* gpr_tmpfile(const char* prefix, char** tmp_filename) {
   FILE* result = nullptr;
diff --git a/src/core/lib/support/tmpfile_windows.cc b/src/core/lib/gpr/tmpfile_windows.cc
similarity index 95%
rename from src/core/lib/support/tmpfile_windows.cc
rename to src/core/lib/gpr/tmpfile_windows.cc
index 2b10bcde437..d4868084189 100644
--- a/src/core/lib/support/tmpfile_windows.cc
+++ b/src/core/lib/gpr/tmpfile_windows.cc
@@ -29,8 +29,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/support/string_windows.h"
-#include "src/core/lib/support/tmpfile.h"
+#include "src/core/lib/gpr/string_windows.h"
+#include "src/core/lib/gpr/tmpfile.h"
 
 FILE* gpr_tmpfile(const char* prefix, char** tmp_filename_out) {
   FILE* result = NULL;
diff --git a/src/core/lib/support/wrap_memcpy.cc b/src/core/lib/gpr/wrap_memcpy.cc
similarity index 100%
rename from src/core/lib/support/wrap_memcpy.cc
rename to src/core/lib/gpr/wrap_memcpy.cc
diff --git a/src/core/lib/http/format_request.cc b/src/core/lib/http/format_request.cc
index f3f3cbda7b2..473fa71790a 100644
--- a/src/core/lib/http/format_request.cc
+++ b/src/core/lib/http/format_request.cc
@@ -26,7 +26,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 static void fill_common_header(const grpc_httpcli_request* request,
                                gpr_strvec* buf, bool connection_close) {
diff --git a/src/core/lib/http/httpcli.cc b/src/core/lib/http/httpcli.cc
index ed874c4265c..c43c92b57d5 100644
--- a/src/core/lib/http/httpcli.cc
+++ b/src/core/lib/http/httpcli.cc
@@ -26,6 +26,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/http/format_request.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/endpoint.h"
@@ -34,7 +35,6 @@
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/tcp_client.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/string.h"
 
 typedef struct {
   grpc_slice request_text;
diff --git a/src/core/lib/http/httpcli_security_connector.cc b/src/core/lib/http/httpcli_security_connector.cc
index bfb536a9211..86644181441 100644
--- a/src/core/lib/http/httpcli_security_connector.cc
+++ b/src/core/lib/http/httpcli_security_connector.cc
@@ -26,9 +26,9 @@
 
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/handshaker_registry.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/security/transport/security_handshaker.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/tsi/ssl_transport_security.h"
 #include "src/core/tsi/transport_security_adapter.h"
 
diff --git a/src/core/lib/iomgr/call_combiner.h b/src/core/lib/iomgr/call_combiner.h
index 9f7e6ce1c94..4814dbf8d21 100644
--- a/src/core/lib/iomgr/call_combiner.h
+++ b/src/core/lib/iomgr/call_combiner.h
@@ -23,9 +23,9 @@
 
 #include <grpc/support/atm.h>
 
+#include "src/core/lib/gpr/mpscq.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
-#include "src/core/lib/support/mpscq.h"
 
 // A simple, lock-free mechanism for serializing activity related to a
 // single call.  This is similar to a combiner but is more lightweight.
diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h
index 4c58c0e4bf3..249fca6cd6e 100644
--- a/src/core/lib/iomgr/closure.h
+++ b/src/core/lib/iomgr/closure.h
@@ -25,9 +25,9 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <stdbool.h>
+#include "src/core/lib/gpr/mpscq.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/mpscq.h"
 
 struct grpc_closure;
 typedef struct grpc_closure grpc_closure;
diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h
index 46b9ac58be4..c62d21a0513 100644
--- a/src/core/lib/iomgr/combiner.h
+++ b/src/core/lib/iomgr/combiner.h
@@ -23,8 +23,8 @@
 
 #include <grpc/support/atm.h>
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gpr/mpscq.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
-#include "src/core/lib/support/mpscq.h"
 
 // Provides serialized access to some resource.
 // Each action queued on a combiner is executed serially in a borrowed thread.
diff --git a/src/core/lib/iomgr/endpoint_pair_posix.cc b/src/core/lib/iomgr/endpoint_pair_posix.cc
index 0b4aefd1b70..3ad6b477561 100644
--- a/src/core/lib/iomgr/endpoint_pair_posix.cc
+++ b/src/core/lib/iomgr/endpoint_pair_posix.cc
@@ -33,8 +33,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/tcp_posix.h"
-#include "src/core/lib/support/string.h"
 
 static void create_sockets(int sv[2]) {
   int flags;
diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc
index aa14d5931a3..1cb0150f450 100644
--- a/src/core/lib/iomgr/ev_epoll1_linux.cc
+++ b/src/core/lib/iomgr/ev_epoll1_linux.cc
@@ -43,14 +43,14 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr++/manual_constructor.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/lockfree_event.h"
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
 #include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/manual_constructor.h"
-#include "src/core/lib/support/string.h"
 
 static grpc_wakeup_fd global_wakeup_fd;
 
diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc
index e4a2d67e4b4..b81c00ca7aa 100644
--- a/src/core/lib/iomgr/ev_epollex_linux.cc
+++ b/src/core/lib/iomgr/ev_epollex_linux.cc
@@ -41,6 +41,8 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr++/manual_constructor.h"
+#include "src/core/lib/gpr/spinlock.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/is_epollexclusive_available.h"
@@ -49,8 +51,6 @@
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
 #include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/manual_constructor.h"
-#include "src/core/lib/support/spinlock.h"
 
 // debug aid: create workers on the heap (allows asan to spot
 // use-after-destruction)
diff --git a/src/core/lib/iomgr/ev_epollsig_linux.cc b/src/core/lib/iomgr/ev_epollsig_linux.cc
index 3544d4f3a4d..11c64d080c7 100644
--- a/src/core/lib/iomgr/ev_epollsig_linux.cc
+++ b/src/core/lib/iomgr/ev_epollsig_linux.cc
@@ -43,6 +43,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr++/manual_constructor.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
@@ -50,7 +51,6 @@
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
 #include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/manual_constructor.h"
 
 #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker*)1)
 
diff --git a/src/core/lib/iomgr/ev_poll_posix.cc b/src/core/lib/iomgr/ev_poll_posix.cc
index 08edff5159b..3b797280559 100644
--- a/src/core/lib/iomgr/ev_poll_posix.cc
+++ b/src/core/lib/iomgr/ev_poll_posix.cc
@@ -38,12 +38,12 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr/murmur_hash.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/wakeup_fd_cv.h"
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
 #include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/murmur_hash.h"
 
 #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker*)1)
 
diff --git a/src/core/lib/iomgr/ev_posix.cc b/src/core/lib/iomgr/ev_posix.cc
index b516f93058a..3a5714132d0 100644
--- a/src/core/lib/iomgr/ev_posix.cc
+++ b/src/core/lib/iomgr/ev_posix.cc
@@ -30,11 +30,11 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/iomgr/ev_epoll1_linux.h"
 #include "src/core/lib/iomgr/ev_epollex_linux.h"
 #include "src/core/lib/iomgr/ev_epollsig_linux.h"
 #include "src/core/lib/iomgr/ev_poll_posix.h"
-#include "src/core/lib/support/env.h"
 
 grpc_core::TraceFlag grpc_polling_trace(false,
                                         "polling"); /* Disabled by default */
diff --git a/src/core/lib/iomgr/executor.cc b/src/core/lib/iomgr/executor.cc
index 67a0412dc38..835dc9d0f7e 100644
--- a/src/core/lib/iomgr/executor.cc
+++ b/src/core/lib/iomgr/executor.cc
@@ -29,8 +29,8 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr/spinlock.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
-#include "src/core/lib/support/spinlock.h"
 
 #define MAX_DEPTH 2
 
diff --git a/src/core/lib/iomgr/fork_posix.cc b/src/core/lib/iomgr/fork_posix.cc
index cc131408af7..9bfa79219af 100644
--- a/src/core/lib/iomgr/fork_posix.cc
+++ b/src/core/lib/iomgr/fork_posix.cc
@@ -27,13 +27,13 @@
 #include <grpc/support/thd.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/fork.h"
+#include "src/core/lib/gpr/thd_internal.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/timer_manager.h"
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/fork.h"
-#include "src/core/lib/support/thd_internal.h"
 #include "src/core/lib/surface/init.h"
 
 /*
diff --git a/src/core/lib/iomgr/iomgr.cc b/src/core/lib/iomgr/iomgr.cc
index 70807c479d7..4cd0a4f1de7 100644
--- a/src/core/lib/iomgr/iomgr.cc
+++ b/src/core/lib/iomgr/iomgr.cc
@@ -31,14 +31,14 @@
 #include <grpc/support/thd.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/network_status_tracker.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/iomgr/timer_manager.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
 
 static gpr_mu g_mu;
 static gpr_cv g_rcv;
diff --git a/src/core/lib/iomgr/load_file.cc b/src/core/lib/iomgr/load_file.cc
index 4a05de1410c..b6586fbc732 100644
--- a/src/core/lib/iomgr/load_file.cc
+++ b/src/core/lib/iomgr/load_file.cc
@@ -25,8 +25,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/block_annotate.h"
-#include "src/core/lib/support/string.h"
 
 grpc_error* grpc_load_file(const char* filename, int add_null_terminator,
                            grpc_slice* output) {
diff --git a/src/core/lib/iomgr/resolve_address_posix.cc b/src/core/lib/iomgr/resolve_address_posix.cc
index cc3d4fd7cfa..176caee4907 100644
--- a/src/core/lib/iomgr/resolve_address_posix.cc
+++ b/src/core/lib/iomgr/resolve_address_posix.cc
@@ -33,11 +33,11 @@
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
-#include "src/core/lib/support/string.h"
 
 static grpc_error* blocking_resolve_address_impl(
     const char* name, const char* default_port,
diff --git a/src/core/lib/iomgr/resolve_address_windows.cc b/src/core/lib/iomgr/resolve_address_windows.cc
index ccb1dae689c..e44ab396b35 100644
--- a/src/core/lib/iomgr/resolve_address_windows.cc
+++ b/src/core/lib/iomgr/resolve_address_windows.cc
@@ -34,11 +34,11 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
-#include "src/core/lib/support/string.h"
 
 typedef struct {
   char* name;
diff --git a/src/core/lib/iomgr/sockaddr_utils.cc b/src/core/lib/iomgr/sockaddr_utils.cc
index 0c0a2fe5b2e..71e3e386243 100644
--- a/src/core/lib/iomgr/sockaddr_utils.cc
+++ b/src/core/lib/iomgr/sockaddr_utils.cc
@@ -28,10 +28,10 @@
 #include <grpc/support/port_platform.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/socket_utils.h"
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
-#include "src/core/lib/support/string.h"
 
 static const uint8_t kV4MappedPrefix[] = {0, 0, 0, 0, 0,    0,
                                           0, 0, 0, 0, 0xff, 0xff};
diff --git a/src/core/lib/iomgr/socket_utils_common_posix.cc b/src/core/lib/iomgr/socket_utils_common_posix.cc
index 2d4b8f0adda..5068a8081d3 100644
--- a/src/core/lib/iomgr/socket_utils_common_posix.cc
+++ b/src/core/lib/iomgr/socket_utils_common_posix.cc
@@ -40,8 +40,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/sync.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
-#include "src/core/lib/support/string.h"
 
 /* set a socket to non blocking mode */
 grpc_error* grpc_set_socket_nonblocking(int fd, int non_blocking) {
diff --git a/src/core/lib/iomgr/tcp_client_posix.cc b/src/core/lib/iomgr/tcp_client_posix.cc
index 8cd5f8d6183..3dff6240654 100644
--- a/src/core/lib/iomgr/tcp_client_posix.cc
+++ b/src/core/lib/iomgr/tcp_client_posix.cc
@@ -33,6 +33,7 @@
 #include <grpc/support/time.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_posix.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
@@ -41,7 +42,6 @@
 #include "src/core/lib/iomgr/tcp_posix.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
-#include "src/core/lib/support/string.h"
 
 extern grpc_core::TraceFlag grpc_tcp_trace;
 
diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc
index d47a077251a..0ec5926227c 100644
--- a/src/core/lib/iomgr/tcp_posix.cc
+++ b/src/core/lib/iomgr/tcp_posix.cc
@@ -42,12 +42,12 @@
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 
 #ifdef GRPC_HAVE_MSG_NOSIGNAL
 #define SENDMSG_FLAGS MSG_NOSIGNAL
diff --git a/src/core/lib/iomgr/tcp_server_posix.cc b/src/core/lib/iomgr/tcp_server_posix.cc
index 99e1c6cd069..2fa00a81d2c 100644
--- a/src/core/lib/iomgr/tcp_server_posix.cc
+++ b/src/core/lib/iomgr/tcp_server_posix.cc
@@ -45,6 +45,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
@@ -52,7 +53,6 @@
 #include "src/core/lib/iomgr/tcp_posix.h"
 #include "src/core/lib/iomgr/tcp_server_utils_posix.h"
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
-#include "src/core/lib/support/string.h"
 
 static gpr_once check_init = GPR_ONCE_INIT;
 static bool has_so_reuseport = false;
diff --git a/src/core/lib/iomgr/tcp_uv.cc b/src/core/lib/iomgr/tcp_uv.cc
index baa49d5cc5d..b384623a5e9 100644
--- a/src/core/lib/iomgr/tcp_uv.cc
+++ b/src/core/lib/iomgr/tcp_uv.cc
@@ -29,6 +29,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/network_status_tracker.h"
@@ -36,7 +37,6 @@
 #include "src/core/lib/iomgr/tcp_uv.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 
 grpc_core::TraceFlag grpc_tcp_trace(false, "tcp");
 
diff --git a/src/core/lib/iomgr/timer_generic.cc b/src/core/lib/iomgr/timer_generic.cc
index 103144eb3bf..177bdec8df1 100644
--- a/src/core/lib/iomgr/timer_generic.cc
+++ b/src/core/lib/iomgr/timer_generic.cc
@@ -32,9 +32,9 @@
 #include <grpc/support/tls.h>
 #include <grpc/support/useful.h>
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gpr/spinlock.h"
 #include "src/core/lib/iomgr/time_averaged_stats.h"
 #include "src/core/lib/iomgr/timer_heap.h"
-#include "src/core/lib/support/spinlock.h"
 
 #define INVALID_HEAP_INDEX 0xffffffffu
 
diff --git a/src/core/lib/iomgr/udp_server.cc b/src/core/lib/iomgr/udp_server.cc
index 946846a8b87..27d32c59aeb 100644
--- a/src/core/lib/iomgr/udp_server.cc
+++ b/src/core/lib/iomgr/udp_server.cc
@@ -49,6 +49,7 @@
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/executor.h"
@@ -58,7 +59,6 @@
 #include "src/core/lib/iomgr/socket_factory_posix.h"
 #include "src/core/lib/iomgr/socket_utils_posix.h"
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
-#include "src/core/lib/support/string.h"
 
 /* one listening port */
 typedef struct grpc_udp_listener grpc_udp_listener;
diff --git a/src/core/lib/profiling/basic_timers.cc b/src/core/lib/profiling/basic_timers.cc
index 87dd4ab120b..d1c9fd7decb 100644
--- a/src/core/lib/profiling/basic_timers.cc
+++ b/src/core/lib/profiling/basic_timers.cc
@@ -30,7 +30,7 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 
 typedef enum { BEGIN = '{', END = '}', MARK = '.' } marker_type;
 
diff --git a/src/core/lib/security/context/security_context.cc b/src/core/lib/security/context/security_context.cc
index 03710279948..63ec42cf862 100644
--- a/src/core/lib/security/context/security_context.cc
+++ b/src/core/lib/security/context/security_context.cc
@@ -19,8 +19,8 @@
 #include <string.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/security/context/security_context.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 
diff --git a/src/core/lib/security/credentials/credentials.cc b/src/core/lib/security/credentials/credentials.cc
index 48b459e1be9..009a5ce4ec3 100644
--- a/src/core/lib/security/credentials/credentials.cc
+++ b/src/core/lib/security/credentials/credentials.cc
@@ -22,11 +22,11 @@
 #include <string.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/json/json.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 
 #include <grpc/support/alloc.h>
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.cc b/src/core/lib/security/credentials/fake/fake_credentials.cc
index 99b1214951d..9065325cdd9 100644
--- a/src/core/lib/security/credentials/fake/fake_credentials.cc
+++ b/src/core/lib/security/credentials/fake/fake_credentials.cc
@@ -25,8 +25,8 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/executor.h"
-#include "src/core/lib/support/string.h"
 
 /* -- Fake transport security credentials. -- */
 
diff --git a/src/core/lib/security/credentials/google_default/credentials_generic.cc b/src/core/lib/security/credentials/google_default/credentials_generic.cc
index af103f5dc56..15ae9d64280 100644
--- a/src/core/lib/security/credentials/google_default/credentials_generic.cc
+++ b/src/core/lib/security/credentials/google_default/credentials_generic.cc
@@ -22,8 +22,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 
 char* grpc_get_well_known_google_credentials_file_path_impl(void) {
   char* result = nullptr;
diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.cc b/src/core/lib/security/credentials/google_default/google_default_credentials.cc
index 03d52850d90..dc192020335 100644
--- a/src/core/lib/security/credentials/google_default/google_default_credentials.cc
+++ b/src/core/lib/security/credentials/google_default/google_default_credentials.cc
@@ -24,6 +24,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/load_file.h"
@@ -33,8 +35,6 @@
 #include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 
 /* -- Constants. -- */
diff --git a/src/core/lib/security/credentials/jwt/json_token.cc b/src/core/lib/security/credentials/jwt/json_token.cc
index a152ddcaaf6..078f04ed118 100644
--- a/src/core/lib/security/credentials/jwt/json_token.cc
+++ b/src/core/lib/security/credentials/jwt/json_token.cc
@@ -26,9 +26,9 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/security/util/json_util.h"
 #include "src/core/lib/slice/b64.h"
-#include "src/core/lib/support/string.h"
 
 extern "C" {
 #include <openssl/bio.h>
diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.cc b/src/core/lib/security/credentials/jwt/jwt_verifier.cc
index 39339f07d73..860506df39b 100644
--- a/src/core/lib/security/credentials/jwt/jwt_verifier.cc
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.cc
@@ -31,11 +31,11 @@ extern "C" {
 #include <openssl/pem.h>
 }
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/slice/b64.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/tsi/ssl_types.h"
 
 /* --- Utils. --- */
diff --git a/src/core/lib/security/transport/client_auth_filter.cc b/src/core/lib/security/transport/client_auth_filter.cc
index 6d7f89873b3..045cb3e2396 100644
--- a/src/core/lib/security/transport/client_auth_filter.cc
+++ b/src/core/lib/security/transport/client_auth_filter.cc
@@ -25,13 +25,13 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/transport/static_metadata.h"
 
diff --git a/src/core/lib/security/transport/secure_endpoint.cc b/src/core/lib/security/transport/secure_endpoint.cc
index e5c089de9ca..bd8a6cd76aa 100644
--- a/src/core/lib/security/transport/secure_endpoint.cc
+++ b/src/core/lib/security/transport/secure_endpoint.cc
@@ -28,12 +28,12 @@
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/security/transport/tsi_error.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/tsi/transport_security_grpc.h"
 
 #define STAGING_BUFFER_SIZE 8192
diff --git a/src/core/lib/security/transport/security_connector.cc b/src/core/lib/security/transport/security_connector.cc
index fd139714da2..1d962f94b28 100644
--- a/src/core/lib/security/transport/security_connector.cc
+++ b/src/core/lib/security/transport/security_connector.cc
@@ -30,6 +30,8 @@
 #include "src/core/ext/transport/chttp2/alpn/alpn.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/handshaker.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/load_file.h"
 #include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/security/credentials/credentials.h"
@@ -38,8 +40,6 @@
 #include "src/core/lib/security/transport/lb_targets_info.h"
 #include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/security/transport/security_handshaker.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/tsi/fake_transport_security.h"
 #include "src/core/tsi/ssl_transport_security.h"
 #include "src/core/tsi/transport_security_adapter.h"
diff --git a/src/core/lib/slice/slice_intern.cc b/src/core/lib/slice/slice_intern.cc
index c578c6d9de1..fe1770b92c4 100644
--- a/src/core/lib/slice/slice_intern.cc
+++ b/src/core/lib/slice/slice_intern.cc
@@ -24,10 +24,10 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
+#include "src/core/lib/gpr/murmur_hash.h"
 #include "src/core/lib/iomgr/iomgr_internal.h" /* for iomgr_abort_on_leaks() */
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/murmur_hash.h"
 #include "src/core/lib/transport/static_metadata.h"
 
 #define LOG2_SHARD_COUNT 5
diff --git a/src/core/lib/slice/slice_string_helpers.cc b/src/core/lib/slice/slice_string_helpers.cc
index 5385be9866b..be0db09252d 100644
--- a/src/core/lib/slice/slice_string_helpers.cc
+++ b/src/core/lib/slice/slice_string_helpers.cc
@@ -22,8 +22,8 @@
 
 #include <grpc/support/log.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/string.h"
 
 char* grpc_dump_slice(grpc_slice s, uint32_t flags) {
   return gpr_dump((const char*)GRPC_SLICE_START_PTR(s), GRPC_SLICE_LENGTH(s),
diff --git a/src/core/lib/slice/slice_string_helpers.h b/src/core/lib/slice/slice_string_helpers.h
index 7f51b11b9cf..109084be1f7 100644
--- a/src/core/lib/slice/slice_string_helpers.h
+++ b/src/core/lib/slice/slice_string_helpers.h
@@ -26,7 +26,7 @@
 #include <grpc/slice_buffer.h>
 #include <grpc/support/port_platform.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 /* Calls gpr_dump on a slice. */
 char* grpc_dump_slice(grpc_slice slice, uint32_t flags);
diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc
index d677576c145..8a579cabe7d 100644
--- a/src/core/lib/surface/call.cc
+++ b/src/core/lib/surface/call.cc
@@ -33,12 +33,12 @@
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/compression/algorithm_metadata.h"
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr/arena.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/arena.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/call_test_only.h"
diff --git a/src/core/lib/surface/call_log_batch.cc b/src/core/lib/surface/call_log_batch.cc
index 535a3d32821..d56ea2a9328 100644
--- a/src/core/lib/surface/call_log_batch.cc
+++ b/src/core/lib/surface/call_log_batch.cc
@@ -22,8 +22,8 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 
 static void add_metadata(gpr_strvec* b, const grpc_metadata* md, size_t count) {
   size_t i;
diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc
index cf5e8c2150d..62db82a1d9e 100644
--- a/src/core/lib/surface/channel.cc
+++ b/src/core/lib/surface/channel.cc
@@ -29,9 +29,9 @@
 
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel_init.h"
diff --git a/src/core/lib/surface/completion_queue.cc b/src/core/lib/surface/completion_queue.cc
index aa5808da4c2..e731e2139fd 100644
--- a/src/core/lib/surface/completion_queue.cc
+++ b/src/core/lib/surface/completion_queue.cc
@@ -31,11 +31,11 @@
 #include <grpc/support/tls.h>
 
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr/spinlock.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/spinlock.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/event_string.h"
diff --git a/src/core/lib/surface/event_string.cc b/src/core/lib/surface/event_string.cc
index 5168edc9b50..7f40bb2405e 100644
--- a/src/core/lib/surface/event_string.cc
+++ b/src/core/lib/surface/event_string.cc
@@ -22,7 +22,7 @@
 
 #include <grpc/byte_buffer.h>
 #include <grpc/support/string_util.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 static void addhdr(gpr_strvec* buf, grpc_event* ev) {
   char* tmp;
diff --git a/src/core/lib/surface/init.cc b/src/core/lib/surface/init.cc
index 0f40965f16c..70329b09f43 100644
--- a/src/core/lib/surface/init.cc
+++ b/src/core/lib/surface/init.cc
@@ -31,6 +31,8 @@
 #include "src/core/lib/channel/handshaker_registry.h"
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gpr/fork.h"
+#include "src/core/lib/gpr/thd_internal.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/call_combiner.h"
 #include "src/core/lib/iomgr/combiner.h"
@@ -40,8 +42,6 @@
 #include "src/core/lib/iomgr/timer_manager.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/fork.h"
-#include "src/core/lib/support/thd_internal.h"
 #include "src/core/lib/surface/alarm_internal.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
diff --git a/src/core/lib/surface/lame_client.cc b/src/core/lib/surface/lame_client.cc
index 08611ff730f..27a2a4eeb6f 100644
--- a/src/core/lib/surface/lame_client.cc
+++ b/src/core/lib/surface/lame_client.cc
@@ -23,10 +23,10 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/lib/support/atomic.h"
+#include "src/core/lib/gpr++/atomic.h"
 
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc
index ee98cf2693a..c8c1db337f9 100644
--- a/src/core/lib/surface/server.cc
+++ b/src/core/lib/surface/server.cc
@@ -30,12 +30,12 @@
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/connected_channel.h"
 #include "src/core/lib/debug/stats.h"
+#include "src/core/lib/gpr/mpscq.h"
+#include "src/core/lib/gpr/spinlock.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/mpscq.h"
-#include "src/core/lib/support/spinlock.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
diff --git a/src/core/lib/transport/metadata.cc b/src/core/lib/transport/metadata.cc
index 5f0673e0142..652222bd0b8 100644
--- a/src/core/lib/transport/metadata.cc
+++ b/src/core/lib/transport/metadata.cc
@@ -31,12 +31,12 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
+#include "src/core/lib/gpr/murmur_hash.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/murmur_hash.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/static_metadata.h"
 
 /* There are two kinds of mdelem and mdstr instances.
diff --git a/src/core/lib/transport/service_config.cc b/src/core/lib/transport/service_config.cc
index cbafc338404..5c9930a273f 100644
--- a/src/core/lib/transport/service_config.cc
+++ b/src/core/lib/transport/service_config.cc
@@ -23,11 +23,11 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/json/json.h"
 #include "src/core/lib/slice/slice_hash_table.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 
 // The main purpose of the code here is to parse the service config in
 // JSON form, which will look like this:
diff --git a/src/core/lib/transport/timeout_encoding.cc b/src/core/lib/transport/timeout_encoding.cc
index 86db6c83443..47f964af454 100644
--- a/src/core/lib/transport/timeout_encoding.cc
+++ b/src/core/lib/transport/timeout_encoding.cc
@@ -22,7 +22,7 @@
 #include <string.h>
 
 #include <grpc/support/port_platform.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 static int64_t round_up(int64_t x, int64_t divisor) {
   return (x / divisor + (x % divisor != 0)) * divisor;
diff --git a/src/core/lib/transport/timeout_encoding.h b/src/core/lib/transport/timeout_encoding.h
index 8611f49b009..4e9268262e4 100644
--- a/src/core/lib/transport/timeout_encoding.h
+++ b/src/core/lib/transport/timeout_encoding.h
@@ -22,8 +22,8 @@
 #include <grpc/slice.h>
 #include <grpc/support/time.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
-#include "src/core/lib/support/string.h"
 
 #define GRPC_HTTP2_TIMEOUT_ENCODE_MIN_BUFSIZE (GPR_LTOA_MIN_BUFSIZE + 1)
 
diff --git a/src/core/lib/transport/transport.cc b/src/core/lib/transport/transport.cc
index 08aee04ac94..ea0380e5912 100644
--- a/src/core/lib/transport/transport.cc
+++ b/src/core/lib/transport/transport.cc
@@ -25,10 +25,10 @@
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/transport_impl.h"
 
 grpc_core::DebugOnlyTraceFlag grpc_trace_stream_refcount(false,
diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h
index 80a7ff98bef..b392c696cda 100644
--- a/src/core/lib/transport/transport.h
+++ b/src/core/lib/transport/transport.h
@@ -22,12 +22,12 @@
 #include <stddef.h>
 
 #include "src/core/lib/channel/context.h"
+#include "src/core/lib/gpr/arena.h"
 #include "src/core/lib/iomgr/call_combiner.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/pollset_set.h"
-#include "src/core/lib/support/arena.h"
 #include "src/core/lib/transport/byte_stream.h"
 #include "src/core/lib/transport/metadata_batch.h"
 
diff --git a/src/core/lib/transport/transport_op_string.cc b/src/core/lib/transport/transport_op_string.cc
index c0f82fea0dc..58a21e9b608 100644
--- a/src/core/lib/transport/transport_op_string.cc
+++ b/src/core/lib/transport/transport_op_string.cc
@@ -28,8 +28,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
 /* These routines are here to facilitate debugging - they produce string
diff --git a/src/cpp/client/channel_cc.cc b/src/cpp/client/channel_cc.cc
index 08420817c87..362dff642cf 100644
--- a/src/cpp/client/channel_cc.cc
+++ b/src/cpp/client/channel_cc.cc
@@ -42,9 +42,9 @@
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
 
 namespace grpc {
 
diff --git a/src/csharp/ext/grpc_csharp_ext.c b/src/csharp/ext/grpc_csharp_ext.c
index 6875d40aa08..eb69b5829cd 100644
--- a/src/csharp/ext/grpc_csharp_ext.c
+++ b/src/csharp/ext/grpc_csharp_ext.c
@@ -16,7 +16,7 @@
  *
  */
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 #include <grpc/byte_buffer_reader.h>
 #include <grpc/grpc.h>
diff --git a/src/objective-c/examples/Sample/Podfile b/src/objective-c/examples/Sample/Podfile
index f6f0c00d5d0..9ea2f61927c 100644
--- a/src/objective-c/examples/Sample/Podfile
+++ b/src/objective-c/examples/Sample/Podfile
@@ -40,7 +40,7 @@ pre_install do |installer|
     'HEADER_SEARCH_PATHS' => '"$(inherited)" "$(GRPC_SRC_ROOT)/include"',
     'USER_HEADER_SEARCH_PATHS' => '"$(GRPC_SRC_ROOT)"',
     # If we don't set these two settings, `include/grpc/support/time.h` and
-    # `src/core/lib/support/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
+    # `src/core/lib/gpr/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
     # build.
     'USE_HEADERMAP' => 'NO',
     'ALWAYS_SEARCH_USER_PATHS' => 'NO',
diff --git a/src/objective-c/examples/SwiftSample/Podfile b/src/objective-c/examples/SwiftSample/Podfile
index b08a346ae2a..a2c2b82cc9c 100644
--- a/src/objective-c/examples/SwiftSample/Podfile
+++ b/src/objective-c/examples/SwiftSample/Podfile
@@ -40,7 +40,7 @@ pre_install do |installer|
     'HEADER_SEARCH_PATHS' => '"$(inherited)" "$(GRPC_SRC_ROOT)/include"',
     'USER_HEADER_SEARCH_PATHS' => '"$(GRPC_SRC_ROOT)"',
     # If we don't set these two settings, `include/grpc/support/time.h` and
-    # `src/core/lib/support/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
+    # `src/core/lib/gpr/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
     # build.
     'USE_HEADERMAP' => 'NO',
     'ALWAYS_SEARCH_USER_PATHS' => 'NO',
diff --git a/src/objective-c/tests/Connectivity/Podfile b/src/objective-c/tests/Connectivity/Podfile
index 27ff935c544..cdbc6dde59f 100644
--- a/src/objective-c/tests/Connectivity/Podfile
+++ b/src/objective-c/tests/Connectivity/Podfile
@@ -24,7 +24,7 @@ pre_install do |installer|
     'HEADER_SEARCH_PATHS' => '"$(inherited)" "$(GRPC_SRC_ROOT)/include"',
     'USER_HEADER_SEARCH_PATHS' => '"$(GRPC_SRC_ROOT)"',
     # If we don't set these two settings, `include/grpc/support/time.h` and
-    # `src/core/lib/support/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
+    # `src/core/lib/gpr/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
     # build.
     'USE_HEADERMAP' => 'NO',
     'ALWAYS_SEARCH_USER_PATHS' => 'NO',
diff --git a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.mm b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.mm
index d1309713644..16940a49179 100644
--- a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.mm
+++ b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.mm
@@ -39,9 +39,9 @@
 
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/security/credentials/credentials.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/tmpfile.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/tmpfile.h"
 #include "test/core/end2end/data/ssl_test_data.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m
index 92bc20e5b92..09ee062596c 100644
--- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m
+++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m
@@ -31,9 +31,9 @@
 #import <grpc/support/log.h>
 
 #import "src/core/lib/channel/channel_args.h"
-#import "src/core/lib/support/env.h"
-#import "src/core/lib/support/string.h"
-#import "src/core/lib/support/tmpfile.h"
+#import "src/core/lib/gpr/env.h"
+#import "src/core/lib/gpr/string.h"
+#import "src/core/lib/gpr/tmpfile.h"
 #import "test/core/end2end/data/ssl_test_data.h"
 #import "test/core/util/test_config.h"
 
diff --git a/src/objective-c/tests/Podfile b/src/objective-c/tests/Podfile
index 8f1cb041d86..9e9db1fe6ce 100644
--- a/src/objective-c/tests/Podfile
+++ b/src/objective-c/tests/Podfile
@@ -75,7 +75,7 @@ pre_install do |installer|
     'HEADER_SEARCH_PATHS' => '"$(inherited)" "$(GRPC_SRC_ROOT)/include"',
     'USER_HEADER_SEARCH_PATHS' => '"$(GRPC_SRC_ROOT)"',
     # If we don't set these two settings, `include/grpc/support/time.h` and
-    # `src/core/lib/support/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
+    # `src/core/lib/gpr/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
     # build.
     'USE_HEADERMAP' => 'NO',
     'ALWAYS_SEARCH_USER_PATHS' => 'NO',
diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py
index aea0786890b..9debb22249d 100644
--- a/src/python/grpcio/grpc_core_dependencies.py
+++ b/src/python/grpcio/grpc_core_dependencies.py
@@ -15,50 +15,50 @@
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_core_dependencies.py.template`!!!
 
 CORE_SOURCE_FILES = [
+    'src/core/lib/gpr/alloc.cc',
+    'src/core/lib/gpr/arena.cc',
+    'src/core/lib/gpr/atm.cc',
+    'src/core/lib/gpr/avl.cc',
+    'src/core/lib/gpr/cmdline.cc',
+    'src/core/lib/gpr/cpu_iphone.cc',
+    'src/core/lib/gpr/cpu_linux.cc',
+    'src/core/lib/gpr/cpu_posix.cc',
+    'src/core/lib/gpr/cpu_windows.cc',
+    'src/core/lib/gpr/env_linux.cc',
+    'src/core/lib/gpr/env_posix.cc',
+    'src/core/lib/gpr/env_windows.cc',
+    'src/core/lib/gpr/fork.cc',
+    'src/core/lib/gpr/host_port.cc',
+    'src/core/lib/gpr/log.cc',
+    'src/core/lib/gpr/log_android.cc',
+    'src/core/lib/gpr/log_linux.cc',
+    'src/core/lib/gpr/log_posix.cc',
+    'src/core/lib/gpr/log_windows.cc',
+    'src/core/lib/gpr/mpscq.cc',
+    'src/core/lib/gpr/murmur_hash.cc',
+    'src/core/lib/gpr/string.cc',
+    'src/core/lib/gpr/string_posix.cc',
+    'src/core/lib/gpr/string_util_windows.cc',
+    'src/core/lib/gpr/string_windows.cc',
+    'src/core/lib/gpr/subprocess_posix.cc',
+    'src/core/lib/gpr/subprocess_windows.cc',
+    'src/core/lib/gpr/sync.cc',
+    'src/core/lib/gpr/sync_posix.cc',
+    'src/core/lib/gpr/sync_windows.cc',
+    'src/core/lib/gpr/thd.cc',
+    'src/core/lib/gpr/thd_posix.cc',
+    'src/core/lib/gpr/thd_windows.cc',
+    'src/core/lib/gpr/time.cc',
+    'src/core/lib/gpr/time_posix.cc',
+    'src/core/lib/gpr/time_precise.cc',
+    'src/core/lib/gpr/time_windows.cc',
+    'src/core/lib/gpr/tls_pthread.cc',
+    'src/core/lib/gpr/tmpfile_msys.cc',
+    'src/core/lib/gpr/tmpfile_posix.cc',
+    'src/core/lib/gpr/tmpfile_windows.cc',
+    'src/core/lib/gpr/wrap_memcpy.cc',
     'src/core/lib/profiling/basic_timers.cc',
     'src/core/lib/profiling/stap_timers.cc',
-    'src/core/lib/support/alloc.cc',
-    'src/core/lib/support/arena.cc',
-    'src/core/lib/support/atm.cc',
-    'src/core/lib/support/avl.cc',
-    'src/core/lib/support/cmdline.cc',
-    'src/core/lib/support/cpu_iphone.cc',
-    'src/core/lib/support/cpu_linux.cc',
-    'src/core/lib/support/cpu_posix.cc',
-    'src/core/lib/support/cpu_windows.cc',
-    'src/core/lib/support/env_linux.cc',
-    'src/core/lib/support/env_posix.cc',
-    'src/core/lib/support/env_windows.cc',
-    'src/core/lib/support/fork.cc',
-    'src/core/lib/support/host_port.cc',
-    'src/core/lib/support/log.cc',
-    'src/core/lib/support/log_android.cc',
-    'src/core/lib/support/log_linux.cc',
-    'src/core/lib/support/log_posix.cc',
-    'src/core/lib/support/log_windows.cc',
-    'src/core/lib/support/mpscq.cc',
-    'src/core/lib/support/murmur_hash.cc',
-    'src/core/lib/support/string.cc',
-    'src/core/lib/support/string_posix.cc',
-    'src/core/lib/support/string_util_windows.cc',
-    'src/core/lib/support/string_windows.cc',
-    'src/core/lib/support/subprocess_posix.cc',
-    'src/core/lib/support/subprocess_windows.cc',
-    'src/core/lib/support/sync.cc',
-    'src/core/lib/support/sync_posix.cc',
-    'src/core/lib/support/sync_windows.cc',
-    'src/core/lib/support/thd.cc',
-    'src/core/lib/support/thd_posix.cc',
-    'src/core/lib/support/thd_windows.cc',
-    'src/core/lib/support/time.cc',
-    'src/core/lib/support/time_posix.cc',
-    'src/core/lib/support/time_precise.cc',
-    'src/core/lib/support/time_windows.cc',
-    'src/core/lib/support/tls_pthread.cc',
-    'src/core/lib/support/tmpfile_msys.cc',
-    'src/core/lib/support/tmpfile_posix.cc',
-    'src/core/lib/support/tmpfile_windows.cc',
-    'src/core/lib/support/wrap_memcpy.cc',
     'src/core/lib/surface/init.cc',
     'src/core/lib/backoff/backoff.cc',
     'src/core/lib/channel/channel_args.cc',
diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template
index da404e2fefa..2be7692e04c 100644
--- a/templates/gRPC-Core.podspec.template
+++ b/templates/gRPC-Core.podspec.template
@@ -135,7 +135,7 @@
       'HEADER_SEARCH_PATHS' => '"$(inherited)" "$(GRPC_SRC_ROOT)/include"',
       'USER_HEADER_SEARCH_PATHS' => '"$(GRPC_SRC_ROOT)"',
       # If we don't set these two settings, `include/grpc/support/time.h` and
-      # `src/core/lib/support/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
+      # `src/core/lib/gpr/string.h` shadow the system `<time.h>` and `<string.h>`, breaking the
       # build.
       'USE_HEADERMAP' => 'NO',
       'ALWAYS_SEARCH_USER_PATHS' => 'NO',
diff --git a/test/core/bad_client/bad_client.cc b/test/core/bad_client/bad_client.cc
index 4c1642aa5d4..dd8d88170e4 100644
--- a/test/core/bad_client/bad_client.cc
+++ b/test/core/bad_client/bad_client.cc
@@ -28,10 +28,10 @@
 #include "src/core/ext/filters/http/server/http_server_filter.h"
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/gpr/murmur_hash.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/endpoint_pair.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/murmur_hash.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/completion_queue.h"
 #include "src/core/lib/surface/server.h"
 
diff --git a/test/core/bad_client/tests/large_metadata.cc b/test/core/bad_client/tests/large_metadata.cc
index 1ce0f28967c..ff3e9eb932f 100644
--- a/test/core/bad_client/tests/large_metadata.cc
+++ b/test/core/bad_client/tests/large_metadata.cc
@@ -22,7 +22,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/surface/server.h"
 #include "test/core/end2end/cq_verifier.h"
 
diff --git a/test/core/bad_ssl/bad_ssl_test.cc b/test/core/bad_ssl/bad_ssl_test.cc
index 0e74a62f192..8a7960b5edb 100644
--- a/test/core/bad_ssl/bad_ssl_test.cc
+++ b/test/core/bad_ssl/bad_ssl_test.cc
@@ -26,8 +26,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/subprocess.h>
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
diff --git a/test/core/channel/minimal_stack_is_minimal_test.cc b/test/core/channel/minimal_stack_is_minimal_test.cc
index 3495f603e4f..f02c8180f20 100644
--- a/test/core/channel/minimal_stack_is_minimal_test.cc
+++ b/test/core/channel/minimal_stack_is_minimal_test.cc
@@ -35,7 +35,7 @@
 #include <string.h>
 
 #include "src/core/lib/channel/channel_stack_builder.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/surface/channel_init.h"
 #include "src/core/lib/surface/channel_stack_type.h"
 #include "src/core/lib/transport/transport_impl.h"
diff --git a/test/core/client_channel/lb_policies_test.cc b/test/core/client_channel/lb_policies_test.cc
index 847ea0066ba..716c63b9d0e 100644
--- a/test/core/client_channel/lb_policies_test.cc
+++ b/test/core/client_channel/lb_policies_test.cc
@@ -30,7 +30,7 @@
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/server.h"
 #include "test/core/end2end/cq_verifier.h"
diff --git a/test/core/compression/message_compress_test.cc b/test/core/compression/message_compress_test.cc
index 6ca07b70c4f..b03ca4c4cbb 100644
--- a/test/core/compression/message_compress_test.cc
+++ b/test/core/compression/message_compress_test.cc
@@ -25,8 +25,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/gpr/murmur_hash.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
-#include "src/core/lib/support/murmur_hash.h"
 #include "test/core/util/slice_splitter.h"
 #include "test/core/util/test_config.h"
 
diff --git a/test/core/end2end/bad_server_response_test.cc b/test/core/end2end/bad_server_response_test.cc
index 93809ac37a6..a8e5e291c8d 100644
--- a/test/core/end2end/bad_server_response_test.cc
+++ b/test/core/end2end/bad_server_response_test.cc
@@ -31,10 +31,10 @@
 #include <grpc/support/log.h>
 #include <grpc/support/thd.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "test/core/end2end/cq_verifier.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
diff --git a/test/core/end2end/cq_verifier.cc b/test/core/end2end/cq_verifier.cc
index 8686f4e7b74..7bf8ae0f6eb 100644
--- a/test/core/end2end/cq_verifier.cc
+++ b/test/core/end2end/cq_verifier.cc
@@ -29,7 +29,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/surface/event_string.h"
 
 #define ROOT_EXPECTATION 1000
diff --git a/test/core/end2end/dualstack_socket_test.cc b/test/core/end2end/dualstack_socket_test.cc
index 04c727e689a..bb30547cd22 100644
--- a/test/core/end2end/dualstack_socket_test.cc
+++ b/test/core/end2end/dualstack_socket_test.cc
@@ -29,12 +29,12 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/socket_utils_posix.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "test/core/end2end/cq_verifier.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
diff --git a/test/core/end2end/fixtures/h2_full+trace.cc b/test/core/end2end/fixtures/h2_full+trace.cc
index a49de96009c..7104fbc581e 100644
--- a/test/core/end2end/fixtures/h2_full+trace.cc
+++ b/test/core/end2end/fixtures/h2_full+trace.cc
@@ -35,7 +35,7 @@
 #include "src/core/ext/filters/http/server/http_server_filter.h"
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/connected_channel.h"
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/server.h"
 #include "test/core/util/port.h"
diff --git a/test/core/end2end/fixtures/h2_http_proxy.cc b/test/core/end2end/fixtures/h2_http_proxy.cc
index 099367d91b8..e8e81f09304 100644
--- a/test/core/end2end/fixtures/h2_http_proxy.cc
+++ b/test/core/end2end/fixtures/h2_http_proxy.cc
@@ -31,7 +31,7 @@
 #include "src/core/ext/filters/http/server/http_server_filter.h"
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/connected_channel.h"
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/server.h"
 #include "test/core/end2end/fixtures/http_proxy_fixture.h"
diff --git a/test/core/end2end/fixtures/h2_sockpair+trace.cc b/test/core/end2end/fixtures/h2_sockpair+trace.cc
index 9807e929af6..236780b8d38 100644
--- a/test/core/end2end/fixtures/h2_sockpair+trace.cc
+++ b/test/core/end2end/fixtures/h2_sockpair+trace.cc
@@ -36,9 +36,9 @@
 #include "src/core/ext/filters/http/server/http_server_filter.h"
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/connected_channel.h"
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/iomgr/endpoint_pair.h"
 #include "src/core/lib/iomgr/iomgr.h"
-#include "src/core/lib/support/env.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/completion_queue.h"
 #include "src/core/lib/surface/server.h"
diff --git a/test/core/end2end/fixtures/h2_ssl.cc b/test/core/end2end/fixtures/h2_ssl.cc
index 9a0680c40ef..8c5c8a2f3fa 100644
--- a/test/core/end2end/fixtures/h2_ssl.cc
+++ b/test/core/end2end/fixtures/h2_ssl.cc
@@ -26,10 +26,10 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/tmpfile.h"
 #include "src/core/lib/security/credentials/credentials.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/tmpfile.h"
 #include "test/core/end2end/data/ssl_test_data.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
diff --git a/test/core/end2end/fixtures/h2_ssl_proxy.cc b/test/core/end2end/fixtures/h2_ssl_proxy.cc
index 5ddbdefc8c3..3f0646cf0fd 100644
--- a/test/core/end2end/fixtures/h2_ssl_proxy.cc
+++ b/test/core/end2end/fixtures/h2_ssl_proxy.cc
@@ -26,10 +26,10 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/tmpfile.h"
 #include "src/core/lib/security/credentials/credentials.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/tmpfile.h"
 #include "test/core/end2end/data/ssl_test_data.h"
 #include "test/core/end2end/fixtures/proxy.h"
 #include "test/core/util/port.h"
diff --git a/test/core/end2end/fixtures/h2_uds.cc b/test/core/end2end/fixtures/h2_uds.cc
index 28f0a50e153..1944dd84a37 100644
--- a/test/core/end2end/fixtures/h2_uds.cc
+++ b/test/core/end2end/fixtures/h2_uds.cc
@@ -33,7 +33,7 @@
 #include "src/core/ext/filters/http/server/http_server_filter.h"
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/connected_channel.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/server.h"
 #include "test/core/util/port.h"
diff --git a/test/core/end2end/fixtures/http_proxy_fixture.cc b/test/core/end2end/fixtures/http_proxy_fixture.cc
index 137f7c9fa37..8ec97df3e41 100644
--- a/test/core/end2end/fixtures/http_proxy_fixture.cc
+++ b/test/core/end2end/fixtures/http_proxy_fixture.cc
@@ -34,6 +34,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/combiner.h"
@@ -49,7 +50,6 @@
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/slice/b64.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/string.h"
 #include "test/core/util/port.h"
 
 struct grpc_end2end_http_proxy {
diff --git a/test/core/end2end/fuzzers/api_fuzzer.cc b/test/core/end2end/fuzzers/api_fuzzer.cc
index 43c9fa19c6d..14c155502a5 100644
--- a/test/core/end2end/fuzzers/api_fuzzer.cc
+++ b/test/core/end2end/fuzzers/api_fuzzer.cc
@@ -28,13 +28,13 @@
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/tcp_client.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/iomgr/timer_manager.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/env.h"
 #include "src/core/lib/surface/server.h"
 #include "src/core/lib/transport/metadata.h"
 #include "test/core/end2end/data/ssl_test_data.h"
diff --git a/test/core/end2end/h2_ssl_cert_test.cc b/test/core/end2end/h2_ssl_cert_test.cc
index d50d1f4d81e..3383d6d5d1b 100644
--- a/test/core/end2end/h2_ssl_cert_test.cc
+++ b/test/core/end2end/h2_ssl_cert_test.cc
@@ -26,10 +26,10 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/tmpfile.h"
 #include "src/core/lib/security/credentials/credentials.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/tmpfile.h"
 #include "test/core/end2end/cq_verifier.h"
 #include "test/core/end2end/data/ssl_test_data.h"
 #include "test/core/util/port.h"
diff --git a/test/core/end2end/tests/bad_hostname.cc b/test/core/end2end/tests/bad_hostname.cc
index 97ef62b5e34..85e9ba1307a 100644
--- a/test/core/end2end/tests/bad_hostname.cc
+++ b/test/core/end2end/tests/bad_hostname.cc
@@ -27,7 +27,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/call_creds.cc b/test/core/end2end/tests/call_creds.cc
index e1c868232c2..c5ea101c53d 100644
--- a/test/core/end2end/tests/call_creds.cc
+++ b/test/core/end2end/tests/call_creds.cc
@@ -27,8 +27,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/security/credentials/credentials.h"
-#include "src/core/lib/support/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static const char iam_token[] = "token";
diff --git a/test/core/end2end/tests/cancel_with_status.cc b/test/core/end2end/tests/cancel_with_status.cc
index c867751d531..7937fd161f2 100644
--- a/test/core/end2end/tests/cancel_with_status.cc
+++ b/test/core/end2end/tests/cancel_with_status.cc
@@ -28,7 +28,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/default_host.cc b/test/core/end2end/tests/default_host.cc
index 85f92b0ab07..7c944205401 100644
--- a/test/core/end2end/tests/default_host.cc
+++ b/test/core/end2end/tests/default_host.cc
@@ -27,7 +27,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/empty_batch.cc b/test/core/end2end/tests/empty_batch.cc
index b249c141dec..c41e65ddd2c 100644
--- a/test/core/end2end/tests/empty_batch.cc
+++ b/test/core/end2end/tests/empty_batch.cc
@@ -27,7 +27,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/high_initial_seqno.cc b/test/core/end2end/tests/high_initial_seqno.cc
index d390a954e5c..d4d4f5a817f 100644
--- a/test/core/end2end/tests/high_initial_seqno.cc
+++ b/test/core/end2end/tests/high_initial_seqno.cc
@@ -29,7 +29,7 @@
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/hpack_size.cc b/test/core/end2end/tests/hpack_size.cc
index 7ac5fefa229..0d6ec01e36c 100644
--- a/test/core/end2end/tests/hpack_size.cc
+++ b/test/core/end2end/tests/hpack_size.cc
@@ -29,7 +29,7 @@
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/idempotent_request.cc b/test/core/end2end/tests/idempotent_request.cc
index e39975382ad..7487e4d83df 100644
--- a/test/core/end2end/tests/idempotent_request.cc
+++ b/test/core/end2end/tests/idempotent_request.cc
@@ -27,7 +27,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/keepalive_timeout.cc b/test/core/end2end/tests/keepalive_timeout.cc
index 822565510f3..6482b868258 100644
--- a/test/core/end2end/tests/keepalive_timeout.cc
+++ b/test/core/end2end/tests/keepalive_timeout.cc
@@ -28,8 +28,8 @@
 #include <grpc/support/useful.h>
 #include "src/core/ext/transport/chttp2/transport/frame_ping.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
-#include "src/core/lib/support/env.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/negative_deadline.cc b/test/core/end2end/tests/negative_deadline.cc
index b793964b482..b752bf9482d 100644
--- a/test/core/end2end/tests/negative_deadline.cc
+++ b/test/core/end2end/tests/negative_deadline.cc
@@ -27,7 +27,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/no_logging.cc b/test/core/end2end/tests/no_logging.cc
index bf8651221a6..d89918bd3e1 100644
--- a/test/core/end2end/tests/no_logging.cc
+++ b/test/core/end2end/tests/no_logging.cc
@@ -28,8 +28,8 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/error.h"
-#include "src/core/lib/support/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 enum { TIMEOUT = 200000 };
diff --git a/test/core/end2end/tests/proxy_auth.cc b/test/core/end2end/tests/proxy_auth.cc
index e4b91ab8798..495151b592b 100644
--- a/test/core/end2end/tests/proxy_auth.cc
+++ b/test/core/end2end/tests/proxy_auth.cc
@@ -32,7 +32,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/registered_call.cc b/test/core/end2end/tests/registered_call.cc
index 440d817cf13..cefa89db4db 100644
--- a/test/core/end2end/tests/registered_call.cc
+++ b/test/core/end2end/tests/registered_call.cc
@@ -27,7 +27,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/server_finishes_request.cc b/test/core/end2end/tests/server_finishes_request.cc
index 46b874b5697..743b3aeb911 100644
--- a/test/core/end2end/tests/server_finishes_request.cc
+++ b/test/core/end2end/tests/server_finishes_request.cc
@@ -27,7 +27,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/end2end/tests/simple_request.cc b/test/core/end2end/tests/simple_request.cc
index 7eb74679816..ae93f79c9d6 100644
--- a/test/core/end2end/tests/simple_request.cc
+++ b/test/core/end2end/tests/simple_request.cc
@@ -28,7 +28,7 @@
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
 #include "src/core/lib/debug/stats.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
 static void* tag(intptr_t t) { return (void*)t; }
diff --git a/test/core/fling/fling_stream_test.cc b/test/core/fling/fling_stream_test.cc
index b476f2e128e..b5a5ce816e6 100644
--- a/test/core/fling/fling_stream_test.cc
+++ b/test/core/fling/fling_stream_test.cc
@@ -23,7 +23,7 @@
 #include <grpc/support/host_port.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/subprocess.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/util/port.h"
 
 int main(int argc, char** argv) {
diff --git a/test/core/fling/fling_test.cc b/test/core/fling/fling_test.cc
index 0e8b3c10282..3792e45c429 100644
--- a/test/core/fling/fling_test.cc
+++ b/test/core/fling/fling_test.cc
@@ -23,7 +23,7 @@
 #include <grpc/support/host_port.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/subprocess.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/util/port.h"
 
 int main(int argc, const char** argv) {
diff --git a/test/core/gpr++/BUILD b/test/core/gpr++/BUILD
new file mode 100644
index 00000000000..93324a378b2
--- /dev/null
+++ b/test/core/gpr++/BUILD
@@ -0,0 +1,95 @@
+# Copyright 2016 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
+
+licenses(["notice"])  # Apache v2
+
+grpc_package(name = "test/core/gpr++")
+
+grpc_cc_test(
+    name = "manual_constructor_test",
+    srcs = ["manual_constructor_test.cc"],
+    language = "C++",
+    deps = [
+        "//:gpr",
+        "//test/core/util:gpr_test_util",
+    ],
+)
+
+grpc_cc_test(
+    name = "memory_test",
+    srcs = ["memory_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
+    language = "C++",
+    deps = [
+        "//:grpc",
+        "//test/core/util:gpr_test_util",
+    ],
+)
+
+grpc_cc_test(
+    name = "inlined_vector_test",
+    srcs = ["inlined_vector_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
+    language = "C++",
+    deps = [
+        "//:grpc",
+        "//test/core/util:gpr_test_util",
+    ],
+)
+
+grpc_cc_test(
+    name = "orphanable_test",
+    srcs = ["orphanable_test.cc"],
+    language = "C++",
+    deps = [
+        "//:orphanable",
+        "//test/core/util:gpr_test_util",
+    ],
+    external_deps = [
+        "gtest",
+    ],
+)
+
+grpc_cc_test(
+    name = "ref_counted_test",
+    srcs = ["ref_counted_test.cc"],
+    language = "C++",
+    deps = [
+        "//:ref_counted",
+        "//test/core/util:gpr_test_util",
+    ],
+    external_deps = [
+        "gtest",
+    ],
+)
+
+grpc_cc_test(
+    name = "ref_counted_ptr_test",
+    srcs = ["ref_counted_ptr_test.cc"],
+    language = "C++",
+    deps = [
+        "//:ref_counted",
+        "//:ref_counted_ptr",
+        "//test/core/util:gpr_test_util",
+    ],
+    external_deps = [
+        "gtest",
+    ],
+)
diff --git a/test/core/support/vector_test.cc b/test/core/gpr++/inlined_vector_test.cc
similarity index 95%
rename from test/core/support/vector_test.cc
rename to test/core/gpr++/inlined_vector_test.cc
index 82607a1b260..09d5453a56a 100644
--- a/test/core/support/vector_test.cc
+++ b/test/core/gpr++/inlined_vector_test.cc
@@ -16,9 +16,9 @@
  *
  */
 
-#include "src/core/lib/support/vector.h"
+#include "src/core/lib/gpr++/inlined_vector.h"
 #include <gtest/gtest.h>
-#include "src/core/lib/support/memory.h"
+#include "src/core/lib/gpr++/memory.h"
 #include "test/core/util/test_config.h"
 
 namespace grpc_core {
diff --git a/test/core/support/manual_constructor_test.cc b/test/core/gpr++/manual_constructor_test.cc
similarity index 96%
rename from test/core/support/manual_constructor_test.cc
rename to test/core/gpr++/manual_constructor_test.cc
index 714f8b21a01..e049b793f61 100644
--- a/test/core/support/manual_constructor_test.cc
+++ b/test/core/gpr++/manual_constructor_test.cc
@@ -18,7 +18,7 @@
 
 /* Test of gpr synchronization support. */
 
-#include "src/core/lib/support/manual_constructor.h"
+#include "src/core/lib/gpr++/manual_constructor.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
@@ -26,7 +26,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <cstring>
-#include "src/core/lib/support/abstract.h"
+#include "src/core/lib/gpr++/abstract.h"
 #include "test/core/util/test_config.h"
 
 class A {
diff --git a/test/core/support/memory_test.cc b/test/core/gpr++/memory_test.cc
similarity index 97%
rename from test/core/support/memory_test.cc
rename to test/core/gpr++/memory_test.cc
index 79ab631a78d..3553e119e3b 100644
--- a/test/core/support/memory_test.cc
+++ b/test/core/gpr++/memory_test.cc
@@ -16,7 +16,7 @@
  *
  */
 
-#include "src/core/lib/support/memory.h"
+#include "src/core/lib/gpr++/memory.h"
 #include <gtest/gtest.h>
 #include "test/core/util/test_config.h"
 
diff --git a/test/core/support/orphanable_test.cc b/test/core/gpr++/orphanable_test.cc
similarity index 96%
rename from test/core/support/orphanable_test.cc
rename to test/core/gpr++/orphanable_test.cc
index e07017ab1ed..4513d25da17 100644
--- a/test/core/support/orphanable_test.cc
+++ b/test/core/gpr++/orphanable_test.cc
@@ -16,11 +16,11 @@
  *
  */
 
-#include "src/core/lib/support/orphanable.h"
+#include "src/core/lib/gpr++/orphanable.h"
 
 #include <gtest/gtest.h>
 
-#include "src/core/lib/support/memory.h"
+#include "src/core/lib/gpr++/memory.h"
 #include "test/core/util/test_config.h"
 
 namespace grpc_core {
diff --git a/test/core/support/ref_counted_ptr_test.cc b/test/core/gpr++/ref_counted_ptr_test.cc
similarity index 97%
rename from test/core/support/ref_counted_ptr_test.cc
rename to test/core/gpr++/ref_counted_ptr_test.cc
index ce4975d3477..e897b2859c8 100644
--- a/test/core/support/ref_counted_ptr_test.cc
+++ b/test/core/gpr++/ref_counted_ptr_test.cc
@@ -16,14 +16,14 @@
  *
  */
 
-#include "src/core/lib/support/ref_counted_ptr.h"
+#include "src/core/lib/gpr++/ref_counted_ptr.h"
 
 #include <gtest/gtest.h>
 
 #include <grpc/support/log.h>
 
-#include "src/core/lib/support/memory.h"
-#include "src/core/lib/support/ref_counted.h"
+#include "src/core/lib/gpr++/memory.h"
+#include "src/core/lib/gpr++/ref_counted.h"
 #include "test/core/util/test_config.h"
 
 namespace grpc_core {
diff --git a/test/core/support/ref_counted_test.cc b/test/core/gpr++/ref_counted_test.cc
similarity index 95%
rename from test/core/support/ref_counted_test.cc
rename to test/core/gpr++/ref_counted_test.cc
index 0629e3ff5fd..568ec51c64a 100644
--- a/test/core/support/ref_counted_test.cc
+++ b/test/core/gpr++/ref_counted_test.cc
@@ -16,11 +16,11 @@
  *
  */
 
-#include "src/core/lib/support/ref_counted.h"
+#include "src/core/lib/gpr++/ref_counted.h"
 
 #include <gtest/gtest.h>
 
-#include "src/core/lib/support/memory.h"
+#include "src/core/lib/gpr++/memory.h"
 #include "test/core/util/test_config.h"
 
 namespace grpc_core {
diff --git a/test/core/support/BUILD b/test/core/gpr/BUILD
similarity index 70%
rename from test/core/support/BUILD
rename to test/core/gpr/BUILD
index c8fa046da1d..1be1036d041 100644
--- a/test/core/support/BUILD
+++ b/test/core/gpr/BUILD
@@ -16,7 +16,7 @@ load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_c
 
 licenses(["notice"])  # Apache v2
 
-grpc_package(name = "test/core/support")
+grpc_package(name = "test/core/gpr")
 
 grpc_cc_test(
     name = "alloc_test",
@@ -118,16 +118,6 @@ grpc_cc_test(
     ],
 )
 
-grpc_cc_test(
-    name = "manual_constructor_test",
-    srcs = ["manual_constructor_test.cc"],
-    language = "C++",
-    deps = [
-        "//:gpr",
-        "//test/core/util:gpr_test_util",
-    ],
-)
-
 grpc_cc_test(
     name = "spinlock_test",
     srcs = ["spinlock_test.cc"],
@@ -187,69 +177,3 @@ grpc_cc_test(
         "//test/core/util:gpr_test_util",
     ],
 )
-
-grpc_cc_test(
-    name = "memory_test",
-    srcs = ["memory_test.cc"],
-    external_deps = [
-        "gtest",
-    ],
-    language = "C++",
-    deps = [
-        "//:grpc",
-        "//test/core/util:gpr_test_util",
-    ],
-)
-
-grpc_cc_test(
-    name = "vector_test",
-    srcs = ["vector_test.cc"],
-    external_deps = [
-        "gtest",
-    ],
-    language = "C++",
-    deps = [
-        "//:grpc",
-        "//test/core/util:gpr_test_util",
-    ],
-)
-
-grpc_cc_test(
-    name = "orphanable_test",
-    srcs = ["orphanable_test.cc"],
-    language = "C++",
-    deps = [
-        "//:orphanable",
-        "//test/core/util:gpr_test_util",
-    ],
-    external_deps = [
-        "gtest",
-    ],
-)
-
-grpc_cc_test(
-    name = "ref_counted_test",
-    srcs = ["ref_counted_test.cc"],
-    language = "C++",
-    deps = [
-        "//:ref_counted",
-        "//test/core/util:gpr_test_util",
-    ],
-    external_deps = [
-        "gtest",
-    ],
-)
-
-grpc_cc_test(
-    name = "ref_counted_ptr_test",
-    srcs = ["ref_counted_ptr_test.cc"],
-    language = "C++",
-    deps = [
-        "//:ref_counted",
-        "//:ref_counted_ptr",
-        "//test/core/util:gpr_test_util",
-    ],
-    external_deps = [
-        "gtest",
-    ],
-)
diff --git a/test/core/support/alloc_test.cc b/test/core/gpr/alloc_test.cc
similarity index 100%
rename from test/core/support/alloc_test.cc
rename to test/core/gpr/alloc_test.cc
diff --git a/test/core/support/arena_test.cc b/test/core/gpr/arena_test.cc
similarity index 97%
rename from test/core/support/arena_test.cc
rename to test/core/gpr/arena_test.cc
index ada0f438542..59ea04c0edb 100644
--- a/test/core/support/arena_test.cc
+++ b/test/core/gpr/arena_test.cc
@@ -16,7 +16,7 @@
  *
  */
 
-#include "src/core/lib/support/arena.h"
+#include "src/core/lib/gpr/arena.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -27,7 +27,7 @@
 #include <inttypes.h>
 #include <string.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/util/test_config.h"
 
 static void test_noop(void) { gpr_arena_destroy(gpr_arena_create(1)); }
diff --git a/test/core/support/avl_test.cc b/test/core/gpr/avl_test.cc
similarity index 100%
rename from test/core/support/avl_test.cc
rename to test/core/gpr/avl_test.cc
diff --git a/test/core/support/cmdline_test.cc b/test/core/gpr/cmdline_test.cc
similarity index 100%
rename from test/core/support/cmdline_test.cc
rename to test/core/gpr/cmdline_test.cc
diff --git a/test/core/support/cpu_test.cc b/test/core/gpr/cpu_test.cc
similarity index 100%
rename from test/core/support/cpu_test.cc
rename to test/core/gpr/cpu_test.cc
diff --git a/test/core/support/env_test.cc b/test/core/gpr/env_test.cc
similarity index 94%
rename from test/core/support/env_test.cc
rename to test/core/gpr/env_test.cc
index b12c04d06fb..3f4b4932394 100644
--- a/test/core/support/env_test.cc
+++ b/test/core/gpr/env_test.cc
@@ -22,8 +22,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/util/test_config.h"
 
 #define LOG_TEST_NAME(x) gpr_log(GPR_INFO, "%s", x)
diff --git a/test/core/support/host_port_test.cc b/test/core/gpr/host_port_test.cc
similarity index 100%
rename from test/core/support/host_port_test.cc
rename to test/core/gpr/host_port_test.cc
diff --git a/test/core/support/log_test.cc b/test/core/gpr/log_test.cc
similarity index 98%
rename from test/core/support/log_test.cc
rename to test/core/gpr/log_test.cc
index 7dba35c13ee..839ff0aef94 100644
--- a/test/core/support/log_test.cc
+++ b/test/core/gpr/log_test.cc
@@ -21,7 +21,7 @@
 #include <stdbool.h>
 #include <string.h>
 
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 #include "test/core/util/test_config.h"
 
 static bool log_func_reached = false;
diff --git a/test/core/support/mpscq_test.cc b/test/core/gpr/mpscq_test.cc
similarity index 99%
rename from test/core/support/mpscq_test.cc
rename to test/core/gpr/mpscq_test.cc
index 1b83f7d5be9..5a8177543c9 100644
--- a/test/core/support/mpscq_test.cc
+++ b/test/core/gpr/mpscq_test.cc
@@ -16,7 +16,7 @@
  *
  */
 
-#include "src/core/lib/support/mpscq.h"
+#include "src/core/lib/gpr/mpscq.h"
 
 #include <stdlib.h>
 
diff --git a/test/core/support/murmur_hash_test.cc b/test/core/gpr/murmur_hash_test.cc
similarity index 97%
rename from test/core/support/murmur_hash_test.cc
rename to test/core/gpr/murmur_hash_test.cc
index 461c728951c..d920dd3f95d 100644
--- a/test/core/support/murmur_hash_test.cc
+++ b/test/core/gpr/murmur_hash_test.cc
@@ -16,7 +16,7 @@
  *
  */
 
-#include "src/core/lib/support/murmur_hash.h"
+#include "src/core/lib/gpr/murmur_hash.h"
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include "test/core/util/test_config.h"
diff --git a/test/core/support/spinlock_test.cc b/test/core/gpr/spinlock_test.cc
similarity index 99%
rename from test/core/support/spinlock_test.cc
rename to test/core/gpr/spinlock_test.cc
index ea0dbbf7c6b..77e3dfbede5 100644
--- a/test/core/support/spinlock_test.cc
+++ b/test/core/gpr/spinlock_test.cc
@@ -18,7 +18,7 @@
 
 /* Test of gpr synchronization support. */
 
-#include "src/core/lib/support/spinlock.h"
+#include "src/core/lib/gpr/spinlock.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
diff --git a/test/core/support/string_test.cc b/test/core/gpr/string_test.cc
similarity index 99%
rename from test/core/support/string_test.cc
rename to test/core/gpr/string_test.cc
index fd7f7cde59e..57068eb2c99 100644
--- a/test/core/support/string_test.cc
+++ b/test/core/gpr/string_test.cc
@@ -16,7 +16,7 @@
  *
  */
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 
 #include <limits.h>
 #include <stddef.h>
diff --git a/test/core/support/sync_test.cc b/test/core/gpr/sync_test.cc
similarity index 100%
rename from test/core/support/sync_test.cc
rename to test/core/gpr/sync_test.cc
diff --git a/test/core/support/thd_test.cc b/test/core/gpr/thd_test.cc
similarity index 100%
rename from test/core/support/thd_test.cc
rename to test/core/gpr/thd_test.cc
diff --git a/test/core/support/time_test.cc b/test/core/gpr/time_test.cc
similarity index 100%
rename from test/core/support/time_test.cc
rename to test/core/gpr/time_test.cc
diff --git a/test/core/support/tls_test.cc b/test/core/gpr/tls_test.cc
similarity index 100%
rename from test/core/support/tls_test.cc
rename to test/core/gpr/tls_test.cc
diff --git a/test/core/support/useful_test.cc b/test/core/gpr/useful_test.cc
similarity index 100%
rename from test/core/support/useful_test.cc
rename to test/core/gpr/useful_test.cc
diff --git a/test/core/iomgr/load_file_test.cc b/test/core/iomgr/load_file_test.cc
index 797d0ef1a4d..38c8710535b 100644
--- a/test/core/iomgr/load_file_test.cc
+++ b/test/core/iomgr/load_file_test.cc
@@ -24,9 +24,9 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/tmpfile.h"
 #include "src/core/lib/iomgr/load_file.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/tmpfile.h"
 #include "test/core/util/test_config.h"
 
 #define LOG_TEST_NAME(x) gpr_log(GPR_INFO, "%s", x)
diff --git a/test/core/iomgr/wakeup_fd_cv_test.cc b/test/core/iomgr/wakeup_fd_cv_test.cc
index d4e05bd7ef8..c092a8f3bf9 100644
--- a/test/core/iomgr/wakeup_fd_cv_test.cc
+++ b/test/core/iomgr/wakeup_fd_cv_test.cc
@@ -27,9 +27,9 @@
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_posix.h"
-#include "src/core/lib/support/env.h"
 
 typedef struct poll_args {
   struct pollfd* fds;
diff --git a/test/core/json/json_test.cc b/test/core/json/json_test.cc
index 18b9c55ee79..902f1cd90ee 100644
--- a/test/core/json/json_test.cc
+++ b/test/core/json/json_test.cc
@@ -22,8 +22,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/json/json.h"
-#include "src/core/lib/support/string.h"
 
 #include "test/core/util/test_config.h"
 
diff --git a/test/core/memory_usage/client.cc b/test/core/memory_usage/client.cc
index eb90067970d..ca841434aa8 100644
--- a/test/core/memory_usage/client.cc
+++ b/test/core/memory_usage/client.cc
@@ -28,8 +28,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/util/memory_counters.h"
 #include "test/core/util/test_config.h"
 
diff --git a/test/core/memory_usage/memory_usage_test.cc b/test/core/memory_usage/memory_usage_test.cc
index 58e31c95319..fb6d290130f 100644
--- a/test/core/memory_usage/memory_usage_test.cc
+++ b/test/core/memory_usage/memory_usage_test.cc
@@ -23,7 +23,7 @@
 #include <grpc/support/host_port.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/subprocess.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/util/port.h"
 
 int main(int argc, char** argv) {
diff --git a/test/core/security/auth_context_test.cc b/test/core/security/auth_context_test.cc
index d8e41326c00..58f0d8e1c27 100644
--- a/test/core/security/auth_context_test.cc
+++ b/test/core/security/auth_context_test.cc
@@ -18,8 +18,8 @@
 
 #include <string.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/security/context/security_context.h"
-#include "src/core/lib/support/string.h"
 #include "test/core/util/test_config.h"
 
 #include <grpc/support/log.h>
diff --git a/test/core/security/credentials_test.cc b/test/core/security/credentials_test.cc
index ecc61928f59..90310469aa7 100644
--- a/test/core/security/credentials_test.cc
+++ b/test/core/security/credentials_test.cc
@@ -31,6 +31,9 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/tmpfile.h"
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/security/credentials/composite/composite_credentials.h"
 #include "src/core/lib/security/credentials/fake/fake_credentials.h"
@@ -38,9 +41,6 @@
 #include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
 #include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
 #include "src/core/lib/security/transport/auth_filters.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/tmpfile.h"
 #include "test/core/util/test_config.h"
 
 /* -- Mock channel credentials. -- */
diff --git a/test/core/security/print_google_default_creds_token.cc b/test/core/security/print_google_default_creds_token.cc
index d71116d8f6a..a90f997bda0 100644
--- a/test/core/security/print_google_default_creds_token.cc
+++ b/test/core/security/print_google_default_creds_token.cc
@@ -27,10 +27,10 @@
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/security/credentials/composite/composite_credentials.h"
 #include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 
 typedef struct {
   gpr_mu* mu;
diff --git a/test/core/security/security_connector_test.cc b/test/core/security/security_connector_test.cc
index 9a68e176dbc..6eaef2bf49b 100644
--- a/test/core/security/security_connector_test.cc
+++ b/test/core/security/security_connector_test.cc
@@ -25,12 +25,12 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/tmpfile.h"
 #include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/tmpfile.h"
 #include "src/core/tsi/ssl_transport_security.h"
 #include "src/core/tsi/transport_security.h"
 #include "test/core/util/test_config.h"
diff --git a/test/core/slice/percent_encoding_test.cc b/test/core/slice/percent_encoding_test.cc
index 11f3995f98e..e8d04fcc83c 100644
--- a/test/core/slice/percent_encoding_test.cc
+++ b/test/core/slice/percent_encoding_test.cc
@@ -22,8 +22,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "test/core/util/test_config.h"
 
 #define TEST_VECTOR(raw, encoded, dict) \
diff --git a/test/core/slice/slice_string_helpers_test.cc b/test/core/slice/slice_string_helpers_test.cc
index f1d470461a4..a443f17c692 100644
--- a/test/core/slice/slice_string_helpers_test.cc
+++ b/test/core/slice/slice_string_helpers_test.cc
@@ -29,7 +29,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/util/test_config.h"
 
 #define LOG_TEST_NAME(x) gpr_log(GPR_INFO, "%s", x)
diff --git a/test/core/transport/bdp_estimator_test.cc b/test/core/transport/bdp_estimator_test.cc
index 445823b628b..3afcad7f17d 100644
--- a/test/core/transport/bdp_estimator_test.cc
+++ b/test/core/transport/bdp_estimator_test.cc
@@ -25,8 +25,8 @@
 #include <grpc/support/useful.h>
 #include <gtest/gtest.h>
 #include <limits.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/timer_manager.h"
-#include "src/core/lib/support/string.h"
 #include "test/core/util/test_config.h"
 
 extern gpr_timespec (*gpr_now_impl)(gpr_clock_type clock_type);
diff --git a/test/core/transport/chttp2/bin_decoder_test.cc b/test/core/transport/chttp2/bin_decoder_test.cc
index 6d70a4261b0..283eebbacf5 100644
--- a/test/core/transport/chttp2/bin_decoder_test.cc
+++ b/test/core/transport/chttp2/bin_decoder_test.cc
@@ -24,9 +24,9 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include "src/core/ext/transport/chttp2/transport/bin_encoder.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 
 static int all_ok = 1;
 
diff --git a/test/core/transport/chttp2/bin_encoder_test.cc b/test/core/transport/chttp2/bin_encoder_test.cc
index 44f5de8a50d..bd62b0e479a 100644
--- a/test/core/transport/chttp2/bin_encoder_test.cc
+++ b/test/core/transport/chttp2/bin_encoder_test.cc
@@ -26,8 +26,8 @@
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 
 static int all_ok = 1;
 
diff --git a/test/core/transport/chttp2/hpack_encoder_test.cc b/test/core/transport/chttp2/hpack_encoder_test.cc
index d2dbd4a7986..a40bd643ecc 100644
--- a/test/core/transport/chttp2/hpack_encoder_test.cc
+++ b/test/core/transport/chttp2/hpack_encoder_test.cc
@@ -26,9 +26,9 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/transport/chttp2/transport/hpack_parser.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/metadata.h"
 #include "test/core/util/parse_hexstring.h"
 #include "test/core/util/slice_splitter.h"
diff --git a/test/core/transport/chttp2/hpack_table_test.cc b/test/core/transport/chttp2/hpack_table_test.cc
index 3f3cb2ee9db..e316cf63a07 100644
--- a/test/core/transport/chttp2/hpack_table_test.cc
+++ b/test/core/transport/chttp2/hpack_table_test.cc
@@ -26,7 +26,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/util/test_config.h"
 
 #define LOG_TEST(x) gpr_log(GPR_INFO, "%s", x)
diff --git a/test/core/transport/metadata_test.cc b/test/core/transport/metadata_test.cc
index 5c52ae8d5f5..7d943fd5c7d 100644
--- a/test/core/transport/metadata_test.cc
+++ b/test/core/transport/metadata_test.cc
@@ -27,8 +27,8 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/transport/chttp2/transport/bin_encoder.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/static_metadata.h"
 #include "test/core/util/test_config.h"
 
diff --git a/test/core/transport/pid_controller_test.cc b/test/core/transport/pid_controller_test.cc
index 081d03472a4..1a499c2fb77 100644
--- a/test/core/transport/pid_controller_test.cc
+++ b/test/core/transport/pid_controller_test.cc
@@ -26,7 +26,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 #include <gtest/gtest.h>
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/util/test_config.h"
 
 namespace grpc_core {
diff --git a/test/core/transport/timeout_encoding_test.cc b/test/core/transport/timeout_encoding_test.cc
index 0930bc836d7..e94be138ddc 100644
--- a/test/core/transport/timeout_encoding_test.cc
+++ b/test/core/transport/timeout_encoding_test.cc
@@ -25,8 +25,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/support/murmur_hash.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/murmur_hash.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/core/util/test_config.h"
 
 #define LOG_TEST(x) gpr_log(GPR_INFO, "%s", x)
diff --git a/test/core/tsi/transport_security_test.cc b/test/core/tsi/transport_security_test.cc
index c788ad96ad5..42e17df25d1 100644
--- a/test/core/tsi/transport_security_test.cc
+++ b/test/core/tsi/transport_security_test.cc
@@ -27,7 +27,7 @@
 
 #include <openssl/crypto.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/tsi/fake_transport_security.h"
 #include "src/core/tsi/ssl_transport_security.h"
 #include "test/core/util/test_config.h"
diff --git a/test/core/util/test_config.cc b/test/core/util/test_config.cc
index 9ebb52d83e6..6b410440c22 100644
--- a/test/core/util/test_config.cc
+++ b/test/core/util/test_config.cc
@@ -27,8 +27,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 
 int64_t g_fixture_slowdown_factor = 1;
 int64_t g_poller_slowdown_factor = 1;
diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc
index 1ea087e706a..44cd81a6a42 100644
--- a/test/cpp/end2end/async_end2end_test.cc
+++ b/test/cpp/end2end/async_end2end_test.cc
@@ -34,8 +34,8 @@
 #include <grpc/support/time.h>
 #include <grpc/support/tls.h>
 
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/iomgr/port.h"
-#include "src/core/lib/support/env.h"
 #include "src/proto/grpc/health/v1/health.grpc.pb.h"
 #include "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h"
 #include "src/proto/grpc/testing/echo.grpc.pb.h"
diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc
index 5a7e52e9e94..02f9f904176 100644
--- a/test/cpp/end2end/client_lb_end2end_test.cc
+++ b/test/cpp/end2end/client_lb_end2end_test.cc
@@ -37,7 +37,7 @@
 #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
 #include "src/core/ext/filters/client_channel/subchannel_index.h"
 #include "src/core/lib/backoff/backoff.h"
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 
 #include "src/proto/grpc/testing/echo.grpc.pb.h"
 #include "test/core/util/port.h"
diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc
index a6ea5aada9f..e25de810f1a 100644
--- a/test/cpp/end2end/end2end_test.cc
+++ b/test/cpp/end2end/end2end_test.cc
@@ -35,8 +35,8 @@
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
 
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/security/credentials/credentials.h"
-#include "src/core/lib/support/env.h"
 #include "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h"
 #include "src/proto/grpc/testing/echo.grpc.pb.h"
 #include "test/core/util/port.h"
diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc
index d4ee6b429f2..815dfd0c4f8 100644
--- a/test/cpp/end2end/grpclb_end2end_test.cc
+++ b/test/cpp/end2end/grpclb_end2end_test.cc
@@ -34,8 +34,8 @@
 #include <grpc/support/time.h>
 
 #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/iomgr/sockaddr.h"
-#include "src/core/lib/support/env.h"
 
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
diff --git a/test/cpp/end2end/shutdown_test.cc b/test/cpp/end2end/shutdown_test.cc
index 14ba7c96cca..9119102f7ac 100644
--- a/test/cpp/end2end/shutdown_test.cc
+++ b/test/cpp/end2end/shutdown_test.cc
@@ -28,7 +28,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 #include "src/proto/grpc/testing/echo.grpc.pb.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
diff --git a/test/cpp/grpclb/grpclb_test.cc b/test/cpp/grpclb/grpclb_test.cc
index 38b65fbc78f..d241594af1b 100644
--- a/test/cpp/grpclb/grpclb_test.cc
+++ b/test/cpp/grpclb/grpclb_test.cc
@@ -40,11 +40,11 @@
 #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/tmpfile.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/security/credentials/fake/fake_credentials.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/support/tmpfile.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/surface/server.h"
 #include "test/core/end2end/cq_verifier.h"
diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc
index 716fb963827..8958b9884df 100644
--- a/test/cpp/interop/client.cc
+++ b/test/cpp/interop/client.cc
@@ -27,7 +27,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/cpp/interop/client_helper.h"
 #include "test/cpp/interop/interop_client.h"
 #include "test/cpp/util/test_config.h"
diff --git a/test/cpp/interop/http2_client.cc b/test/cpp/interop/http2_client.cc
index 2de7abcf17e..7a9e3ced6af 100644
--- a/test/cpp/interop/http2_client.cc
+++ b/test/cpp/interop/http2_client.cc
@@ -30,7 +30,7 @@
 #include "src/proto/grpc/testing/test.grpc.pb.h"
 #include "test/cpp/interop/http2_client.h"
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "test/cpp/util/create_test_channel.h"
 #include "test/cpp/util/test_config.h"
 
diff --git a/test/cpp/interop/interop_server.cc b/test/cpp/interop/interop_server.cc
index 30bd8bfef8b..7cfdb2f9e9d 100644
--- a/test/cpp/interop/interop_server.cc
+++ b/test/cpp/interop/interop_server.cc
@@ -31,7 +31,7 @@
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/lib/support/string.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/transport/byte_stream.h"
 #include "src/proto/grpc/testing/empty.pb.h"
 #include "src/proto/grpc/testing/messages.pb.h"
diff --git a/test/cpp/interop/interop_test.cc b/test/cpp/interop/interop_test.cc
index 1bf0d8d10fc..563b7abb5e9 100644
--- a/test/cpp/interop/interop_test.cc
+++ b/test/cpp/interop/interop_test.cc
@@ -37,8 +37,8 @@
 #include "test/core/util/port.h"
 #include "test/cpp/util/test_config.h"
 
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/socket_utils_posix.h"
-#include "src/core/lib/support/string.h"
 
 DEFINE_string(extra_server_flags, "", "Extra flags to pass to server.");
 
diff --git a/test/cpp/microbenchmarks/bm_arena.cc b/test/cpp/microbenchmarks/bm_arena.cc
index 5b7c6119199..69c8c1c0299 100644
--- a/test/cpp/microbenchmarks/bm_arena.cc
+++ b/test/cpp/microbenchmarks/bm_arena.cc
@@ -18,7 +18,7 @@
 
 /* Benchmark arenas */
 
-#include "src/core/lib/support/arena.h"
+#include "src/core/lib/gpr/arena.h"
 #include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc
index 4d5a82c3f67..6d88faecc09 100644
--- a/test/cpp/microbenchmarks/bm_closure.cc
+++ b/test/cpp/microbenchmarks/bm_closure.cc
@@ -22,10 +22,10 @@
 #include <grpc/grpc.h>
 #include <sstream>
 
+#include "src/core/lib/gpr/spinlock.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
-#include "src/core/lib/support/spinlock.h"
 
 #include "test/cpp/microbenchmarks/helpers.h"
 
diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc
index 3481d9d1aaf..8b5523f01aa 100644
--- a/test/cpp/naming/resolver_component_test.cc
+++ b/test/cpp/naming/resolver_component_test.cc
@@ -37,13 +37,13 @@
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
-#include "src/core/lib/support/env.h"
-#include "src/core/lib/support/string.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
 
diff --git a/test/cpp/naming/resolver_component_tests_runner_invoker.cc b/test/cpp/naming/resolver_component_tests_runner_invoker.cc
index 0beb27de1b8..65f11243fe1 100644
--- a/test/cpp/naming/resolver_component_tests_runner_invoker.cc
+++ b/test/cpp/naming/resolver_component_tests_runner_invoker.cc
@@ -32,7 +32,7 @@
 #include "test/cpp/util/subprocess.h"
 #include "test/cpp/util/test_config.h"
 
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 #include "test/core/util/port.h"
 
 DEFINE_bool(
diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc
index 22d039d4b72..2f765a6d1e3 100644
--- a/test/cpp/qps/driver.cc
+++ b/test/cpp/qps/driver.cc
@@ -31,8 +31,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/support/env.h"
 #include "src/proto/grpc/testing/services.grpc.pb.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
diff --git a/test/cpp/qps/json_run_localhost.cc b/test/cpp/qps/json_run_localhost.cc
index db8b2a39437..948c3088e67 100644
--- a/test/cpp/qps/json_run_localhost.cc
+++ b/test/cpp/qps/json_run_localhost.cc
@@ -26,7 +26,7 @@
 
 #include <grpc/support/log.h>
 
-#include "src/core/lib/support/env.h"
+#include "src/core/lib/gpr/env.h"
 #include "test/core/util/port.h"
 #include "test/cpp/util/subprocess.h"
 
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index 85bbeed088e..55b10268ba1 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -950,6 +950,28 @@ src/core/lib/compression/stream_compression_identity.h \
 src/core/lib/debug/stats.h \
 src/core/lib/debug/stats_data.h \
 src/core/lib/debug/trace.h \
+src/core/lib/gpr++/abstract.h \
+src/core/lib/gpr++/atomic.h \
+src/core/lib/gpr++/atomic_with_atm.h \
+src/core/lib/gpr++/atomic_with_std.h \
+src/core/lib/gpr++/debug_location.h \
+src/core/lib/gpr++/inlined_vector.h \
+src/core/lib/gpr++/manual_constructor.h \
+src/core/lib/gpr++/memory.h \
+src/core/lib/gpr++/orphanable.h \
+src/core/lib/gpr++/ref_counted.h \
+src/core/lib/gpr++/ref_counted_ptr.h \
+src/core/lib/gpr/arena.h \
+src/core/lib/gpr/env.h \
+src/core/lib/gpr/fork.h \
+src/core/lib/gpr/mpscq.h \
+src/core/lib/gpr/murmur_hash.h \
+src/core/lib/gpr/spinlock.h \
+src/core/lib/gpr/string.h \
+src/core/lib/gpr/string_windows.h \
+src/core/lib/gpr/thd_internal.h \
+src/core/lib/gpr/time_precise.h \
+src/core/lib/gpr/tmpfile.h \
 src/core/lib/http/format_request.h \
 src/core/lib/http/httpcli.h \
 src/core/lib/http/parser.h \
@@ -1026,28 +1048,6 @@ src/core/lib/slice/percent_encoding.h \
 src/core/lib/slice/slice_hash_table.h \
 src/core/lib/slice/slice_internal.h \
 src/core/lib/slice/slice_string_helpers.h \
-src/core/lib/support/abstract.h \
-src/core/lib/support/arena.h \
-src/core/lib/support/atomic.h \
-src/core/lib/support/atomic_with_atm.h \
-src/core/lib/support/atomic_with_std.h \
-src/core/lib/support/debug_location.h \
-src/core/lib/support/env.h \
-src/core/lib/support/fork.h \
-src/core/lib/support/manual_constructor.h \
-src/core/lib/support/memory.h \
-src/core/lib/support/mpscq.h \
-src/core/lib/support/murmur_hash.h \
-src/core/lib/support/orphanable.h \
-src/core/lib/support/ref_counted.h \
-src/core/lib/support/ref_counted_ptr.h \
-src/core/lib/support/spinlock.h \
-src/core/lib/support/string.h \
-src/core/lib/support/string_windows.h \
-src/core/lib/support/thd_internal.h \
-src/core/lib/support/time_precise.h \
-src/core/lib/support/tmpfile.h \
-src/core/lib/support/vector.h \
 src/core/lib/surface/alarm_internal.h \
 src/core/lib/surface/api_trace.h \
 src/core/lib/surface/call.h \
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 4bf0fc74d1d..7857ec72fd4 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -1064,6 +1064,72 @@ src/core/lib/debug/stats_data.cc \
 src/core/lib/debug/stats_data.h \
 src/core/lib/debug/trace.cc \
 src/core/lib/debug/trace.h \
+src/core/lib/gpr++/README.md \
+src/core/lib/gpr++/abstract.h \
+src/core/lib/gpr++/atomic.h \
+src/core/lib/gpr++/atomic_with_atm.h \
+src/core/lib/gpr++/atomic_with_std.h \
+src/core/lib/gpr++/debug_location.h \
+src/core/lib/gpr++/inlined_vector.h \
+src/core/lib/gpr++/manual_constructor.h \
+src/core/lib/gpr++/memory.h \
+src/core/lib/gpr++/orphanable.h \
+src/core/lib/gpr++/ref_counted.h \
+src/core/lib/gpr++/ref_counted_ptr.h \
+src/core/lib/gpr/README.md \
+src/core/lib/gpr/alloc.cc \
+src/core/lib/gpr/arena.cc \
+src/core/lib/gpr/arena.h \
+src/core/lib/gpr/atm.cc \
+src/core/lib/gpr/avl.cc \
+src/core/lib/gpr/cmdline.cc \
+src/core/lib/gpr/cpu_iphone.cc \
+src/core/lib/gpr/cpu_linux.cc \
+src/core/lib/gpr/cpu_posix.cc \
+src/core/lib/gpr/cpu_windows.cc \
+src/core/lib/gpr/env.h \
+src/core/lib/gpr/env_linux.cc \
+src/core/lib/gpr/env_posix.cc \
+src/core/lib/gpr/env_windows.cc \
+src/core/lib/gpr/fork.cc \
+src/core/lib/gpr/fork.h \
+src/core/lib/gpr/host_port.cc \
+src/core/lib/gpr/log.cc \
+src/core/lib/gpr/log_android.cc \
+src/core/lib/gpr/log_linux.cc \
+src/core/lib/gpr/log_posix.cc \
+src/core/lib/gpr/log_windows.cc \
+src/core/lib/gpr/mpscq.cc \
+src/core/lib/gpr/mpscq.h \
+src/core/lib/gpr/murmur_hash.cc \
+src/core/lib/gpr/murmur_hash.h \
+src/core/lib/gpr/spinlock.h \
+src/core/lib/gpr/string.cc \
+src/core/lib/gpr/string.h \
+src/core/lib/gpr/string_posix.cc \
+src/core/lib/gpr/string_util_windows.cc \
+src/core/lib/gpr/string_windows.cc \
+src/core/lib/gpr/string_windows.h \
+src/core/lib/gpr/subprocess_posix.cc \
+src/core/lib/gpr/subprocess_windows.cc \
+src/core/lib/gpr/sync.cc \
+src/core/lib/gpr/sync_posix.cc \
+src/core/lib/gpr/sync_windows.cc \
+src/core/lib/gpr/thd.cc \
+src/core/lib/gpr/thd_internal.h \
+src/core/lib/gpr/thd_posix.cc \
+src/core/lib/gpr/thd_windows.cc \
+src/core/lib/gpr/time.cc \
+src/core/lib/gpr/time_posix.cc \
+src/core/lib/gpr/time_precise.cc \
+src/core/lib/gpr/time_precise.h \
+src/core/lib/gpr/time_windows.cc \
+src/core/lib/gpr/tls_pthread.cc \
+src/core/lib/gpr/tmpfile.h \
+src/core/lib/gpr/tmpfile_msys.cc \
+src/core/lib/gpr/tmpfile_posix.cc \
+src/core/lib/gpr/tmpfile_windows.cc \
+src/core/lib/gpr/wrap_memcpy.cc \
 src/core/lib/http/format_request.cc \
 src/core/lib/http/format_request.h \
 src/core/lib/http/httpcli.cc \
@@ -1271,70 +1337,6 @@ src/core/lib/slice/slice_intern.cc \
 src/core/lib/slice/slice_internal.h \
 src/core/lib/slice/slice_string_helpers.cc \
 src/core/lib/slice/slice_string_helpers.h \
-src/core/lib/support/abstract.h \
-src/core/lib/support/alloc.cc \
-src/core/lib/support/arena.cc \
-src/core/lib/support/arena.h \
-src/core/lib/support/atm.cc \
-src/core/lib/support/atomic.h \
-src/core/lib/support/atomic_with_atm.h \
-src/core/lib/support/atomic_with_std.h \
-src/core/lib/support/avl.cc \
-src/core/lib/support/cmdline.cc \
-src/core/lib/support/cpu_iphone.cc \
-src/core/lib/support/cpu_linux.cc \
-src/core/lib/support/cpu_posix.cc \
-src/core/lib/support/cpu_windows.cc \
-src/core/lib/support/debug_location.h \
-src/core/lib/support/env.h \
-src/core/lib/support/env_linux.cc \
-src/core/lib/support/env_posix.cc \
-src/core/lib/support/env_windows.cc \
-src/core/lib/support/fork.cc \
-src/core/lib/support/fork.h \
-src/core/lib/support/host_port.cc \
-src/core/lib/support/log.cc \
-src/core/lib/support/log_android.cc \
-src/core/lib/support/log_linux.cc \
-src/core/lib/support/log_posix.cc \
-src/core/lib/support/log_windows.cc \
-src/core/lib/support/manual_constructor.h \
-src/core/lib/support/memory.h \
-src/core/lib/support/mpscq.cc \
-src/core/lib/support/mpscq.h \
-src/core/lib/support/murmur_hash.cc \
-src/core/lib/support/murmur_hash.h \
-src/core/lib/support/orphanable.h \
-src/core/lib/support/ref_counted.h \
-src/core/lib/support/ref_counted_ptr.h \
-src/core/lib/support/spinlock.h \
-src/core/lib/support/string.cc \
-src/core/lib/support/string.h \
-src/core/lib/support/string_posix.cc \
-src/core/lib/support/string_util_windows.cc \
-src/core/lib/support/string_windows.cc \
-src/core/lib/support/string_windows.h \
-src/core/lib/support/subprocess_posix.cc \
-src/core/lib/support/subprocess_windows.cc \
-src/core/lib/support/sync.cc \
-src/core/lib/support/sync_posix.cc \
-src/core/lib/support/sync_windows.cc \
-src/core/lib/support/thd.cc \
-src/core/lib/support/thd_internal.h \
-src/core/lib/support/thd_posix.cc \
-src/core/lib/support/thd_windows.cc \
-src/core/lib/support/time.cc \
-src/core/lib/support/time_posix.cc \
-src/core/lib/support/time_precise.cc \
-src/core/lib/support/time_precise.h \
-src/core/lib/support/time_windows.cc \
-src/core/lib/support/tls_pthread.cc \
-src/core/lib/support/tmpfile.h \
-src/core/lib/support/tmpfile_msys.cc \
-src/core/lib/support/tmpfile_posix.cc \
-src/core/lib/support/tmpfile_windows.cc \
-src/core/lib/support/vector.h \
-src/core/lib/support/wrap_memcpy.cc \
 src/core/lib/surface/README.md \
 src/core/lib/surface/alarm.cc \
 src/core/lib/surface/alarm_internal.h \
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index df45489d5e8..982d9ef6496 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -45,7 +45,7 @@
     "language": "c", 
     "name": "alloc_test", 
     "src": [
-      "test/core/support/alloc_test.cc"
+      "test/core/gpr/alloc_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -94,7 +94,7 @@
     "language": "c", 
     "name": "arena_test", 
     "src": [
-      "test/core/support/arena_test.cc"
+      "test/core/gpr/arena_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -597,7 +597,7 @@
     "language": "c", 
     "name": "gpr_avl_test", 
     "src": [
-      "test/core/support/avl_test.cc"
+      "test/core/gpr/avl_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -612,7 +612,7 @@
     "language": "c", 
     "name": "gpr_cmdline_test", 
     "src": [
-      "test/core/support/cmdline_test.cc"
+      "test/core/gpr/cmdline_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -627,7 +627,7 @@
     "language": "c", 
     "name": "gpr_cpu_test", 
     "src": [
-      "test/core/support/cpu_test.cc"
+      "test/core/gpr/cpu_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -642,7 +642,7 @@
     "language": "c", 
     "name": "gpr_env_test", 
     "src": [
-      "test/core/support/env_test.cc"
+      "test/core/gpr/env_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -657,7 +657,7 @@
     "language": "c", 
     "name": "gpr_host_port_test", 
     "src": [
-      "test/core/support/host_port_test.cc"
+      "test/core/gpr/host_port_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -672,7 +672,7 @@
     "language": "c", 
     "name": "gpr_log_test", 
     "src": [
-      "test/core/support/log_test.cc"
+      "test/core/gpr/log_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -687,7 +687,7 @@
     "language": "c", 
     "name": "gpr_manual_constructor_test", 
     "src": [
-      "test/core/support/manual_constructor_test.cc"
+      "test/core/gpr++/manual_constructor_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -702,7 +702,7 @@
     "language": "c", 
     "name": "gpr_mpscq_test", 
     "src": [
-      "test/core/support/mpscq_test.cc"
+      "test/core/gpr/mpscq_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -717,7 +717,7 @@
     "language": "c", 
     "name": "gpr_spinlock_test", 
     "src": [
-      "test/core/support/spinlock_test.cc"
+      "test/core/gpr/spinlock_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -732,7 +732,7 @@
     "language": "c", 
     "name": "gpr_string_test", 
     "src": [
-      "test/core/support/string_test.cc"
+      "test/core/gpr/string_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -747,7 +747,7 @@
     "language": "c", 
     "name": "gpr_sync_test", 
     "src": [
-      "test/core/support/sync_test.cc"
+      "test/core/gpr/sync_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -762,7 +762,7 @@
     "language": "c", 
     "name": "gpr_thd_test", 
     "src": [
-      "test/core/support/thd_test.cc"
+      "test/core/gpr/thd_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -777,7 +777,7 @@
     "language": "c", 
     "name": "gpr_time_test", 
     "src": [
-      "test/core/support/time_test.cc"
+      "test/core/gpr/time_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -792,7 +792,7 @@
     "language": "c", 
     "name": "gpr_tls_test", 
     "src": [
-      "test/core/support/tls_test.cc"
+      "test/core/gpr/tls_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -807,7 +807,7 @@
     "language": "c", 
     "name": "gpr_useful_test", 
     "src": [
-      "test/core/support/useful_test.cc"
+      "test/core/gpr/useful_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -1636,7 +1636,7 @@
     "language": "c", 
     "name": "murmur_hash_test", 
     "src": [
-      "test/core/support/murmur_hash_test.cc"
+      "test/core/gpr/murmur_hash_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -3528,6 +3528,25 @@
     "third_party": false, 
     "type": "target"
   }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc++", 
+      "grpc++_test", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c++", 
+    "name": "inlined_vector_test", 
+    "src": [
+      "test/core/gpr++/inlined_vector_test.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
   {
     "deps": [
       "gpr", 
@@ -3643,7 +3662,7 @@
     "language": "c++", 
     "name": "memory_test", 
     "src": [
-      "test/core/support/memory_test.cc"
+      "test/core/gpr++/memory_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -3721,7 +3740,7 @@
     "language": "c++", 
     "name": "orphanable_test", 
     "src": [
-      "test/core/support/orphanable_test.cc"
+      "test/core/gpr++/orphanable_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -3932,7 +3951,7 @@
     "language": "c++", 
     "name": "ref_counted_ptr_test", 
     "src": [
-      "test/core/support/ref_counted_ptr_test.cc"
+      "test/core/gpr++/ref_counted_ptr_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -3951,7 +3970,7 @@
     "language": "c++", 
     "name": "ref_counted_test", 
     "src": [
-      "test/core/support/ref_counted_test.cc"
+      "test/core/gpr++/ref_counted_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -4300,25 +4319,6 @@
     "third_party": false, 
     "type": "target"
   }, 
-  {
-    "deps": [
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc++", 
-      "grpc++_test", 
-      "grpc_test_util"
-    ], 
-    "headers": [], 
-    "is_filegroup": false, 
-    "language": "c++", 
-    "name": "vector_test", 
-    "src": [
-      "test/core/support/vector_test.cc"
-    ], 
-    "third_party": false, 
-    "type": "target"
-  }, 
   {
     "deps": [
       "gpr", 
@@ -7827,50 +7827,50 @@
     "language": "c", 
     "name": "gpr_base", 
     "src": [
+      "src/core/lib/gpr/alloc.cc", 
+      "src/core/lib/gpr/arena.cc", 
+      "src/core/lib/gpr/atm.cc", 
+      "src/core/lib/gpr/avl.cc", 
+      "src/core/lib/gpr/cmdline.cc", 
+      "src/core/lib/gpr/cpu_iphone.cc", 
+      "src/core/lib/gpr/cpu_linux.cc", 
+      "src/core/lib/gpr/cpu_posix.cc", 
+      "src/core/lib/gpr/cpu_windows.cc", 
+      "src/core/lib/gpr/env_linux.cc", 
+      "src/core/lib/gpr/env_posix.cc", 
+      "src/core/lib/gpr/env_windows.cc", 
+      "src/core/lib/gpr/fork.cc", 
+      "src/core/lib/gpr/host_port.cc", 
+      "src/core/lib/gpr/log.cc", 
+      "src/core/lib/gpr/log_android.cc", 
+      "src/core/lib/gpr/log_linux.cc", 
+      "src/core/lib/gpr/log_posix.cc", 
+      "src/core/lib/gpr/log_windows.cc", 
+      "src/core/lib/gpr/mpscq.cc", 
+      "src/core/lib/gpr/murmur_hash.cc", 
+      "src/core/lib/gpr/string.cc", 
+      "src/core/lib/gpr/string_posix.cc", 
+      "src/core/lib/gpr/string_util_windows.cc", 
+      "src/core/lib/gpr/string_windows.cc", 
+      "src/core/lib/gpr/subprocess_posix.cc", 
+      "src/core/lib/gpr/subprocess_windows.cc", 
+      "src/core/lib/gpr/sync.cc", 
+      "src/core/lib/gpr/sync_posix.cc", 
+      "src/core/lib/gpr/sync_windows.cc", 
+      "src/core/lib/gpr/thd.cc", 
+      "src/core/lib/gpr/thd_posix.cc", 
+      "src/core/lib/gpr/thd_windows.cc", 
+      "src/core/lib/gpr/time.cc", 
+      "src/core/lib/gpr/time_posix.cc", 
+      "src/core/lib/gpr/time_precise.cc", 
+      "src/core/lib/gpr/time_windows.cc", 
+      "src/core/lib/gpr/tls_pthread.cc", 
+      "src/core/lib/gpr/tmpfile_msys.cc", 
+      "src/core/lib/gpr/tmpfile_posix.cc", 
+      "src/core/lib/gpr/tmpfile_windows.cc", 
+      "src/core/lib/gpr/wrap_memcpy.cc", 
       "src/core/lib/profiling/basic_timers.cc", 
-      "src/core/lib/profiling/stap_timers.cc", 
-      "src/core/lib/support/alloc.cc", 
-      "src/core/lib/support/arena.cc", 
-      "src/core/lib/support/atm.cc", 
-      "src/core/lib/support/avl.cc", 
-      "src/core/lib/support/cmdline.cc", 
-      "src/core/lib/support/cpu_iphone.cc", 
-      "src/core/lib/support/cpu_linux.cc", 
-      "src/core/lib/support/cpu_posix.cc", 
-      "src/core/lib/support/cpu_windows.cc", 
-      "src/core/lib/support/env_linux.cc", 
-      "src/core/lib/support/env_posix.cc", 
-      "src/core/lib/support/env_windows.cc", 
-      "src/core/lib/support/fork.cc", 
-      "src/core/lib/support/host_port.cc", 
-      "src/core/lib/support/log.cc", 
-      "src/core/lib/support/log_android.cc", 
-      "src/core/lib/support/log_linux.cc", 
-      "src/core/lib/support/log_posix.cc", 
-      "src/core/lib/support/log_windows.cc", 
-      "src/core/lib/support/mpscq.cc", 
-      "src/core/lib/support/murmur_hash.cc", 
-      "src/core/lib/support/string.cc", 
-      "src/core/lib/support/string_posix.cc", 
-      "src/core/lib/support/string_util_windows.cc", 
-      "src/core/lib/support/string_windows.cc", 
-      "src/core/lib/support/subprocess_posix.cc", 
-      "src/core/lib/support/subprocess_windows.cc", 
-      "src/core/lib/support/sync.cc", 
-      "src/core/lib/support/sync_posix.cc", 
-      "src/core/lib/support/sync_windows.cc", 
-      "src/core/lib/support/thd.cc", 
-      "src/core/lib/support/thd_posix.cc", 
-      "src/core/lib/support/thd_windows.cc", 
-      "src/core/lib/support/time.cc", 
-      "src/core/lib/support/time_posix.cc", 
-      "src/core/lib/support/time_precise.cc", 
-      "src/core/lib/support/time_windows.cc", 
-      "src/core/lib/support/tls_pthread.cc", 
-      "src/core/lib/support/tmpfile_msys.cc", 
-      "src/core/lib/support/tmpfile_posix.cc", 
-      "src/core/lib/support/tmpfile_windows.cc", 
-      "src/core/lib/support/wrap_memcpy.cc"
+      "src/core/lib/profiling/stap_timers.cc"
     ], 
     "third_party": false, 
     "type": "filegroup"
@@ -7906,24 +7906,24 @@
       "include/grpc/support/tls_msvc.h", 
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
-      "src/core/lib/profiling/timers.h", 
-      "src/core/lib/support/abstract.h", 
-      "src/core/lib/support/arena.h", 
-      "src/core/lib/support/atomic.h", 
-      "src/core/lib/support/atomic_with_atm.h", 
-      "src/core/lib/support/atomic_with_std.h", 
-      "src/core/lib/support/env.h", 
-      "src/core/lib/support/fork.h", 
-      "src/core/lib/support/manual_constructor.h", 
-      "src/core/lib/support/memory.h", 
-      "src/core/lib/support/mpscq.h", 
-      "src/core/lib/support/murmur_hash.h", 
-      "src/core/lib/support/spinlock.h", 
-      "src/core/lib/support/string.h", 
-      "src/core/lib/support/string_windows.h", 
-      "src/core/lib/support/thd_internal.h", 
-      "src/core/lib/support/time_precise.h", 
-      "src/core/lib/support/tmpfile.h"
+      "src/core/lib/gpr++/abstract.h", 
+      "src/core/lib/gpr++/atomic.h", 
+      "src/core/lib/gpr++/atomic_with_atm.h", 
+      "src/core/lib/gpr++/atomic_with_std.h", 
+      "src/core/lib/gpr++/manual_constructor.h", 
+      "src/core/lib/gpr++/memory.h", 
+      "src/core/lib/gpr/arena.h", 
+      "src/core/lib/gpr/env.h", 
+      "src/core/lib/gpr/fork.h", 
+      "src/core/lib/gpr/mpscq.h", 
+      "src/core/lib/gpr/murmur_hash.h", 
+      "src/core/lib/gpr/spinlock.h", 
+      "src/core/lib/gpr/string.h", 
+      "src/core/lib/gpr/string_windows.h", 
+      "src/core/lib/gpr/thd_internal.h", 
+      "src/core/lib/gpr/time_precise.h", 
+      "src/core/lib/gpr/tmpfile.h", 
+      "src/core/lib/profiling/timers.h"
     ], 
     "is_filegroup": true, 
     "language": "c", 
@@ -7955,24 +7955,24 @@
       "include/grpc/support/tls_msvc.h", 
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
-      "src/core/lib/profiling/timers.h", 
-      "src/core/lib/support/abstract.h", 
-      "src/core/lib/support/arena.h", 
-      "src/core/lib/support/atomic.h", 
-      "src/core/lib/support/atomic_with_atm.h", 
-      "src/core/lib/support/atomic_with_std.h", 
-      "src/core/lib/support/env.h", 
-      "src/core/lib/support/fork.h", 
-      "src/core/lib/support/manual_constructor.h", 
-      "src/core/lib/support/memory.h", 
-      "src/core/lib/support/mpscq.h", 
-      "src/core/lib/support/murmur_hash.h", 
-      "src/core/lib/support/spinlock.h", 
-      "src/core/lib/support/string.h", 
-      "src/core/lib/support/string_windows.h", 
-      "src/core/lib/support/thd_internal.h", 
-      "src/core/lib/support/time_precise.h", 
-      "src/core/lib/support/tmpfile.h"
+      "src/core/lib/gpr++/abstract.h", 
+      "src/core/lib/gpr++/atomic.h", 
+      "src/core/lib/gpr++/atomic_with_atm.h", 
+      "src/core/lib/gpr++/atomic_with_std.h", 
+      "src/core/lib/gpr++/manual_constructor.h", 
+      "src/core/lib/gpr++/memory.h", 
+      "src/core/lib/gpr/arena.h", 
+      "src/core/lib/gpr/env.h", 
+      "src/core/lib/gpr/fork.h", 
+      "src/core/lib/gpr/mpscq.h", 
+      "src/core/lib/gpr/murmur_hash.h", 
+      "src/core/lib/gpr/spinlock.h", 
+      "src/core/lib/gpr/string.h", 
+      "src/core/lib/gpr/string_windows.h", 
+      "src/core/lib/gpr/thd_internal.h", 
+      "src/core/lib/gpr/time_precise.h", 
+      "src/core/lib/gpr/tmpfile.h", 
+      "src/core/lib/profiling/timers.h"
     ], 
     "third_party": false, 
     "type": "filegroup"
@@ -8229,6 +8229,11 @@
       "src/core/lib/compression/stream_compression_identity.h", 
       "src/core/lib/debug/stats.h", 
       "src/core/lib/debug/stats_data.h", 
+      "src/core/lib/gpr++/debug_location.h", 
+      "src/core/lib/gpr++/inlined_vector.h", 
+      "src/core/lib/gpr++/orphanable.h", 
+      "src/core/lib/gpr++/ref_counted.h", 
+      "src/core/lib/gpr++/ref_counted_ptr.h", 
       "src/core/lib/http/format_request.h", 
       "src/core/lib/http/httpcli.h", 
       "src/core/lib/http/parser.h", 
@@ -8304,11 +8309,6 @@
       "src/core/lib/slice/slice_hash_table.h", 
       "src/core/lib/slice/slice_internal.h", 
       "src/core/lib/slice/slice_string_helpers.h", 
-      "src/core/lib/support/debug_location.h", 
-      "src/core/lib/support/orphanable.h", 
-      "src/core/lib/support/ref_counted.h", 
-      "src/core/lib/support/ref_counted_ptr.h", 
-      "src/core/lib/support/vector.h", 
       "src/core/lib/surface/alarm_internal.h", 
       "src/core/lib/surface/api_trace.h", 
       "src/core/lib/surface/call.h", 
@@ -8370,6 +8370,11 @@
       "src/core/lib/compression/stream_compression_identity.h", 
       "src/core/lib/debug/stats.h", 
       "src/core/lib/debug/stats_data.h", 
+      "src/core/lib/gpr++/debug_location.h", 
+      "src/core/lib/gpr++/inlined_vector.h", 
+      "src/core/lib/gpr++/orphanable.h", 
+      "src/core/lib/gpr++/ref_counted.h", 
+      "src/core/lib/gpr++/ref_counted_ptr.h", 
       "src/core/lib/http/format_request.h", 
       "src/core/lib/http/httpcli.h", 
       "src/core/lib/http/parser.h", 
@@ -8445,11 +8450,6 @@
       "src/core/lib/slice/slice_hash_table.h", 
       "src/core/lib/slice/slice_internal.h", 
       "src/core/lib/slice/slice_string_helpers.h", 
-      "src/core/lib/support/debug_location.h", 
-      "src/core/lib/support/orphanable.h", 
-      "src/core/lib/support/ref_counted.h", 
-      "src/core/lib/support/ref_counted_ptr.h", 
-      "src/core/lib/support/vector.h", 
       "src/core/lib/surface/alarm_internal.h", 
       "src/core/lib/surface/api_trace.h", 
       "src/core/lib/surface/call.h", 
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index dc2b39eb217..7b23cab5066 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -3961,6 +3961,30 @@
     ], 
     "uses_polling": true
   }, 
+  {
+    "args": [], 
+    "benchmark": false, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": true, 
+    "language": "c++", 
+    "name": "inlined_vector_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "uses_polling": true
+  }, 
   {
     "args": [], 
     "benchmark": false, 
@@ -4552,30 +4576,6 @@
     ], 
     "uses_polling": true
   }, 
-  {
-    "args": [], 
-    "benchmark": false, 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "cpu_cost": 1.0, 
-    "exclude_configs": [], 
-    "exclude_iomgrs": [], 
-    "flaky": false, 
-    "gtest": true, 
-    "language": "c++", 
-    "name": "vector_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "uses_polling": true
-  }, 
   {
     "args": [], 
     "benchmark": false, 
diff --git a/tools/run_tests/sanity/core_banned_functions.py b/tools/run_tests/sanity/core_banned_functions.py
index 9ee28964a5a..989990542e3 100755
--- a/tools/run_tests/sanity/core_banned_functions.py
+++ b/tools/run_tests/sanity/core_banned_functions.py
@@ -42,8 +42,8 @@ BANNED_EXCEPT = {
     'grpc_closure_run(': ['src/core/lib/iomgr/closure.c'],
     'grpc_closure_list_sched(': ['src/core/lib/iomgr/closure.c'],
     'gpr_getenv_silent(': [
-        'src/core/lib/support/log.c', 'src/core/lib/support/env_linux.c',
-        'src/core/lib/support/env_posix.c', 'src/core/lib/support/env_windows.c'
+        'src/core/lib/gpr/log.c', 'src/core/lib/gpr/env_linux.c',
+        'src/core/lib/gpr/env_posix.c', 'src/core/lib/gpr/env_windows.c'
     ],
 }
 

From 9cd22deaef7e168c96cfee64a3bc3e25725fad67 Mon Sep 17 00:00:00 2001
From: Yash Tibrewal <yashkt@google.com>
Date: Thu, 18 Jan 2018 14:56:45 -0800
Subject: [PATCH 153/161] Remove extraneous Finish in ExecCtx

---
 src/core/lib/iomgr/exec_ctx.h | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/core/lib/iomgr/exec_ctx.h b/src/core/lib/iomgr/exec_ctx.h
index 5c6007c3c70..2e71482fb70 100644
--- a/src/core/lib/iomgr/exec_ctx.h
+++ b/src/core/lib/iomgr/exec_ctx.h
@@ -158,10 +158,6 @@ on outside context */
     now_is_valid_ = true;
   }
 
-  /** Finish any pending work for a grpc_exec_ctx. Must be called before
-   *  the instance is destroyed, or work may be lost. */
-  void Finish();
-
   /** Global initialization for ExecCtx. Called by iomgr */
   static void GlobalInit(void);
 

From f0798f91ff0cc2cbb1424dd896a91972c47a8a7f Mon Sep 17 00:00:00 2001
From: Yash Tibrewal <yashkt@google.com>
Date: Thu, 18 Jan 2018 15:19:04 -0800
Subject: [PATCH 154/161] Solve a stack_use_after_scope issue in
 sockaddr_resolver_test

---
 test/core/client_channel/resolvers/sockaddr_resolver_test.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/core/client_channel/resolvers/sockaddr_resolver_test.cc b/test/core/client_channel/resolvers/sockaddr_resolver_test.cc
index 4d16a779245..645749ffefb 100644
--- a/test/core/client_channel/resolvers/sockaddr_resolver_test.cc
+++ b/test/core/client_channel/resolvers/sockaddr_resolver_test.cc
@@ -63,8 +63,10 @@ static void test_succeeds(grpc_resolver_factory* factory, const char* string) {
   grpc_resolver_next_locked(resolver, &on_res_arg.resolver_result,
                             on_resolution);
   GRPC_RESOLVER_UNREF(resolver, "test_succeeds");
-
   grpc_uri_destroy(uri);
+  /* Flush ExecCtx to avoid stack-use-after-scope on on_res_arg which is
+   * accessed in the closure on_resolition_cb */
+  grpc_core::ExecCtx::Get()->Flush();
 }
 
 static void test_fails(grpc_resolver_factory* factory, const char* string) {

From 45e754a5afe955526b9309f362953c8258da1cca Mon Sep 17 00:00:00 2001
From: Yash Tibrewal <yashkt@google.com>
Date: Thu, 18 Jan 2018 15:28:09 -0800
Subject: [PATCH 155/161] Fix typo s/resolition/resolution

---
 test/core/client_channel/resolvers/sockaddr_resolver_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/core/client_channel/resolvers/sockaddr_resolver_test.cc b/test/core/client_channel/resolvers/sockaddr_resolver_test.cc
index 645749ffefb..07ee133ee32 100644
--- a/test/core/client_channel/resolvers/sockaddr_resolver_test.cc
+++ b/test/core/client_channel/resolvers/sockaddr_resolver_test.cc
@@ -65,7 +65,7 @@ static void test_succeeds(grpc_resolver_factory* factory, const char* string) {
   GRPC_RESOLVER_UNREF(resolver, "test_succeeds");
   grpc_uri_destroy(uri);
   /* Flush ExecCtx to avoid stack-use-after-scope on on_res_arg which is
-   * accessed in the closure on_resolition_cb */
+   * accessed in the closure on_resolution_cb */
   grpc_core::ExecCtx::Get()->Flush();
 }
 

From a29d6d0d74e861883ec813e263ce90f0d3d63c03 Mon Sep 17 00:00:00 2001
From: yang-g <yangg@google.com>
Date: Thu, 18 Jan 2018 15:54:29 -0800
Subject: [PATCH 156/161] remove use of random_shuffle

---
 test/cpp/client/client_channel_stress_test.cc | 8 +++++---
 test/cpp/end2end/client_lb_end2end_test.cc    | 7 +++++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/test/cpp/client/client_channel_stress_test.cc b/test/cpp/client/client_channel_stress_test.cc
index e829d5278bd..80d15833334 100644
--- a/test/cpp/client/client_channel_stress_test.cc
+++ b/test/cpp/client/client_channel_stress_test.cc
@@ -19,6 +19,7 @@
 #include <atomic>
 #include <memory>
 #include <mutex>
+#include <random>
 #include <sstream>
 #include <thread>
 
@@ -104,8 +105,8 @@ class BalancerServiceImpl : public LoadBalancer::Service {
     for (size_t i = 0; i < num_drop_entry; ++i) {
       random_backend_indices.push_back(-1);
     }
-    std::random_shuffle(random_backend_indices.begin(),
-                        random_backend_indices.end());
+    std::shuffle(random_backend_indices.begin(), random_backend_indices.end(),
+                 std::mt19937(std::random_device()()));
     // Build the response according to the random list generated above.
     LoadBalanceResponse response;
     for (int index : random_backend_indices) {
@@ -149,7 +150,8 @@ class ClientChannelStressTest {
           addresses.emplace_back(AddressData{balancer_server.port_, true, ""});
         }
       }
-      std::random_shuffle(addresses.begin(), addresses.end());
+      std::shuffle(addresses.begin(), addresses.end(),
+                   std::mt19937(std::random_device()()));
       SetNextResolution(addresses);
       std::this_thread::sleep_for(wait_duration);
     }
diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc
index 02f9f904176..328ad860615 100644
--- a/test/cpp/end2end/client_lb_end2end_test.cc
+++ b/test/cpp/end2end/client_lb_end2end_test.cc
@@ -19,6 +19,7 @@
 #include <algorithm>
 #include <memory>
 #include <mutex>
+#include <random>
 #include <thread>
 
 #include <grpc++/channel.h>
@@ -456,7 +457,8 @@ TEST_F(ClientLbEnd2endTest, PickFirstManyUpdates) {
     grpc_subchannel_index_test_only_set_force_creation(force_creation);
     gpr_log(GPR_INFO, "Force subchannel creation: %d", force_creation);
     for (size_t i = 0; i < 1000; ++i) {
-      std::random_shuffle(ports.begin(), ports.end());
+      std::shuffle(ports.begin(), ports.end(),
+                   std::mt19937(std::random_device()()));
       SetNextResolution(ports);
       if (i % 10 == 0) CheckRpcSendOk();
     }
@@ -621,7 +623,8 @@ TEST_F(ClientLbEnd2endTest, RoundRobinManyUpdates) {
     ports.emplace_back(servers_[i]->port_);
   }
   for (size_t i = 0; i < 1000; ++i) {
-    std::random_shuffle(ports.begin(), ports.end());
+    std::shuffle(ports.begin(), ports.end(),
+                 std::mt19937(std::random_device()()));
     SetNextResolution(ports);
     if (i % 10 == 0) CheckRpcSendOk();
   }

From eb88bf03af7d4d441b0d6ef8127193dddefa9006 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Fri, 19 Jan 2018 18:37:45 +0100
Subject: [PATCH 157/161] dispose requestCallContextPool on environment
 shutdown

---
 src/csharp/Grpc.Core/GrpcEnvironment.cs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs
index 7b4342b74b9..6296f1863ba 100644
--- a/src/csharp/Grpc.Core/GrpcEnvironment.cs
+++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs
@@ -381,6 +381,7 @@ namespace Grpc.Core
             await Task.Run(() => ShuttingDown?.Invoke(this, null)).ConfigureAwait(false);
 
             await threadPool.StopAsync().ConfigureAwait(false);
+            requestCallContextPool.Dispose();
             batchContextPool.Dispose();
             GrpcNativeShutdown();
             isShutdown = true;

From 4f2b0fdadfb1e1eb6c796c40ec5aabfbb348aa89 Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Fri, 19 Jan 2018 12:12:23 -0800
Subject: [PATCH 158/161] Rename 'gpr++' directories to 'gprpp'.

---
 BUILD                                         | 30 ++++++----
 CMakeLists.txt                                | 12 ++--
 Makefile                                      | 24 ++++----
 build.yaml                                    | 34 +++++------
 gRPC-Core.podspec                             | 48 ++++++++--------
 grpc.gemspec                                  | 22 ++++----
 package.xml                                   | 22 ++++----
 .../ext/filters/client_channel/lb_policy.h    |  2 +-
 .../client_channel/lb_policy/grpclb/grpclb.cc |  2 +-
 .../lb_policy/round_robin/round_robin.cc      |  2 +-
 .../lb_policy/subchannel_list.h               |  2 +-
 .../resolver/dns/c_ares/dns_resolver_ares.cc  |  2 +-
 .../resolver/dns/native/dns_resolver.cc       |  2 +-
 .../ext/filters/client_channel/subchannel.cc  |  4 +-
 .../ext/filters/client_channel/subchannel.h   |  4 +-
 .../transport/chttp2/transport/flow_control.h |  4 +-
 .../ext/transport/chttp2/transport/internal.h |  2 +-
 src/core/lib/{gpr++ => gprpp}/README.md       |  0
 src/core/lib/{gpr++ => gprpp}/abstract.h      |  6 +-
 src/core/lib/{gpr++ => gprpp}/atomic.h        | 10 ++--
 .../lib/{gpr++ => gprpp}/atomic_with_atm.h    |  6 +-
 .../lib/{gpr++ => gprpp}/atomic_with_std.h    |  6 +-
 .../lib/{gpr++ => gprpp}/debug_location.h     |  6 +-
 .../lib/{gpr++ => gprpp}/inlined_vector.h     |  8 +--
 .../lib/{gpr++ => gprpp}/manual_constructor.h |  4 +-
 src/core/lib/{gpr++ => gprpp}/memory.h        |  6 +-
 src/core/lib/{gpr++ => gprpp}/orphanable.h    | 12 ++--
 src/core/lib/{gpr++ => gprpp}/ref_counted.h   | 12 ++--
 .../lib/{gpr++ => gprpp}/ref_counted_ptr.h    |  8 +--
 src/core/lib/iomgr/ev_epoll1_linux.cc         |  2 +-
 src/core/lib/iomgr/ev_epollex_linux.cc        |  2 +-
 src/core/lib/iomgr/ev_epollsig_linux.cc       |  2 +-
 src/core/lib/surface/lame_client.cc           |  2 +-
 test/core/{gpr++ => gprpp}/BUILD              |  7 ++-
 .../{gpr++ => gprpp}/inlined_vector_test.cc   |  4 +-
 .../manual_constructor_test.cc                |  4 +-
 test/core/{gpr++ => gprpp}/memory_test.cc     |  2 +-
 test/core/{gpr++ => gprpp}/orphanable_test.cc |  4 +-
 .../{gpr++ => gprpp}/ref_counted_ptr_test.cc  |  6 +-
 .../core/{gpr++ => gprpp}/ref_counted_test.cc |  4 +-
 tools/doxygen/Doxyfile.c++.internal           | 22 ++++----
 tools/doxygen/Doxyfile.core.internal          | 24 ++++----
 .../generated/sources_and_headers.json        | 56 +++++++++----------
 43 files changed, 226 insertions(+), 217 deletions(-)
 rename src/core/lib/{gpr++ => gprpp}/README.md (100%)
 rename src/core/lib/{gpr++ => gprpp}/abstract.h (90%)
 rename src/core/lib/{gpr++ => gprpp}/atomic.h (76%)
 rename src/core/lib/{gpr++ => gprpp}/atomic_with_atm.h (90%)
 rename src/core/lib/{gpr++ => gprpp}/atomic_with_std.h (84%)
 rename src/core/lib/{gpr++ => gprpp}/debug_location.h (90%)
 rename src/core/lib/{gpr++ => gprpp}/inlined_vector.h (94%)
 rename src/core/lib/{gpr++ => gprpp}/manual_constructor.h (98%)
 rename src/core/lib/{gpr++ => gprpp}/memory.h (95%)
 rename src/core/lib/{gpr++ => gprpp}/orphanable.h (94%)
 rename src/core/lib/{gpr++ => gprpp}/ref_counted.h (93%)
 rename src/core/lib/{gpr++ => gprpp}/ref_counted_ptr.h (93%)
 rename test/core/{gpr++ => gprpp}/BUILD (94%)
 rename test/core/{gpr++ => gprpp}/inlined_vector_test.cc (95%)
 rename test/core/{gpr++ => gprpp}/manual_constructor_test.cc (96%)
 rename test/core/{gpr++ => gprpp}/memory_test.cc (97%)
 rename test/core/{gpr++ => gprpp}/orphanable_test.cc (96%)
 rename test/core/{gpr++ => gprpp}/ref_counted_ptr_test.cc (97%)
 rename test/core/{gpr++ => gprpp}/ref_counted_test.cc (95%)

diff --git a/BUILD b/BUILD
index c52a95cf91b..603e4cc2c5c 100644
--- a/BUILD
+++ b/BUILD
@@ -548,9 +548,9 @@ grpc_cc_library(
     name = "gpr++_base",
     language = "c++",
     public_hdrs = [
-        "src/core/lib/gpr++/abstract.h",
-        "src/core/lib/gpr++/manual_constructor.h",
-        "src/core/lib/gpr++/memory.h",
+        "src/core/lib/gprpp/abstract.h",
+        "src/core/lib/gprpp/manual_constructor.h",
+        "src/core/lib/gprpp/memory.h",
     ],
 )
 
@@ -558,11 +558,11 @@ grpc_cc_library(
     name = "atomic",
     language = "c++",
     public_hdrs = [
-        "src/core/lib/gpr++/atomic.h",
+        "src/core/lib/gprpp/atomic.h",
     ],
     hdrs = [
-        "src/core/lib/gpr++/atomic_with_atm.h",
-        "src/core/lib/gpr++/atomic_with_std.h",
+        "src/core/lib/gprpp/atomic_with_atm.h",
+        "src/core/lib/gprpp/atomic_with_std.h",
     ],
     deps = [
         "gpr",
@@ -573,22 +573,26 @@ grpc_cc_library(
     name = "inlined_vector",
     language = "c++",
     public_hdrs = [
-        "src/core/lib/gpr++/inlined_vector.h",
+        "src/core/lib/gprpp/inlined_vector.h",
+    ],
+    deps = [
+        "gpr++_base",
     ],
 )
 
 grpc_cc_library(
     name = "debug_location",
     language = "c++",
-    public_hdrs = ["src/core/lib/gpr++/debug_location.h"],
+    public_hdrs = ["src/core/lib/gprpp/debug_location.h"],
 )
 
 grpc_cc_library(
     name = "orphanable",
     language = "c++",
-    public_hdrs = ["src/core/lib/gpr++/orphanable.h"],
+    public_hdrs = ["src/core/lib/gprpp/orphanable.h"],
     deps = [
         "debug_location",
+        "gpr++_base",
         "grpc_trace",
     ],
 )
@@ -596,9 +600,10 @@ grpc_cc_library(
 grpc_cc_library(
     name = "ref_counted",
     language = "c++",
-    public_hdrs = ["src/core/lib/gpr++/ref_counted.h"],
+    public_hdrs = ["src/core/lib/gprpp/ref_counted.h"],
     deps = [
         "debug_location",
+        "gpr++_base",
         "grpc_trace",
     ],
 )
@@ -606,7 +611,10 @@ grpc_cc_library(
 grpc_cc_library(
     name = "ref_counted_ptr",
     language = "c++",
-    public_hdrs = ["src/core/lib/gpr++/ref_counted_ptr.h"],
+    public_hdrs = ["src/core/lib/gprpp/ref_counted_ptr.h"],
+    deps = [
+        "gpr++_base",
+    ],
 )
 
 grpc_cc_library(
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4450a73e9d5..c3fc489b429 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -5785,7 +5785,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(gpr_manual_constructor_test
-  test/core/gpr++/manual_constructor_test.cc
+  test/core/gprpp/manual_constructor_test.cc
 )
 
 
@@ -10556,7 +10556,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(inlined_vector_test
-  test/core/gpr++/inlined_vector_test.cc
+  test/core/gprpp/inlined_vector_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
   third_party/googletest/googlemock/src/gmock-all.cc
 )
@@ -10801,7 +10801,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(memory_test
-  test/core/gpr++/memory_test.cc
+  test/core/gprpp/memory_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
   third_party/googletest/googlemock/src/gmock-all.cc
 )
@@ -10952,7 +10952,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(orphanable_test
-  test/core/gpr++/orphanable_test.cc
+  test/core/gprpp/orphanable_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
   third_party/googletest/googlemock/src/gmock-all.cc
 )
@@ -11352,7 +11352,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(ref_counted_ptr_test
-  test/core/gpr++/ref_counted_ptr_test.cc
+  test/core/gprpp/ref_counted_ptr_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
   third_party/googletest/googlemock/src/gmock-all.cc
 )
@@ -11389,7 +11389,7 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
 add_executable(ref_counted_test
-  test/core/gpr++/ref_counted_test.cc
+  test/core/gprpp/ref_counted_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
   third_party/googletest/googlemock/src/gmock-all.cc
 )
diff --git a/Makefile b/Makefile
index 69a68a6d66a..8f79ff78c02 100644
--- a/Makefile
+++ b/Makefile
@@ -10049,7 +10049,7 @@ endif
 
 
 GPR_MANUAL_CONSTRUCTOR_TEST_SRC = \
-    test/core/gpr++/manual_constructor_test.cc \
+    test/core/gprpp/manual_constructor_test.cc \
 
 GPR_MANUAL_CONSTRUCTOR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_MANUAL_CONSTRUCTOR_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -10069,7 +10069,7 @@ $(BINDIR)/$(CONFIG)/gpr_manual_constructor_test: $(GPR_MANUAL_CONSTRUCTOR_TEST_O
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/gpr++/manual_constructor_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gprpp/manual_constructor_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_gpr_manual_constructor_test: $(GPR_MANUAL_CONSTRUCTOR_TEST_OBJS:.o=.dep)
 
@@ -15779,7 +15779,7 @@ endif
 
 
 INLINED_VECTOR_TEST_SRC = \
-    test/core/gpr++/inlined_vector_test.cc \
+    test/core/gprpp/inlined_vector_test.cc \
 
 INLINED_VECTOR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(INLINED_VECTOR_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -15810,7 +15810,7 @@ endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/gpr++/inlined_vector_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gprpp/inlined_vector_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_inlined_vector_test: $(INLINED_VECTOR_TEST_OBJS:.o=.dep)
 
@@ -16013,7 +16013,7 @@ endif
 
 
 MEMORY_TEST_SRC = \
-    test/core/gpr++/memory_test.cc \
+    test/core/gprpp/memory_test.cc \
 
 MEMORY_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(MEMORY_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -16044,7 +16044,7 @@ endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/gpr++/memory_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gprpp/memory_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_memory_test: $(MEMORY_TEST_OBJS:.o=.dep)
 
@@ -16190,7 +16190,7 @@ endif
 
 
 ORPHANABLE_TEST_SRC = \
-    test/core/gpr++/orphanable_test.cc \
+    test/core/gprpp/orphanable_test.cc \
 
 ORPHANABLE_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(ORPHANABLE_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -16221,7 +16221,7 @@ endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/gpr++/orphanable_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gprpp/orphanable_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_orphanable_test: $(ORPHANABLE_TEST_OBJS:.o=.dep)
 
@@ -16597,7 +16597,7 @@ $(OBJDIR)/$(CONFIG)/test/cpp/interop/reconnect_interop_server.o: $(GENDIR)/src/p
 
 
 REF_COUNTED_PTR_TEST_SRC = \
-    test/core/gpr++/ref_counted_ptr_test.cc \
+    test/core/gprpp/ref_counted_ptr_test.cc \
 
 REF_COUNTED_PTR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(REF_COUNTED_PTR_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -16628,7 +16628,7 @@ endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/gpr++/ref_counted_ptr_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gprpp/ref_counted_ptr_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_ref_counted_ptr_test: $(REF_COUNTED_PTR_TEST_OBJS:.o=.dep)
 
@@ -16640,7 +16640,7 @@ endif
 
 
 REF_COUNTED_TEST_SRC = \
-    test/core/gpr++/ref_counted_test.cc \
+    test/core/gprpp/ref_counted_test.cc \
 
 REF_COUNTED_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(REF_COUNTED_TEST_SRC))))
 ifeq ($(NO_SECURE),true)
@@ -16671,7 +16671,7 @@ endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/core/gpr++/ref_counted_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/core/gprpp/ref_counted_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_ref_counted_test: $(REF_COUNTED_TEST_OBJS:.o=.dep)
 
diff --git a/build.yaml b/build.yaml
index a8639a8b0b8..5a16d7d701a 100644
--- a/build.yaml
+++ b/build.yaml
@@ -101,12 +101,6 @@ filegroups:
   - include/grpc/support/tls_pthread.h
   - include/grpc/support/useful.h
   headers:
-  - src/core/lib/gpr++/abstract.h
-  - src/core/lib/gpr++/atomic.h
-  - src/core/lib/gpr++/atomic_with_atm.h
-  - src/core/lib/gpr++/atomic_with_std.h
-  - src/core/lib/gpr++/manual_constructor.h
-  - src/core/lib/gpr++/memory.h
   - src/core/lib/gpr/arena.h
   - src/core/lib/gpr/env.h
   - src/core/lib/gpr/fork.h
@@ -118,6 +112,12 @@ filegroups:
   - src/core/lib/gpr/thd_internal.h
   - src/core/lib/gpr/time_precise.h
   - src/core/lib/gpr/tmpfile.h
+  - src/core/lib/gprpp/abstract.h
+  - src/core/lib/gprpp/atomic.h
+  - src/core/lib/gprpp/atomic_with_atm.h
+  - src/core/lib/gprpp/atomic_with_std.h
+  - src/core/lib/gprpp/manual_constructor.h
+  - src/core/lib/gprpp/memory.h
   - src/core/lib/profiling/timers.h
   uses:
   - gpr_codegen
@@ -321,11 +321,11 @@ filegroups:
   - src/core/lib/compression/stream_compression_identity.h
   - src/core/lib/debug/stats.h
   - src/core/lib/debug/stats_data.h
-  - src/core/lib/gpr++/debug_location.h
-  - src/core/lib/gpr++/inlined_vector.h
-  - src/core/lib/gpr++/orphanable.h
-  - src/core/lib/gpr++/ref_counted.h
-  - src/core/lib/gpr++/ref_counted_ptr.h
+  - src/core/lib/gprpp/debug_location.h
+  - src/core/lib/gprpp/inlined_vector.h
+  - src/core/lib/gprpp/orphanable.h
+  - src/core/lib/gprpp/ref_counted.h
+  - src/core/lib/gprpp/ref_counted_ptr.h
   - src/core/lib/http/format_request.h
   - src/core/lib/http/httpcli.h
   - src/core/lib/http/parser.h
@@ -2188,7 +2188,7 @@ targets:
   build: test
   language: c
   src:
-  - test/core/gpr++/manual_constructor_test.cc
+  - test/core/gprpp/manual_constructor_test.cc
   deps:
   - gpr_test_util
   - gpr
@@ -4265,7 +4265,7 @@ targets:
   build: test
   language: c++
   src:
-  - test/core/gpr++/inlined_vector_test.cc
+  - test/core/gprpp/inlined_vector_test.cc
   deps:
   - grpc_test_util
   - grpc++
@@ -4371,7 +4371,7 @@ targets:
   build: test
   language: c++
   src:
-  - test/core/gpr++/memory_test.cc
+  - test/core/gprpp/memory_test.cc
   deps:
   - grpc_test_util
   - grpc++
@@ -4423,7 +4423,7 @@ targets:
   build: test
   language: c++
   src:
-  - test/core/gpr++/orphanable_test.cc
+  - test/core/gprpp/orphanable_test.cc
   deps:
   - grpc_test_util
   - grpc++
@@ -4576,7 +4576,7 @@ targets:
   build: test
   language: c++
   src:
-  - test/core/gpr++/ref_counted_ptr_test.cc
+  - test/core/gprpp/ref_counted_ptr_test.cc
   deps:
   - grpc_test_util
   - grpc++
@@ -4590,7 +4590,7 @@ targets:
   build: test
   language: c++
   src:
-  - test/core/gpr++/ref_counted_test.cc
+  - test/core/gprpp/ref_counted_test.cc
   deps:
   - grpc_test_util
   - grpc++
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index ccaa21194b8..3ebd121e4c5 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -192,13 +192,7 @@ Pod::Spec.new do |s|
     ss.dependency 'nanopb', '~> 0.3'
 
     # To save you from scrolling, this is the last part of the podspec.
-    ss.source_files = 'src/core/lib/gpr++/abstract.h',
-                      'src/core/lib/gpr++/atomic.h',
-                      'src/core/lib/gpr++/atomic_with_atm.h',
-                      'src/core/lib/gpr++/atomic_with_std.h',
-                      'src/core/lib/gpr++/manual_constructor.h',
-                      'src/core/lib/gpr++/memory.h',
-                      'src/core/lib/gpr/arena.h',
+    ss.source_files = 'src/core/lib/gpr/arena.h',
                       'src/core/lib/gpr/env.h',
                       'src/core/lib/gpr/fork.h',
                       'src/core/lib/gpr/mpscq.h',
@@ -209,6 +203,12 @@ Pod::Spec.new do |s|
                       'src/core/lib/gpr/thd_internal.h',
                       'src/core/lib/gpr/time_precise.h',
                       'src/core/lib/gpr/tmpfile.h',
+                      'src/core/lib/gprpp/abstract.h',
+                      'src/core/lib/gprpp/atomic.h',
+                      'src/core/lib/gprpp/atomic_with_atm.h',
+                      'src/core/lib/gprpp/atomic_with_std.h',
+                      'src/core/lib/gprpp/manual_constructor.h',
+                      'src/core/lib/gprpp/memory.h',
                       'src/core/lib/profiling/timers.h',
                       'src/core/lib/gpr/alloc.cc',
                       'src/core/lib/gpr/arena.cc',
@@ -344,11 +344,11 @@ Pod::Spec.new do |s|
                       'src/core/lib/compression/stream_compression_identity.h',
                       'src/core/lib/debug/stats.h',
                       'src/core/lib/debug/stats_data.h',
-                      'src/core/lib/gpr++/debug_location.h',
-                      'src/core/lib/gpr++/inlined_vector.h',
-                      'src/core/lib/gpr++/orphanable.h',
-                      'src/core/lib/gpr++/ref_counted.h',
-                      'src/core/lib/gpr++/ref_counted_ptr.h',
+                      'src/core/lib/gprpp/debug_location.h',
+                      'src/core/lib/gprpp/inlined_vector.h',
+                      'src/core/lib/gprpp/orphanable.h',
+                      'src/core/lib/gprpp/ref_counted.h',
+                      'src/core/lib/gprpp/ref_counted_ptr.h',
                       'src/core/lib/http/format_request.h',
                       'src/core/lib/http/httpcli.h',
                       'src/core/lib/http/parser.h',
@@ -718,13 +718,7 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/workarounds/workaround_utils.cc',
                       'src/core/plugin_registry/grpc_plugin_registry.cc'
 
-    ss.private_header_files = 'src/core/lib/gpr++/abstract.h',
-                              'src/core/lib/gpr++/atomic.h',
-                              'src/core/lib/gpr++/atomic_with_atm.h',
-                              'src/core/lib/gpr++/atomic_with_std.h',
-                              'src/core/lib/gpr++/manual_constructor.h',
-                              'src/core/lib/gpr++/memory.h',
-                              'src/core/lib/gpr/arena.h',
+    ss.private_header_files = 'src/core/lib/gpr/arena.h',
                               'src/core/lib/gpr/env.h',
                               'src/core/lib/gpr/fork.h',
                               'src/core/lib/gpr/mpscq.h',
@@ -735,6 +729,12 @@ Pod::Spec.new do |s|
                               'src/core/lib/gpr/thd_internal.h',
                               'src/core/lib/gpr/time_precise.h',
                               'src/core/lib/gpr/tmpfile.h',
+                              'src/core/lib/gprpp/abstract.h',
+                              'src/core/lib/gprpp/atomic.h',
+                              'src/core/lib/gprpp/atomic_with_atm.h',
+                              'src/core/lib/gprpp/atomic_with_std.h',
+                              'src/core/lib/gprpp/manual_constructor.h',
+                              'src/core/lib/gprpp/memory.h',
                               'src/core/lib/profiling/timers.h',
                               'src/core/ext/transport/chttp2/transport/bin_decoder.h',
                               'src/core/ext/transport/chttp2/transport/bin_encoder.h',
@@ -826,11 +826,11 @@ Pod::Spec.new do |s|
                               'src/core/lib/compression/stream_compression_identity.h',
                               'src/core/lib/debug/stats.h',
                               'src/core/lib/debug/stats_data.h',
-                              'src/core/lib/gpr++/debug_location.h',
-                              'src/core/lib/gpr++/inlined_vector.h',
-                              'src/core/lib/gpr++/orphanable.h',
-                              'src/core/lib/gpr++/ref_counted.h',
-                              'src/core/lib/gpr++/ref_counted_ptr.h',
+                              'src/core/lib/gprpp/debug_location.h',
+                              'src/core/lib/gprpp/inlined_vector.h',
+                              'src/core/lib/gprpp/orphanable.h',
+                              'src/core/lib/gprpp/ref_counted.h',
+                              'src/core/lib/gprpp/ref_counted_ptr.h',
                               'src/core/lib/http/format_request.h',
                               'src/core/lib/http/httpcli.h',
                               'src/core/lib/http/parser.h',
diff --git a/grpc.gemspec b/grpc.gemspec
index 5b1aebeaaa4..35b2776eba4 100644
--- a/grpc.gemspec
+++ b/grpc.gemspec
@@ -83,12 +83,6 @@ Gem::Specification.new do |s|
   s.files += %w( include/grpc/impl/codegen/sync_generic.h )
   s.files += %w( include/grpc/impl/codegen/sync_posix.h )
   s.files += %w( include/grpc/impl/codegen/sync_windows.h )
-  s.files += %w( src/core/lib/gpr++/abstract.h )
-  s.files += %w( src/core/lib/gpr++/atomic.h )
-  s.files += %w( src/core/lib/gpr++/atomic_with_atm.h )
-  s.files += %w( src/core/lib/gpr++/atomic_with_std.h )
-  s.files += %w( src/core/lib/gpr++/manual_constructor.h )
-  s.files += %w( src/core/lib/gpr++/memory.h )
   s.files += %w( src/core/lib/gpr/arena.h )
   s.files += %w( src/core/lib/gpr/env.h )
   s.files += %w( src/core/lib/gpr/fork.h )
@@ -100,6 +94,12 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/gpr/thd_internal.h )
   s.files += %w( src/core/lib/gpr/time_precise.h )
   s.files += %w( src/core/lib/gpr/tmpfile.h )
+  s.files += %w( src/core/lib/gprpp/abstract.h )
+  s.files += %w( src/core/lib/gprpp/atomic.h )
+  s.files += %w( src/core/lib/gprpp/atomic_with_atm.h )
+  s.files += %w( src/core/lib/gprpp/atomic_with_std.h )
+  s.files += %w( src/core/lib/gprpp/manual_constructor.h )
+  s.files += %w( src/core/lib/gprpp/memory.h )
   s.files += %w( src/core/lib/profiling/timers.h )
   s.files += %w( src/core/lib/gpr/alloc.cc )
   s.files += %w( src/core/lib/gpr/arena.cc )
@@ -270,11 +270,11 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/compression/stream_compression_identity.h )
   s.files += %w( src/core/lib/debug/stats.h )
   s.files += %w( src/core/lib/debug/stats_data.h )
-  s.files += %w( src/core/lib/gpr++/debug_location.h )
-  s.files += %w( src/core/lib/gpr++/inlined_vector.h )
-  s.files += %w( src/core/lib/gpr++/orphanable.h )
-  s.files += %w( src/core/lib/gpr++/ref_counted.h )
-  s.files += %w( src/core/lib/gpr++/ref_counted_ptr.h )
+  s.files += %w( src/core/lib/gprpp/debug_location.h )
+  s.files += %w( src/core/lib/gprpp/inlined_vector.h )
+  s.files += %w( src/core/lib/gprpp/orphanable.h )
+  s.files += %w( src/core/lib/gprpp/ref_counted.h )
+  s.files += %w( src/core/lib/gprpp/ref_counted_ptr.h )
   s.files += %w( src/core/lib/http/format_request.h )
   s.files += %w( src/core/lib/http/httpcli.h )
   s.files += %w( src/core/lib/http/parser.h )
diff --git a/package.xml b/package.xml
index f1282964c2b..875120c6e49 100644
--- a/package.xml
+++ b/package.xml
@@ -95,12 +95,6 @@
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_generic.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_posix.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_windows.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr++/abstract.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr++/atomic.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr++/atomic_with_atm.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr++/atomic_with_std.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr++/manual_constructor.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr++/memory.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/arena.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/env.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/fork.h" role="src" />
@@ -112,6 +106,12 @@
     <file baseinstalldir="/" name="src/core/lib/gpr/thd_internal.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/time_precise.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/tmpfile.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/abstract.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/atomic.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/atomic_with_atm.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/atomic_with_std.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/manual_constructor.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/memory.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/profiling/timers.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/alloc.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/arena.cc" role="src" />
@@ -282,11 +282,11 @@
     <file baseinstalldir="/" name="src/core/lib/compression/stream_compression_identity.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/debug/stats.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/debug/stats_data.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr++/debug_location.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr++/inlined_vector.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr++/orphanable.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr++/ref_counted.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr++/ref_counted_ptr.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/debug_location.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/inlined_vector.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/orphanable.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/ref_counted.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/ref_counted_ptr.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/format_request.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/httpcli.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/parser.h" role="src" />
diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h
index e19726efb3b..30660cb83d4 100644
--- a/src/core/ext/filters/client_channel/lb_policy.h
+++ b/src/core/ext/filters/client_channel/lb_policy.h
@@ -20,7 +20,7 @@
 #define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_H
 
 #include "src/core/ext/filters/client_channel/subchannel.h"
-#include "src/core/lib/gpr++/ref_counted_ptr.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
index eb5ced4c201..b89a28c077f 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
@@ -106,8 +106,8 @@
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/gpr++/manual_constructor.h"
 #include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
index e217a0b0c00..24c381a46dd 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc
@@ -34,7 +34,7 @@
 #include "src/core/ext/filters/client_channel/subchannel_index.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gpr++/ref_counted_ptr.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/transport/connectivity_state.h"
diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
index f4e345def6b..33776052631 100644
--- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
+++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h
@@ -22,7 +22,7 @@
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/ext/filters/client_channel/subchannel.h"
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gpr++/ref_counted_ptr.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
 // TODO(roth): This code is intended to be shared between pick_first and
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
index 7ea3cdd6e19..9024ffb0927 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
@@ -34,9 +34,9 @@
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/gpr++/manual_constructor.h"
 #include "src/core/lib/gpr/env.h"
 #include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/gethostname.h"
 #include "src/core/lib/iomgr/resolve_address.h"
diff --git a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
index 93a1fe87a28..5aa7e6cc7e2 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
+++ b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
@@ -29,9 +29,9 @@
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/gpr++/manual_constructor.h"
 #include "src/core/lib/gpr/env.h"
 #include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/timer.h"
diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc
index fe4fcbbb7d3..3edefaae64e 100644
--- a/src/core/ext/filters/client_channel/subchannel.cc
+++ b/src/core/ext/filters/client_channel/subchannel.cc
@@ -37,8 +37,8 @@
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/connected_channel.h"
 #include "src/core/lib/debug/stats.h"
-#include "src/core/lib/gpr++/debug_location.h"
-#include "src/core/lib/gpr++/manual_constructor.h"
+#include "src/core/lib/gprpp/debug_location.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h
index f2a5c1e2732..b7593ec911b 100644
--- a/src/core/ext/filters/client_channel/subchannel.h
+++ b/src/core/ext/filters/client_channel/subchannel.h
@@ -21,9 +21,9 @@
 
 #include "src/core/ext/filters/client_channel/connector.h"
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/gpr++/ref_counted.h"
-#include "src/core/lib/gpr++/ref_counted_ptr.h"
 #include "src/core/lib/gpr/arena.h"
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/metadata.h"
diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h
index 7e83ea05cd5..2ee1345260d 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.h
+++ b/src/core/ext/transport/chttp2/transport/flow_control.h
@@ -24,8 +24,8 @@
 
 #include <grpc/support/useful.h>
 #include "src/core/ext/transport/chttp2/transport/http2_settings.h"
-#include "src/core/lib/gpr++/abstract.h"
-#include "src/core/lib/gpr++/manual_constructor.h"
+#include "src/core/lib/gprpp/abstract.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/transport/bdp_estimator.h"
 #include "src/core/lib/transport/pid_controller.h"
 
diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h
index de901f0ce89..6b6c0b28e2a 100644
--- a/src/core/ext/transport/chttp2/transport/internal.h
+++ b/src/core/ext/transport/chttp2/transport/internal.h
@@ -35,7 +35,7 @@
 #include "src/core/ext/transport/chttp2/transport/incoming_metadata.h"
 #include "src/core/ext/transport/chttp2/transport/stream_map.h"
 #include "src/core/lib/compression/stream_compression.h"
-#include "src/core/lib/gpr++/manual_constructor.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/timer.h"
diff --git a/src/core/lib/gpr++/README.md b/src/core/lib/gprpp/README.md
similarity index 100%
rename from src/core/lib/gpr++/README.md
rename to src/core/lib/gprpp/README.md
diff --git a/src/core/lib/gpr++/abstract.h b/src/core/lib/gprpp/abstract.h
similarity index 90%
rename from src/core/lib/gpr++/abstract.h
rename to src/core/lib/gprpp/abstract.h
index 51d75723026..cc96edc49b7 100644
--- a/src/core/lib/gpr++/abstract.h
+++ b/src/core/lib/gprpp/abstract.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_GPRXX_ABSTRACT_H
-#define GRPC_CORE_LIB_GPRXX_ABSTRACT_H
+#ifndef GRPC_CORE_LIB_GPRPP_ABSTRACT_H
+#define GRPC_CORE_LIB_GPRPP_ABSTRACT_H
 
 // This is needed to support abstract base classes in the c core. Since gRPC
 // doesn't have a c++ runtime, it will hit a linker error on delete unless
@@ -31,4 +31,4 @@
 #define GRPC_ABSTRACT \
   { GPR_ASSERT(false); }
 
-#endif /* GRPC_CORE_LIB_GPRXX_ABSTRACT_H */
+#endif /* GRPC_CORE_LIB_GPRPP_ABSTRACT_H */
diff --git a/src/core/lib/gpr++/atomic.h b/src/core/lib/gprpp/atomic.h
similarity index 76%
rename from src/core/lib/gpr++/atomic.h
rename to src/core/lib/gprpp/atomic.h
index d68ccea864f..8b08fc4e9c4 100644
--- a/src/core/lib/gpr++/atomic.h
+++ b/src/core/lib/gprpp/atomic.h
@@ -16,15 +16,15 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_GPRXX_ATOMIC_H
-#define GRPC_CORE_LIB_GPRXX_ATOMIC_H
+#ifndef GRPC_CORE_LIB_GPRPP_ATOMIC_H
+#define GRPC_CORE_LIB_GPRPP_ATOMIC_H
 
 #include <grpc/support/port_platform.h>
 
 #ifdef GPR_HAS_CXX11_ATOMIC
-#include "src/core/lib/gpr++/atomic_with_std.h"
+#include "src/core/lib/gprpp/atomic_with_std.h"
 #else
-#include "src/core/lib/gpr++/atomic_with_atm.h"
+#include "src/core/lib/gprpp/atomic_with_atm.h"
 #endif
 
-#endif /* GRPC_CORE_LIB_GPRXX_ATOMIC_H */
+#endif /* GRPC_CORE_LIB_GPRPP_ATOMIC_H */
diff --git a/src/core/lib/gpr++/atomic_with_atm.h b/src/core/lib/gprpp/atomic_with_atm.h
similarity index 90%
rename from src/core/lib/gpr++/atomic_with_atm.h
rename to src/core/lib/gprpp/atomic_with_atm.h
index 09490e8148e..6abf0bc38dd 100644
--- a/src/core/lib/gpr++/atomic_with_atm.h
+++ b/src/core/lib/gprpp/atomic_with_atm.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_ATM_H
-#define GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_ATM_H
+#ifndef GRPC_CORE_LIB_GPRPP_ATOMIC_WITH_ATM_H
+#define GRPC_CORE_LIB_GPRPP_ATOMIC_WITH_ATM_H
 
 #include <grpc/support/atm.h>
 
@@ -52,4 +52,4 @@ class atomic<bool> {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_ATM_H */
+#endif /* GRPC_CORE_LIB_GPRPP_ATOMIC_WITH_ATM_H */
diff --git a/src/core/lib/gpr++/atomic_with_std.h b/src/core/lib/gprpp/atomic_with_std.h
similarity index 84%
rename from src/core/lib/gpr++/atomic_with_std.h
rename to src/core/lib/gprpp/atomic_with_std.h
index b6ff90655e9..83322b81c1a 100644
--- a/src/core/lib/gpr++/atomic_with_std.h
+++ b/src/core/lib/gprpp/atomic_with_std.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_STD_H
-#define GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_STD_H
+#ifndef GRPC_CORE_LIB_GPRPP_ATOMIC_WITH_STD_H
+#define GRPC_CORE_LIB_GPRPP_ATOMIC_WITH_STD_H
 
 #include <atomic>
 
@@ -30,4 +30,4 @@ typedef std::memory_order memory_order;
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_GPRXX_ATOMIC_WITH_STD_H */
+#endif /* GRPC_CORE_LIB_GPRPP_ATOMIC_WITH_STD_H */
diff --git a/src/core/lib/gpr++/debug_location.h b/src/core/lib/gprpp/debug_location.h
similarity index 90%
rename from src/core/lib/gpr++/debug_location.h
rename to src/core/lib/gprpp/debug_location.h
index 5a8665ce193..287761beafb 100644
--- a/src/core/lib/gpr++/debug_location.h
+++ b/src/core/lib/gprpp/debug_location.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_GPRXX_DEBUG_LOCATION_H
-#define GRPC_CORE_LIB_GPRXX_DEBUG_LOCATION_H
+#ifndef GRPC_CORE_LIB_GPRPP_DEBUG_LOCATION_H
+#define GRPC_CORE_LIB_GPRPP_DEBUG_LOCATION_H
 
 namespace grpc_core {
 
@@ -49,4 +49,4 @@ class DebugLocation {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_GPRXX_DEBUG_LOCATION_H */
+#endif /* GRPC_CORE_LIB_GPRPP_DEBUG_LOCATION_H */
diff --git a/src/core/lib/gpr++/inlined_vector.h b/src/core/lib/gprpp/inlined_vector.h
similarity index 94%
rename from src/core/lib/gpr++/inlined_vector.h
rename to src/core/lib/gprpp/inlined_vector.h
index 17ee9e16bbe..b78f85b8931 100644
--- a/src/core/lib/gpr++/inlined_vector.h
+++ b/src/core/lib/gprpp/inlined_vector.h
@@ -16,12 +16,12 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_GPRXX_INLINED_VECTOR_H
-#define GRPC_CORE_LIB_GPRXX_INLINED_VECTOR_H
+#ifndef GRPC_CORE_LIB_GPRPP_INLINED_VECTOR_H
+#define GRPC_CORE_LIB_GPRPP_INLINED_VECTOR_H
 
 #include <cassert>
 
-#include "src/core/lib/gpr++/memory.h"
+#include "src/core/lib/gprpp/memory.h"
 
 namespace grpc_core {
 
@@ -109,4 +109,4 @@ class InlinedVector {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_GPRXX_INLINED_VECTOR_H */
+#endif /* GRPC_CORE_LIB_GPRPP_INLINED_VECTOR_H */
diff --git a/src/core/lib/gpr++/manual_constructor.h b/src/core/lib/gprpp/manual_constructor.h
similarity index 98%
rename from src/core/lib/gpr++/manual_constructor.h
rename to src/core/lib/gprpp/manual_constructor.h
index a3f006da341..cee38abc1bb 100644
--- a/src/core/lib/gpr++/manual_constructor.h
+++ b/src/core/lib/gprpp/manual_constructor.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_GPRXX_MANUAL_CONSTRUCTOR_H
-#define GRPC_CORE_LIB_GPRXX_MANUAL_CONSTRUCTOR_H
+#ifndef GRPC_CORE_LIB_GPRPP_MANUAL_CONSTRUCTOR_H
+#define GRPC_CORE_LIB_GPRPP_MANUAL_CONSTRUCTOR_H
 
 // manually construct a region of memory with some type
 
diff --git a/src/core/lib/gpr++/memory.h b/src/core/lib/gprpp/memory.h
similarity index 95%
rename from src/core/lib/gpr++/memory.h
rename to src/core/lib/gprpp/memory.h
index 75ed3d6ceae..17f42f59833 100644
--- a/src/core/lib/gpr++/memory.h
+++ b/src/core/lib/gprpp/memory.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_GPRXX_MEMORY_H
-#define GRPC_CORE_LIB_GPRXX_MEMORY_H
+#ifndef GRPC_CORE_LIB_GPRPP_MEMORY_H
+#define GRPC_CORE_LIB_GPRPP_MEMORY_H
 
 #include <grpc/support/alloc.h>
 
@@ -97,4 +97,4 @@ class Allocator {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_GPRXX_MEMORY_H */
+#endif /* GRPC_CORE_LIB_GPRPP_MEMORY_H */
diff --git a/src/core/lib/gpr++/orphanable.h b/src/core/lib/gprpp/orphanable.h
similarity index 94%
rename from src/core/lib/gpr++/orphanable.h
rename to src/core/lib/gprpp/orphanable.h
index f106e74dde0..50199730c90 100644
--- a/src/core/lib/gpr++/orphanable.h
+++ b/src/core/lib/gprpp/orphanable.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_GPRXX_ORPHANABLE_H
-#define GRPC_CORE_LIB_GPRXX_ORPHANABLE_H
+#ifndef GRPC_CORE_LIB_GPRPP_ORPHANABLE_H
+#define GRPC_CORE_LIB_GPRPP_ORPHANABLE_H
 
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
@@ -25,9 +25,9 @@
 #include <memory>
 
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gpr++/abstract.h"
-#include "src/core/lib/gpr++/debug_location.h"
-#include "src/core/lib/gpr++/memory.h"
+#include "src/core/lib/gprpp/abstract.h"
+#include "src/core/lib/gprpp/debug_location.h"
+#include "src/core/lib/gprpp/memory.h"
 
 namespace grpc_core {
 
@@ -168,4 +168,4 @@ class InternallyRefCountedWithTracing : public Orphanable {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_GPRXX_ORPHANABLE_H */
+#endif /* GRPC_CORE_LIB_GPRPP_ORPHANABLE_H */
diff --git a/src/core/lib/gpr++/ref_counted.h b/src/core/lib/gprpp/ref_counted.h
similarity index 93%
rename from src/core/lib/gpr++/ref_counted.h
rename to src/core/lib/gprpp/ref_counted.h
index c2ae76c0aef..c68118a71a6 100644
--- a/src/core/lib/gpr++/ref_counted.h
+++ b/src/core/lib/gprpp/ref_counted.h
@@ -16,16 +16,16 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_GPRXX_REF_COUNTED_H
-#define GRPC_CORE_LIB_GPRXX_REF_COUNTED_H
+#ifndef GRPC_CORE_LIB_GPRPP_REF_COUNTED_H
+#define GRPC_CORE_LIB_GPRPP_REF_COUNTED_H
 
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gpr++/abstract.h"
-#include "src/core/lib/gpr++/debug_location.h"
-#include "src/core/lib/gpr++/memory.h"
+#include "src/core/lib/gprpp/abstract.h"
+#include "src/core/lib/gprpp/debug_location.h"
+#include "src/core/lib/gprpp/memory.h"
 
 namespace grpc_core {
 
@@ -130,4 +130,4 @@ class RefCountedWithTracing {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_GPRXX_REF_COUNTED_H */
+#endif /* GRPC_CORE_LIB_GPRPP_REF_COUNTED_H */
diff --git a/src/core/lib/gpr++/ref_counted_ptr.h b/src/core/lib/gprpp/ref_counted_ptr.h
similarity index 93%
rename from src/core/lib/gpr++/ref_counted_ptr.h
rename to src/core/lib/gprpp/ref_counted_ptr.h
index 862294d1aa0..dda0f00d79d 100644
--- a/src/core/lib/gpr++/ref_counted_ptr.h
+++ b/src/core/lib/gprpp/ref_counted_ptr.h
@@ -16,12 +16,12 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_GPRXX_REF_COUNTED_PTR_H
-#define GRPC_CORE_LIB_GPRXX_REF_COUNTED_PTR_H
+#ifndef GRPC_CORE_LIB_GPRPP_REF_COUNTED_PTR_H
+#define GRPC_CORE_LIB_GPRPP_REF_COUNTED_PTR_H
 
 #include <utility>
 
-#include "src/core/lib/gpr++/memory.h"
+#include "src/core/lib/gprpp/memory.h"
 
 namespace grpc_core {
 
@@ -96,4 +96,4 @@ inline RefCountedPtr<T> MakeRefCounted(Args&&... args) {
 
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_LIB_GPRXX_REF_COUNTED_PTR_H */
+#endif /* GRPC_CORE_LIB_GPRPP_REF_COUNTED_PTR_H */
diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc
index 1cb0150f450..42d7cdd348d 100644
--- a/src/core/lib/iomgr/ev_epoll1_linux.cc
+++ b/src/core/lib/iomgr/ev_epoll1_linux.cc
@@ -43,8 +43,8 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/debug/stats.h"
-#include "src/core/lib/gpr++/manual_constructor.h"
 #include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc
index b81c00ca7aa..416e8384b45 100644
--- a/src/core/lib/iomgr/ev_epollex_linux.cc
+++ b/src/core/lib/iomgr/ev_epollex_linux.cc
@@ -41,8 +41,8 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/debug/stats.h"
-#include "src/core/lib/gpr++/manual_constructor.h"
 #include "src/core/lib/gpr/spinlock.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/is_epollexclusive_available.h"
diff --git a/src/core/lib/iomgr/ev_epollsig_linux.cc b/src/core/lib/iomgr/ev_epollsig_linux.cc
index 11c64d080c7..15183489925 100644
--- a/src/core/lib/iomgr/ev_epollsig_linux.cc
+++ b/src/core/lib/iomgr/ev_epollsig_linux.cc
@@ -43,7 +43,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/debug/stats.h"
-#include "src/core/lib/gpr++/manual_constructor.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
diff --git a/src/core/lib/surface/lame_client.cc b/src/core/lib/surface/lame_client.cc
index 27a2a4eeb6f..a1f1cf11070 100644
--- a/src/core/lib/surface/lame_client.cc
+++ b/src/core/lib/surface/lame_client.cc
@@ -23,7 +23,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
-#include "src/core/lib/gpr++/atomic.h"
+#include "src/core/lib/gprpp/atomic.h"
 
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/gpr/string.h"
diff --git a/test/core/gpr++/BUILD b/test/core/gprpp/BUILD
similarity index 94%
rename from test/core/gpr++/BUILD
rename to test/core/gprpp/BUILD
index 93324a378b2..1c11e0bdb5b 100644
--- a/test/core/gpr++/BUILD
+++ b/test/core/gprpp/BUILD
@@ -16,7 +16,7 @@ load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_c
 
 licenses(["notice"])  # Apache v2
 
-grpc_package(name = "test/core/gpr++")
+grpc_package(name = "test/core/gprpp")
 
 grpc_cc_test(
     name = "manual_constructor_test",
@@ -24,6 +24,7 @@ grpc_cc_test(
     language = "C++",
     deps = [
         "//:gpr",
+        "//:gpr++_base",
         "//test/core/util:gpr_test_util",
     ],
 )
@@ -36,7 +37,7 @@ grpc_cc_test(
     ],
     language = "C++",
     deps = [
-        "//:grpc",
+        "//:gpr++_base",
         "//test/core/util:gpr_test_util",
     ],
 )
@@ -49,7 +50,7 @@ grpc_cc_test(
     ],
     language = "C++",
     deps = [
-        "//:grpc",
+        "//:inlined_vector",
         "//test/core/util:gpr_test_util",
     ],
 )
diff --git a/test/core/gpr++/inlined_vector_test.cc b/test/core/gprpp/inlined_vector_test.cc
similarity index 95%
rename from test/core/gpr++/inlined_vector_test.cc
rename to test/core/gprpp/inlined_vector_test.cc
index 09d5453a56a..0e712dafe42 100644
--- a/test/core/gpr++/inlined_vector_test.cc
+++ b/test/core/gprpp/inlined_vector_test.cc
@@ -16,9 +16,9 @@
  *
  */
 
-#include "src/core/lib/gpr++/inlined_vector.h"
+#include "src/core/lib/gprpp/inlined_vector.h"
 #include <gtest/gtest.h>
-#include "src/core/lib/gpr++/memory.h"
+#include "src/core/lib/gprpp/memory.h"
 #include "test/core/util/test_config.h"
 
 namespace grpc_core {
diff --git a/test/core/gpr++/manual_constructor_test.cc b/test/core/gprpp/manual_constructor_test.cc
similarity index 96%
rename from test/core/gpr++/manual_constructor_test.cc
rename to test/core/gprpp/manual_constructor_test.cc
index e049b793f61..f06c3cab061 100644
--- a/test/core/gpr++/manual_constructor_test.cc
+++ b/test/core/gprpp/manual_constructor_test.cc
@@ -18,7 +18,7 @@
 
 /* Test of gpr synchronization support. */
 
-#include "src/core/lib/gpr++/manual_constructor.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
@@ -26,7 +26,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <cstring>
-#include "src/core/lib/gpr++/abstract.h"
+#include "src/core/lib/gprpp/abstract.h"
 #include "test/core/util/test_config.h"
 
 class A {
diff --git a/test/core/gpr++/memory_test.cc b/test/core/gprpp/memory_test.cc
similarity index 97%
rename from test/core/gpr++/memory_test.cc
rename to test/core/gprpp/memory_test.cc
index 3553e119e3b..180c36fad7f 100644
--- a/test/core/gpr++/memory_test.cc
+++ b/test/core/gprpp/memory_test.cc
@@ -16,7 +16,7 @@
  *
  */
 
-#include "src/core/lib/gpr++/memory.h"
+#include "src/core/lib/gprpp/memory.h"
 #include <gtest/gtest.h>
 #include "test/core/util/test_config.h"
 
diff --git a/test/core/gpr++/orphanable_test.cc b/test/core/gprpp/orphanable_test.cc
similarity index 96%
rename from test/core/gpr++/orphanable_test.cc
rename to test/core/gprpp/orphanable_test.cc
index 4513d25da17..ff2f6d8bc20 100644
--- a/test/core/gpr++/orphanable_test.cc
+++ b/test/core/gprpp/orphanable_test.cc
@@ -16,11 +16,11 @@
  *
  */
 
-#include "src/core/lib/gpr++/orphanable.h"
+#include "src/core/lib/gprpp/orphanable.h"
 
 #include <gtest/gtest.h>
 
-#include "src/core/lib/gpr++/memory.h"
+#include "src/core/lib/gprpp/memory.h"
 #include "test/core/util/test_config.h"
 
 namespace grpc_core {
diff --git a/test/core/gpr++/ref_counted_ptr_test.cc b/test/core/gprpp/ref_counted_ptr_test.cc
similarity index 97%
rename from test/core/gpr++/ref_counted_ptr_test.cc
rename to test/core/gprpp/ref_counted_ptr_test.cc
index e897b2859c8..f1f13f3183c 100644
--- a/test/core/gpr++/ref_counted_ptr_test.cc
+++ b/test/core/gprpp/ref_counted_ptr_test.cc
@@ -16,14 +16,14 @@
  *
  */
 
-#include "src/core/lib/gpr++/ref_counted_ptr.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 
 #include <gtest/gtest.h>
 
 #include <grpc/support/log.h>
 
-#include "src/core/lib/gpr++/memory.h"
-#include "src/core/lib/gpr++/ref_counted.h"
+#include "src/core/lib/gprpp/memory.h"
+#include "src/core/lib/gprpp/ref_counted.h"
 #include "test/core/util/test_config.h"
 
 namespace grpc_core {
diff --git a/test/core/gpr++/ref_counted_test.cc b/test/core/gprpp/ref_counted_test.cc
similarity index 95%
rename from test/core/gpr++/ref_counted_test.cc
rename to test/core/gprpp/ref_counted_test.cc
index 568ec51c64a..b1b0fee5c0f 100644
--- a/test/core/gpr++/ref_counted_test.cc
+++ b/test/core/gprpp/ref_counted_test.cc
@@ -16,11 +16,11 @@
  *
  */
 
-#include "src/core/lib/gpr++/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted.h"
 
 #include <gtest/gtest.h>
 
-#include "src/core/lib/gpr++/memory.h"
+#include "src/core/lib/gprpp/memory.h"
 #include "test/core/util/test_config.h"
 
 namespace grpc_core {
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index 55b10268ba1..a6c5fb2c8e7 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -950,17 +950,6 @@ src/core/lib/compression/stream_compression_identity.h \
 src/core/lib/debug/stats.h \
 src/core/lib/debug/stats_data.h \
 src/core/lib/debug/trace.h \
-src/core/lib/gpr++/abstract.h \
-src/core/lib/gpr++/atomic.h \
-src/core/lib/gpr++/atomic_with_atm.h \
-src/core/lib/gpr++/atomic_with_std.h \
-src/core/lib/gpr++/debug_location.h \
-src/core/lib/gpr++/inlined_vector.h \
-src/core/lib/gpr++/manual_constructor.h \
-src/core/lib/gpr++/memory.h \
-src/core/lib/gpr++/orphanable.h \
-src/core/lib/gpr++/ref_counted.h \
-src/core/lib/gpr++/ref_counted_ptr.h \
 src/core/lib/gpr/arena.h \
 src/core/lib/gpr/env.h \
 src/core/lib/gpr/fork.h \
@@ -972,6 +961,17 @@ src/core/lib/gpr/string_windows.h \
 src/core/lib/gpr/thd_internal.h \
 src/core/lib/gpr/time_precise.h \
 src/core/lib/gpr/tmpfile.h \
+src/core/lib/gprpp/abstract.h \
+src/core/lib/gprpp/atomic.h \
+src/core/lib/gprpp/atomic_with_atm.h \
+src/core/lib/gprpp/atomic_with_std.h \
+src/core/lib/gprpp/debug_location.h \
+src/core/lib/gprpp/inlined_vector.h \
+src/core/lib/gprpp/manual_constructor.h \
+src/core/lib/gprpp/memory.h \
+src/core/lib/gprpp/orphanable.h \
+src/core/lib/gprpp/ref_counted.h \
+src/core/lib/gprpp/ref_counted_ptr.h \
 src/core/lib/http/format_request.h \
 src/core/lib/http/httpcli.h \
 src/core/lib/http/parser.h \
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 7857ec72fd4..f3e624ba724 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -1064,18 +1064,6 @@ src/core/lib/debug/stats_data.cc \
 src/core/lib/debug/stats_data.h \
 src/core/lib/debug/trace.cc \
 src/core/lib/debug/trace.h \
-src/core/lib/gpr++/README.md \
-src/core/lib/gpr++/abstract.h \
-src/core/lib/gpr++/atomic.h \
-src/core/lib/gpr++/atomic_with_atm.h \
-src/core/lib/gpr++/atomic_with_std.h \
-src/core/lib/gpr++/debug_location.h \
-src/core/lib/gpr++/inlined_vector.h \
-src/core/lib/gpr++/manual_constructor.h \
-src/core/lib/gpr++/memory.h \
-src/core/lib/gpr++/orphanable.h \
-src/core/lib/gpr++/ref_counted.h \
-src/core/lib/gpr++/ref_counted_ptr.h \
 src/core/lib/gpr/README.md \
 src/core/lib/gpr/alloc.cc \
 src/core/lib/gpr/arena.cc \
@@ -1130,6 +1118,18 @@ src/core/lib/gpr/tmpfile_msys.cc \
 src/core/lib/gpr/tmpfile_posix.cc \
 src/core/lib/gpr/tmpfile_windows.cc \
 src/core/lib/gpr/wrap_memcpy.cc \
+src/core/lib/gprpp/README.md \
+src/core/lib/gprpp/abstract.h \
+src/core/lib/gprpp/atomic.h \
+src/core/lib/gprpp/atomic_with_atm.h \
+src/core/lib/gprpp/atomic_with_std.h \
+src/core/lib/gprpp/debug_location.h \
+src/core/lib/gprpp/inlined_vector.h \
+src/core/lib/gprpp/manual_constructor.h \
+src/core/lib/gprpp/memory.h \
+src/core/lib/gprpp/orphanable.h \
+src/core/lib/gprpp/ref_counted.h \
+src/core/lib/gprpp/ref_counted_ptr.h \
 src/core/lib/http/format_request.cc \
 src/core/lib/http/format_request.h \
 src/core/lib/http/httpcli.cc \
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 982d9ef6496..362effd7d2e 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -687,7 +687,7 @@
     "language": "c", 
     "name": "gpr_manual_constructor_test", 
     "src": [
-      "test/core/gpr++/manual_constructor_test.cc"
+      "test/core/gprpp/manual_constructor_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -3542,7 +3542,7 @@
     "language": "c++", 
     "name": "inlined_vector_test", 
     "src": [
-      "test/core/gpr++/inlined_vector_test.cc"
+      "test/core/gprpp/inlined_vector_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -3662,7 +3662,7 @@
     "language": "c++", 
     "name": "memory_test", 
     "src": [
-      "test/core/gpr++/memory_test.cc"
+      "test/core/gprpp/memory_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -3740,7 +3740,7 @@
     "language": "c++", 
     "name": "orphanable_test", 
     "src": [
-      "test/core/gpr++/orphanable_test.cc"
+      "test/core/gprpp/orphanable_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -3951,7 +3951,7 @@
     "language": "c++", 
     "name": "ref_counted_ptr_test", 
     "src": [
-      "test/core/gpr++/ref_counted_ptr_test.cc"
+      "test/core/gprpp/ref_counted_ptr_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -3970,7 +3970,7 @@
     "language": "c++", 
     "name": "ref_counted_test", 
     "src": [
-      "test/core/gpr++/ref_counted_test.cc"
+      "test/core/gprpp/ref_counted_test.cc"
     ], 
     "third_party": false, 
     "type": "target"
@@ -7906,12 +7906,6 @@
       "include/grpc/support/tls_msvc.h", 
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
-      "src/core/lib/gpr++/abstract.h", 
-      "src/core/lib/gpr++/atomic.h", 
-      "src/core/lib/gpr++/atomic_with_atm.h", 
-      "src/core/lib/gpr++/atomic_with_std.h", 
-      "src/core/lib/gpr++/manual_constructor.h", 
-      "src/core/lib/gpr++/memory.h", 
       "src/core/lib/gpr/arena.h", 
       "src/core/lib/gpr/env.h", 
       "src/core/lib/gpr/fork.h", 
@@ -7923,6 +7917,12 @@
       "src/core/lib/gpr/thd_internal.h", 
       "src/core/lib/gpr/time_precise.h", 
       "src/core/lib/gpr/tmpfile.h", 
+      "src/core/lib/gprpp/abstract.h", 
+      "src/core/lib/gprpp/atomic.h", 
+      "src/core/lib/gprpp/atomic_with_atm.h", 
+      "src/core/lib/gprpp/atomic_with_std.h", 
+      "src/core/lib/gprpp/manual_constructor.h", 
+      "src/core/lib/gprpp/memory.h", 
       "src/core/lib/profiling/timers.h"
     ], 
     "is_filegroup": true, 
@@ -7955,12 +7955,6 @@
       "include/grpc/support/tls_msvc.h", 
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
-      "src/core/lib/gpr++/abstract.h", 
-      "src/core/lib/gpr++/atomic.h", 
-      "src/core/lib/gpr++/atomic_with_atm.h", 
-      "src/core/lib/gpr++/atomic_with_std.h", 
-      "src/core/lib/gpr++/manual_constructor.h", 
-      "src/core/lib/gpr++/memory.h", 
       "src/core/lib/gpr/arena.h", 
       "src/core/lib/gpr/env.h", 
       "src/core/lib/gpr/fork.h", 
@@ -7972,6 +7966,12 @@
       "src/core/lib/gpr/thd_internal.h", 
       "src/core/lib/gpr/time_precise.h", 
       "src/core/lib/gpr/tmpfile.h", 
+      "src/core/lib/gprpp/abstract.h", 
+      "src/core/lib/gprpp/atomic.h", 
+      "src/core/lib/gprpp/atomic_with_atm.h", 
+      "src/core/lib/gprpp/atomic_with_std.h", 
+      "src/core/lib/gprpp/manual_constructor.h", 
+      "src/core/lib/gprpp/memory.h", 
       "src/core/lib/profiling/timers.h"
     ], 
     "third_party": false, 
@@ -8229,11 +8229,11 @@
       "src/core/lib/compression/stream_compression_identity.h", 
       "src/core/lib/debug/stats.h", 
       "src/core/lib/debug/stats_data.h", 
-      "src/core/lib/gpr++/debug_location.h", 
-      "src/core/lib/gpr++/inlined_vector.h", 
-      "src/core/lib/gpr++/orphanable.h", 
-      "src/core/lib/gpr++/ref_counted.h", 
-      "src/core/lib/gpr++/ref_counted_ptr.h", 
+      "src/core/lib/gprpp/debug_location.h", 
+      "src/core/lib/gprpp/inlined_vector.h", 
+      "src/core/lib/gprpp/orphanable.h", 
+      "src/core/lib/gprpp/ref_counted.h", 
+      "src/core/lib/gprpp/ref_counted_ptr.h", 
       "src/core/lib/http/format_request.h", 
       "src/core/lib/http/httpcli.h", 
       "src/core/lib/http/parser.h", 
@@ -8370,11 +8370,11 @@
       "src/core/lib/compression/stream_compression_identity.h", 
       "src/core/lib/debug/stats.h", 
       "src/core/lib/debug/stats_data.h", 
-      "src/core/lib/gpr++/debug_location.h", 
-      "src/core/lib/gpr++/inlined_vector.h", 
-      "src/core/lib/gpr++/orphanable.h", 
-      "src/core/lib/gpr++/ref_counted.h", 
-      "src/core/lib/gpr++/ref_counted_ptr.h", 
+      "src/core/lib/gprpp/debug_location.h", 
+      "src/core/lib/gprpp/inlined_vector.h", 
+      "src/core/lib/gprpp/orphanable.h", 
+      "src/core/lib/gprpp/ref_counted.h", 
+      "src/core/lib/gprpp/ref_counted_ptr.h", 
       "src/core/lib/http/format_request.h", 
       "src/core/lib/http/httpcli.h", 
       "src/core/lib/http/parser.h", 

From 63efbc639c084bedfaff13267dddbda1546498a5 Mon Sep 17 00:00:00 2001
From: Menghan Li <menghanl@google.com>
Date: Fri, 19 Jan 2018 13:19:39 -0800
Subject: [PATCH 159/161] Update go release versions in client_matrix.py

---
 tools/interop_matrix/client_matrix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py
index dbbf8a5a090..5de705a52a6 100644
--- a/tools/interop_matrix/client_matrix.py
+++ b/tools/interop_matrix/client_matrix.py
@@ -107,7 +107,7 @@ LANG_RELEASE_MATRIX = {
             'v1.8.2': None
         },
         {
-            'v1.9.1': None
+            'v1.9.2': None
         },
     ],
     'java': [

From d1915a988ed05559ab3cfca57d7207fa5df5a1f4 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Fri, 19 Jan 2018 14:23:17 -0800
Subject: [PATCH 160/161] Bump version to 1.10

---
 BUILD               | 4 ++--
 build.yaml          | 4 ++--
 doc/g_stands_for.md | 1 +
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/BUILD b/BUILD
index c52a95cf91b..f2c03745ec6 100644
--- a/BUILD
+++ b/BUILD
@@ -54,11 +54,11 @@ config_setting(
 )
 
 # This should be updated along with build.yaml
-g_stands_for = "glossy"
+g_stands_for = "glamorous"
 
 core_version = "5.0.0-dev"
 
-version = "1.9.0-dev"
+version = "1.10.0-dev"
 
 GPR_PUBLIC_HDRS = [
     "include/grpc/support/alloc.h",
diff --git a/build.yaml b/build.yaml
index a8639a8b0b8..7508cdf9e88 100644
--- a/build.yaml
+++ b/build.yaml
@@ -13,8 +13,8 @@ settings:
   '#09': Per-language overrides are possible with (eg) ruby_version tag here
   '#10': See the expand_version.py for all the quirks here
   core_version: 5.0.0-dev
-  g_stands_for: glossy
-  version: 1.9.0-dev
+  g_stands_for: glamorous
+  version: 1.10.0-dev
 filegroups:
 - name: census
   public_headers:
diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md
index edc6dc1e798..19875c36a0b 100644
--- a/doc/g_stands_for.md
+++ b/doc/g_stands_for.md
@@ -13,3 +13,4 @@ future), and the corresponding version numbers that used them:
 - 1.7 'g' stands for 'gambit'
 - 1.8 'g' stands for 'generous'
 - 1.9 'g' stands for 'glossy'
+- 1.10 'g' stands for 'glamorous'

From 43884047c9619d64f6cd1775febf4af1823787b5 Mon Sep 17 00:00:00 2001
From: Mehrdad Afshari <mmx@google.com>
Date: Fri, 19 Jan 2018 14:25:15 -0800
Subject: [PATCH 161/161] Regenerate projects

---
 CMakeLists.txt                                    | 2 +-
 Makefile                                          | 4 ++--
 gRPC-Core.podspec                                 | 2 +-
 gRPC-ProtoRPC.podspec                             | 2 +-
 gRPC-RxLibrary.podspec                            | 2 +-
 gRPC.podspec                                      | 2 +-
 package.xml                                       | 4 ++--
 src/core/lib/surface/version.cc                   | 2 +-
 src/cpp/common/version_cc.cc                      | 2 +-
 src/csharp/Grpc.Core/Version.csproj.include       | 2 +-
 src/csharp/Grpc.Core/VersionInfo.cs               | 4 ++--
 src/csharp/build_packages_dotnetcli.bat           | 2 +-
 src/csharp/build_packages_dotnetcli.sh            | 4 ++--
 src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +-
 src/objective-c/GRPCClient/private/version.h      | 2 +-
 src/objective-c/tests/version.h                   | 2 +-
 src/php/composer.json                             | 2 +-
 src/php/ext/grpc/version.h                        | 2 +-
 src/python/grpcio/grpc/_grpcio_metadata.py        | 2 +-
 src/python/grpcio/grpc_version.py                 | 2 +-
 src/python/grpcio_health_checking/grpc_version.py | 2 +-
 src/python/grpcio_reflection/grpc_version.py      | 2 +-
 src/python/grpcio_testing/grpc_version.py         | 2 +-
 src/python/grpcio_tests/grpc_version.py           | 2 +-
 src/ruby/lib/grpc/version.rb                      | 2 +-
 src/ruby/tools/version.rb                         | 2 +-
 tools/distrib/python/grpcio_tools/grpc_version.py | 2 +-
 tools/doxygen/Doxyfile.c++                        | 2 +-
 tools/doxygen/Doxyfile.c++.internal               | 2 +-
 29 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4450a73e9d5..6c54121ea2c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -24,7 +24,7 @@
 cmake_minimum_required(VERSION 2.8)
 
 set(PACKAGE_NAME      "grpc")
-set(PACKAGE_VERSION   "1.9.0-dev")
+set(PACKAGE_VERSION   "1.10.0-dev")
 set(PACKAGE_STRING    "${PACKAGE_NAME} ${PACKAGE_VERSION}")
 set(PACKAGE_TARNAME   "${PACKAGE_NAME}-${PACKAGE_VERSION}")
 set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/")
diff --git a/Makefile b/Makefile
index 69a68a6d66a..a8681940f16 100644
--- a/Makefile
+++ b/Makefile
@@ -419,8 +419,8 @@ Q = @
 endif
 
 CORE_VERSION = 5.0.0-dev
-CPP_VERSION = 1.9.0-dev
-CSHARP_VERSION = 1.9.0-dev
+CPP_VERSION = 1.10.0-dev
+CSHARP_VERSION = 1.10.0-dev
 
 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES))
 CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS)
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index ccaa21194b8..48270189538 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -22,7 +22,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-Core'
-  version = '1.9.0-dev'
+  version = '1.10.0-dev'
   s.version  = version
   s.summary  = 'Core cross-platform gRPC library, written in C'
   s.homepage = 'https://grpc.io'
diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec
index cb1c5480dab..a0375bd0b83 100644
--- a/gRPC-ProtoRPC.podspec
+++ b/gRPC-ProtoRPC.podspec
@@ -21,7 +21,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-ProtoRPC'
-  version = '1.9.0-dev'
+  version = '1.10.0-dev'
   s.version  = version
   s.summary  = 'RPC library for Protocol Buffers, based on gRPC'
   s.homepage = 'https://grpc.io'
diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec
index 0f9abb6f4b3..280aafefb6e 100644
--- a/gRPC-RxLibrary.podspec
+++ b/gRPC-RxLibrary.podspec
@@ -21,7 +21,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-RxLibrary'
-  version = '1.9.0-dev'
+  version = '1.10.0-dev'
   s.version  = version
   s.summary  = 'Reactive Extensions library for iOS/OSX.'
   s.homepage = 'https://grpc.io'
diff --git a/gRPC.podspec b/gRPC.podspec
index 1f3a0a9950c..930d991a6c4 100644
--- a/gRPC.podspec
+++ b/gRPC.podspec
@@ -20,7 +20,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC'
-  version = '1.9.0-dev'
+  version = '1.10.0-dev'
   s.version  = version
   s.summary  = 'gRPC client library for iOS/OSX'
   s.homepage = 'https://grpc.io'
diff --git a/package.xml b/package.xml
index f1282964c2b..5d81fd8b16f 100644
--- a/package.xml
+++ b/package.xml
@@ -13,8 +13,8 @@
  <date>2017-08-24</date>
  <time>16:06:07</time>
  <version>
-  <release>1.9.0dev</release>
-  <api>1.9.0dev</api>
+  <release>1.10.0dev</release>
+  <api>1.10.0dev</api>
  </version>
  <stability>
   <release>beta</release>
diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc
index 7d36c6c9e1d..19498a6df7d 100644
--- a/src/core/lib/surface/version.cc
+++ b/src/core/lib/surface/version.cc
@@ -23,4 +23,4 @@
 
 const char* grpc_version_string(void) { return "5.0.0-dev"; }
 
-const char* grpc_g_stands_for(void) { return "glossy"; }
+const char* grpc_g_stands_for(void) { return "glamorous"; }
diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc
index 7f01a66dcf0..8bc926048fd 100644
--- a/src/cpp/common/version_cc.cc
+++ b/src/cpp/common/version_cc.cc
@@ -22,5 +22,5 @@
 #include <grpc++/grpc++.h>
 
 namespace grpc {
-grpc::string Version() { return "1.9.0-dev"; }
+grpc::string Version() { return "1.10.0-dev"; }
 }  // namespace grpc
diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include
index 2d9e4ba16ae..539d3a9f801 100755
--- a/src/csharp/Grpc.Core/Version.csproj.include
+++ b/src/csharp/Grpc.Core/Version.csproj.include
@@ -1,7 +1,7 @@
 <!-- This file is generated -->
 <Project>
   <PropertyGroup>
-    <GrpcCsharpVersion>1.9.0-dev</GrpcCsharpVersion>
+    <GrpcCsharpVersion>1.10.0-dev</GrpcCsharpVersion>
     <GoogleProtobufVersion>3.3.0</GoogleProtobufVersion>
   </PropertyGroup>
 </Project>
diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs
index 9b5da1c9475..f1aef46c6cb 100644
--- a/src/csharp/Grpc.Core/VersionInfo.cs
+++ b/src/csharp/Grpc.Core/VersionInfo.cs
@@ -33,11 +33,11 @@ namespace Grpc.Core
         /// <summary>
         /// Current <c>AssemblyFileVersion</c> of gRPC C# assemblies
         /// </summary>
-        public const string CurrentAssemblyFileVersion = "1.9.0.0";
+        public const string CurrentAssemblyFileVersion = "1.10.0.0";
 
         /// <summary>
         /// Current version of gRPC C#
         /// </summary>
-        public const string CurrentVersion = "1.9.0-dev";
+        public const string CurrentVersion = "1.10.0-dev";
     }
 }
diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat
index 8f89e2846a5..4087d8b67a6 100755
--- a/src/csharp/build_packages_dotnetcli.bat
+++ b/src/csharp/build_packages_dotnetcli.bat
@@ -13,7 +13,7 @@
 @rem limitations under the License.
 
 @rem Current package versions
-set VERSION=1.9.0-dev
+set VERSION=1.10.0-dev
 
 @rem Adjust the location of nuget.exe
 set NUGET=C:\nuget\nuget.exe
diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh
index 6a6cafe2bdc..8ccc537a609 100755
--- a/src/csharp/build_packages_dotnetcli.sh
+++ b/src/csharp/build_packages_dotnetcli.sh
@@ -39,7 +39,7 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts
 dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts
 dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts
 
-nuget pack Grpc.nuspec -Version "1.9.0-dev" -OutputDirectory ../../artifacts
-nuget pack Grpc.Tools.nuspec -Version "1.9.0-dev" -OutputDirectory ../../artifacts
+nuget pack Grpc.nuspec -Version "1.10.0-dev" -OutputDirectory ../../artifacts
+nuget pack Grpc.Tools.nuspec -Version "1.10.0-dev" -OutputDirectory ../../artifacts
 
 (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg)
diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec
index 22501765f9d..037ad4d9b0e 100644
--- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec
+++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec
@@ -42,7 +42,7 @@ Pod::Spec.new do |s|
   # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed
   # before them.
   s.name     = '!ProtoCompiler-gRPCPlugin'
-  v = '1.9.0-dev'
+  v = '1.10.0-dev'
   s.version  = v
   s.summary  = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.'
   s.description = <<-DESC
diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h
index 69dd6266fd6..5c134e36420 100644
--- a/src/objective-c/GRPCClient/private/version.h
+++ b/src/objective-c/GRPCClient/private/version.h
@@ -23,4 +23,4 @@
 // `tools/buildgen/generate_projects.sh`.
 
 
-#define GRPC_OBJC_VERSION_STRING @"1.9.0-dev"
+#define GRPC_OBJC_VERSION_STRING @"1.10.0-dev"
diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h
index 6e3a073020f..d8581b97796 100644
--- a/src/objective-c/tests/version.h
+++ b/src/objective-c/tests/version.h
@@ -23,5 +23,5 @@
 // `tools/buildgen/generate_projects.sh`.
 
 
-#define GRPC_OBJC_VERSION_STRING @"1.9.0-dev"
+#define GRPC_OBJC_VERSION_STRING @"1.10.0-dev"
 #define GRPC_C_VERSION_STRING @"5.0.0-dev"
diff --git a/src/php/composer.json b/src/php/composer.json
index 43833980f9d..ea214179562 100644
--- a/src/php/composer.json
+++ b/src/php/composer.json
@@ -2,7 +2,7 @@
   "name": "grpc/grpc-dev",
   "description": "gRPC library for PHP - for Developement use only",
   "license": "Apache-2.0",
-  "version": "1.9.0",
+  "version": "1.10.0",
   "require": {
     "php": ">=5.5.0",
     "google/protobuf": "^v3.3.0"
diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h
index 48131d72d1a..408f2a47656 100644
--- a/src/php/ext/grpc/version.h
+++ b/src/php/ext/grpc/version.h
@@ -20,6 +20,6 @@
 #ifndef VERSION_H
 #define VERSION_H
 
-#define PHP_GRPC_VERSION "1.9.0dev"
+#define PHP_GRPC_VERSION "1.10.0dev"
 
 #endif /* VERSION_H */
diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py
index 993c49d4af2..6032828c776 100644
--- a/src/python/grpcio/grpc/_grpcio_metadata.py
+++ b/src/python/grpcio/grpc/_grpcio_metadata.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!!
 
-__version__ = """1.9.0.dev0"""
+__version__ = """1.10.0.dev0"""
diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py
index 1fac57b03a4..a654eb026a7 100644
--- a/src/python/grpcio/grpc_version.py
+++ b/src/python/grpcio/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!!
 
-VERSION = '1.9.0.dev0'
+VERSION = '1.10.0.dev0'
diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py
index 5b7e5859bc5..d3185c69725 100644
--- a/src/python/grpcio_health_checking/grpc_version.py
+++ b/src/python/grpcio_health_checking/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!!
 
-VERSION = '1.9.0.dev0'
+VERSION = '1.10.0.dev0'
diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py
index 0ad96211541..7203d0d3213 100644
--- a/src/python/grpcio_reflection/grpc_version.py
+++ b/src/python/grpcio_reflection/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!!
 
-VERSION = '1.9.0.dev0'
+VERSION = '1.10.0.dev0'
diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py
index 0eb5fbf94d2..bf9e55e10e2 100644
--- a/src/python/grpcio_testing/grpc_version.py
+++ b/src/python/grpcio_testing/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!!
 
-VERSION = '1.9.0.dev0'
+VERSION = '1.10.0.dev0'
diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py
index b1b4d7e0c21..2583e420160 100644
--- a/src/python/grpcio_tests/grpc_version.py
+++ b/src/python/grpcio_tests/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!!
 
-VERSION = '1.9.0.dev0'
+VERSION = '1.10.0.dev0'
diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb
index be1412511ad..9d9f2f49686 100644
--- a/src/ruby/lib/grpc/version.rb
+++ b/src/ruby/lib/grpc/version.rb
@@ -14,5 +14,5 @@
 
 # GRPC contains the General RPC module.
 module GRPC
-  VERSION = '1.9.0.dev'
+  VERSION = '1.10.0.dev'
 end
diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb
index 48aad39e082..2682294bd2c 100644
--- a/src/ruby/tools/version.rb
+++ b/src/ruby/tools/version.rb
@@ -14,6 +14,6 @@
 
 module GRPC
   module Tools
-    VERSION = '1.9.0.dev'
+    VERSION = '1.10.0.dev'
   end
 end
diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py
index c4ed0661223..5caa1d10787 100644
--- a/tools/distrib/python/grpcio_tools/grpc_version.py
+++ b/tools/distrib/python/grpcio_tools/grpc_version.py
@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!!
 
-VERSION = '1.9.0.dev0'
+VERSION = '1.10.0.dev0'
diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++
index 5bdbcd71f57..f84b4fe2d3f 100644
--- a/tools/doxygen/Doxyfile.c++
+++ b/tools/doxygen/Doxyfile.c++
@@ -40,7 +40,7 @@ PROJECT_NAME           = "GRPC C++"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.9.0-dev
+PROJECT_NUMBER         = 1.10.0-dev
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index 55b10268ba1..57ec41f3a5e 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -40,7 +40,7 @@ PROJECT_NAME           = "GRPC C++"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.9.0-dev
+PROJECT_NUMBER         = 1.10.0-dev
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a