From 493d996e977dab60edfe980ed7f33927023ca544 Mon Sep 17 00:00:00 2001
From: Na-Na Pang <ladynana@google.com>
Date: Mon, 23 Sep 2019 17:15:32 -0700
Subject: [PATCH 01/26] Fix bug in system time and user time calculation

---
 test/cpp/qps/driver.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc
index cfc3c8e9a06..cdcc3c71456 100644
--- a/test/cpp/qps/driver.cc
+++ b/test/cpp/qps/driver.cc
@@ -137,14 +137,14 @@ static void postprocess_scenario_result(ScenarioResult* result) {
 
   auto server_system_time = 100.0 *
                             sum(result->server_stats(), ServerSystemTime) /
-                            sum(result->server_stats(), ServerWallTime);
+                            average(result->server_stats(), ServerWallTime);
   auto server_user_time = 100.0 * sum(result->server_stats(), ServerUserTime) /
-                          sum(result->server_stats(), ServerWallTime);
+                          average(result->server_stats(), ServerWallTime);
 
   auto client_system_time = 100.0 * sum(result->client_stats(), SystemTime) /
-                            sum(result->client_stats(), WallTime);
+                            average(result->client_stats(), WallTime);
   auto client_user_time = 100.0 * sum(result->client_stats(), UserTime) /
-                          sum(result->client_stats(), WallTime);
+                          average(result->client_stats(), WallTime);
 
   result->mutable_summary()->set_server_system_time(server_system_time);
   result->mutable_summary()->set_server_user_time(server_user_time);

From ed45c8f8457be40b49ac3770ee2430a420d33d70 Mon Sep 17 00:00:00 2001
From: Yash Tibrewal <yashkt@google.com>
Date: Thu, 10 Oct 2019 15:03:46 -0700
Subject: [PATCH 02/26] Stronger guarantee on failure of read/write operations

---
 include/grpcpp/impl/codegen/client_callback_impl.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/grpcpp/impl/codegen/client_callback_impl.h b/include/grpcpp/impl/codegen/client_callback_impl.h
index 34c738ac1e6..bf3ef2f8272 100644
--- a/include/grpcpp/impl/codegen/client_callback_impl.h
+++ b/include/grpcpp/impl/codegen/client_callback_impl.h
@@ -285,18 +285,18 @@ class ClientBidiReactor {
   /// call of OnReadDone or OnDone.
   ///
   /// \param[in] ok Was the initial metadata read successfully? If false, no
-  ///               further read-side operation will succeed.
+  ///               new read/write operation will succeed.
   virtual void OnReadInitialMetadataDone(bool /*ok*/) {}
 
   /// Notifies the application that a StartRead operation completed.
   ///
-  /// \param[in] ok Was it successful? If false, no further read-side operation
+  /// \param[in] ok Was it successful? If false, no new read/write operation
   ///               will succeed.
   virtual void OnReadDone(bool /*ok*/) {}
 
   /// Notifies the application that a StartWrite operation completed.
   ///
-  /// \param[in] ok Was it successful? If false, no further write-side operation
+  /// \param[in] ok Was it successful? If false, no new read/write operation
   ///               will succeed.
   virtual void OnWriteDone(bool /*ok*/) {}
 

From 203ec89c3e9d53a6b1c2173de5193b0979d6cba7 Mon Sep 17 00:00:00 2001
From: Na-Na Pang <ladynana@google.com>
Date: Fri, 11 Oct 2019 17:29:20 -0700
Subject: [PATCH 03/26] Revert "Fix bug in system time and user time
 calculation"

This reverts commit 493d996e977dab60edfe980ed7f33927023ca544.
---
 test/cpp/qps/driver.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc
index cdcc3c71456..cfc3c8e9a06 100644
--- a/test/cpp/qps/driver.cc
+++ b/test/cpp/qps/driver.cc
@@ -137,14 +137,14 @@ static void postprocess_scenario_result(ScenarioResult* result) {
 
   auto server_system_time = 100.0 *
                             sum(result->server_stats(), ServerSystemTime) /
-                            average(result->server_stats(), ServerWallTime);
+                            sum(result->server_stats(), ServerWallTime);
   auto server_user_time = 100.0 * sum(result->server_stats(), ServerUserTime) /
-                          average(result->server_stats(), ServerWallTime);
+                          sum(result->server_stats(), ServerWallTime);
 
   auto client_system_time = 100.0 * sum(result->client_stats(), SystemTime) /
-                            average(result->client_stats(), WallTime);
+                            sum(result->client_stats(), WallTime);
   auto client_user_time = 100.0 * sum(result->client_stats(), UserTime) /
-                          average(result->client_stats(), WallTime);
+                          sum(result->client_stats(), WallTime);
 
   result->mutable_summary()->set_server_system_time(server_system_time);
   result->mutable_summary()->set_server_user_time(server_user_time);

From fa02a5b0d95758d4d8ee229b98577d2ed5f920d9 Mon Sep 17 00:00:00 2001
From: Na-Na Pang <ladynana@google.com>
Date: Mon, 14 Oct 2019 12:38:45 -0700
Subject: [PATCH 04/26] Modify the calculation methods in driver and add
 comments in post_pocessing function

---
 test/cpp/qps/driver.cc | 52 ++++++++++++++++++++++++------------------
 1 file changed, 30 insertions(+), 22 deletions(-)

diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc
index c37529ea274..bd513e78a5d 100644
--- a/test/cpp/qps/driver.cc
+++ b/test/cpp/qps/driver.cc
@@ -134,36 +134,38 @@ static bool IsSuccess(const Status& s) {
 
 // Postprocess ScenarioResult and populate result summary.
 static void postprocess_scenario_result(ScenarioResult* result) {
+  // Get latencies from ScenarioResult latencies histogram and populate to
+  // result summary.
   Histogram histogram;
   histogram.MergeProto(result->latencies());
-
-  auto time_estimate = average(result->client_stats(), WallTime);
-  auto qps = histogram.Count() / time_estimate;
-  auto qps_per_server_core = qps / sum(result->server_cores(), Cores);
-
-  result->mutable_summary()->set_qps(qps);
-  result->mutable_summary()->set_qps_per_server_core(qps_per_server_core);
   result->mutable_summary()->set_latency_50(histogram.Percentile(50));
   result->mutable_summary()->set_latency_90(histogram.Percentile(90));
   result->mutable_summary()->set_latency_95(histogram.Percentile(95));
   result->mutable_summary()->set_latency_99(histogram.Percentile(99));
   result->mutable_summary()->set_latency_999(histogram.Percentile(99.9));
 
-  auto server_system_time = 100.0 *
-                            sum(result->server_stats(), ServerSystemTime) /
-                            sum(result->server_stats(), ServerWallTime);
-  auto server_user_time = 100.0 * sum(result->server_stats(), ServerUserTime) /
-                          sum(result->server_stats(), ServerWallTime);
-
-  auto client_system_time = 100.0 * sum(result->client_stats(), SystemTime) /
-                            sum(result->client_stats(), WallTime);
-  auto client_user_time = 100.0 * sum(result->client_stats(), UserTime) /
-                          sum(result->client_stats(), WallTime);
-
-  result->mutable_summary()->set_server_system_time(server_system_time);
-  result->mutable_summary()->set_server_user_time(server_user_time);
-  result->mutable_summary()->set_client_system_time(client_system_time);
-  result->mutable_summary()->set_client_user_time(client_user_time);
+  // Calculate qps and cpu load for each client and then aggregate results for all clients 
+  double qps = 0;
+  double client_system_cpu_load = 0, client_user_cpu_load = 0;
+  double server_system_cpu_load = 0, server_user_cpu_load = 0;
+  for (size_t i = 0; i < result->client_stats_size(); i++){
+    auto client_stat = result->client_stats(i);
+    qps += client_stat.latencies().count() / client_stat.time_system();
+    client_system_cpu_load += client_stat.time_system() / client_stat.time_elapsed();
+    client_user_cpu_load += client_stat.time_user() / client_stat.time_elapsed();
+  }
+  // Calculate cpu load for each server and then aggregate results for all servers
+  for (size_t i = 0; i < result->server_stats_size(); i++){
+    auto server_stat = result->server_stats(i);
+    server_system_cpu_load += server_stat.time_system() / server_stat.time_elapsed();
+    server_user_cpu_load += server_stat.time_user() / server_stat.time_elapsed();
+  }
+  result->mutable_summary()->set_qps(qps);
+  // Populate the percentage of cpu load to result summary.
+  result->mutable_summary()->set_server_system_time(100 * server_system_cpu_load);
+  result->mutable_summary()->set_server_user_time(100 * server_user_cpu_load);
+  result->mutable_summary()->set_client_system_time(100* client_system_cpu_load);
+  result->mutable_summary()->set_client_user_time(100 * client_user_cpu_load);
 
   // For Non-linux platform, get_cpu_usage() is not implemented. Thus,
   // ServerTotalCpuTime and ServerIdleCpuTime are both 0.
@@ -176,6 +178,9 @@ static void postprocess_scenario_result(ScenarioResult* result) {
     result->mutable_summary()->set_server_cpu_usage(server_cpu_usage);
   }
 
+  // Calculate and populate successful request per second and failed requests
+  // per seconds to result summary.
+  auto time_estimate = average(result->client_stats(), WallTime);
   if (result->request_results_size() > 0) {
     int64_t successes = 0;
     int64_t failures = 0;
@@ -193,6 +198,9 @@ static void postprocess_scenario_result(ScenarioResult* result) {
                                                               time_estimate);
   }
 
+  // Fill in data for other metrics required in result summary
+  auto qps_per_server_core = qps / sum(result->server_cores(), Cores);
+  result->mutable_summary()->set_qps_per_server_core(qps_per_server_core);
   result->mutable_summary()->set_client_polls_per_request(
       sum(result->client_stats(), CliPollCount) / histogram.Count());
   result->mutable_summary()->set_server_polls_per_request(

From 3b3d1796168382f08b365d2eb2e18befe4588584 Mon Sep 17 00:00:00 2001
From: Na-Na Pang <ladynana@google.com>
Date: Mon, 14 Oct 2019 12:49:28 -0700
Subject: [PATCH 05/26] Add comments in control.proto to clarify each field in
 ScenarioResultSummary

---
 src/proto/grpc/testing/control.proto | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto
index 564d8234b77..e699ea9c4d5 100644
--- a/src/proto/grpc/testing/control.proto
+++ b/src/proto/grpc/testing/control.proto
@@ -220,17 +220,17 @@ message Scenarios {
 // once the scenario has finished.
 message ScenarioResultSummary
 {
-  // Total number of operations per second over all clients.
+  // Total number of queries per second over all clients.
   double qps = 1;
-  // QPS per one server core.
+  // QPS per server core.
   double qps_per_server_core = 2;
-  // server load based on system_time (0.85 => 85%)
+  // The percentage of server cpu load based on system time used by the server process (85 => 85%)
   double server_system_time = 3;
-  // server load based on user_time (0.85 => 85%)
+  // The percentage of server cpu load based on user time used by the server process (85 => 85%)
   double server_user_time = 4;
-  // client load based on system_time (0.85 => 85%)
+  // The percentage of client cpu load based on system time used by the client process (85 => 85%)
   double client_system_time = 5;
-  // client load based on user_time (0.85 => 85%)
+  // The percentage of client cpu load based on user time used by the client process (85 => 85%)
   double client_user_time = 6;
 
   // X% latency percentiles (in nanoseconds)

From cac43de33f906c32e502d1506de4d931a8967c96 Mon Sep 17 00:00:00 2001
From: Na-Na Pang <ladynana@google.com>
Date: Mon, 14 Oct 2019 13:49:24 -0700
Subject: [PATCH 06/26] clang format

---
 test/cpp/qps/driver.cc | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc
index bd513e78a5d..9a52badbc22 100644
--- a/test/cpp/qps/driver.cc
+++ b/test/cpp/qps/driver.cc
@@ -144,27 +144,35 @@ static void postprocess_scenario_result(ScenarioResult* result) {
   result->mutable_summary()->set_latency_99(histogram.Percentile(99));
   result->mutable_summary()->set_latency_999(histogram.Percentile(99.9));
 
-  // Calculate qps and cpu load for each client and then aggregate results for all clients 
+  // Calculate qps and cpu load for each client and then aggregate results for
+  // all clients
   double qps = 0;
   double client_system_cpu_load = 0, client_user_cpu_load = 0;
   double server_system_cpu_load = 0, server_user_cpu_load = 0;
-  for (size_t i = 0; i < result->client_stats_size(); i++){
+  for (size_t i = 0; i < result->client_stats_size(); i++) {
     auto client_stat = result->client_stats(i);
     qps += client_stat.latencies().count() / client_stat.time_system();
-    client_system_cpu_load += client_stat.time_system() / client_stat.time_elapsed();
-    client_user_cpu_load += client_stat.time_user() / client_stat.time_elapsed();
+    client_system_cpu_load +=
+        client_stat.time_system() / client_stat.time_elapsed();
+    client_user_cpu_load +=
+        client_stat.time_user() / client_stat.time_elapsed();
   }
-  // Calculate cpu load for each server and then aggregate results for all servers
-  for (size_t i = 0; i < result->server_stats_size(); i++){
+  // Calculate cpu load for each server and then aggregate results for all
+  // servers
+  for (size_t i = 0; i < result->server_stats_size(); i++) {
     auto server_stat = result->server_stats(i);
-    server_system_cpu_load += server_stat.time_system() / server_stat.time_elapsed();
-    server_user_cpu_load += server_stat.time_user() / server_stat.time_elapsed();
+    server_system_cpu_load +=
+        server_stat.time_system() / server_stat.time_elapsed();
+    server_user_cpu_load +=
+        server_stat.time_user() / server_stat.time_elapsed();
   }
   result->mutable_summary()->set_qps(qps);
   // Populate the percentage of cpu load to result summary.
-  result->mutable_summary()->set_server_system_time(100 * server_system_cpu_load);
+  result->mutable_summary()->set_server_system_time(100 *
+                                                    server_system_cpu_load);
   result->mutable_summary()->set_server_user_time(100 * server_user_cpu_load);
-  result->mutable_summary()->set_client_system_time(100* client_system_cpu_load);
+  result->mutable_summary()->set_client_system_time(100 *
+                                                    client_system_cpu_load);
   result->mutable_summary()->set_client_user_time(100 * client_user_cpu_load);
 
   // For Non-linux platform, get_cpu_usage() is not implemented. Thus,

From f2e266fcd18a817eb6c1e3da639c2c9fc34e3fd3 Mon Sep 17 00:00:00 2001
From: Na-Na Pang <ladynana@google.com>
Date: Mon, 14 Oct 2019 13:54:10 -0700
Subject: [PATCH 07/26] Fix typo

---
 test/cpp/qps/driver.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc
index 9a52badbc22..7cf7a312e78 100644
--- a/test/cpp/qps/driver.cc
+++ b/test/cpp/qps/driver.cc
@@ -151,7 +151,7 @@ static void postprocess_scenario_result(ScenarioResult* result) {
   double server_system_cpu_load = 0, server_user_cpu_load = 0;
   for (size_t i = 0; i < result->client_stats_size(); i++) {
     auto client_stat = result->client_stats(i);
-    qps += client_stat.latencies().count() / client_stat.time_system();
+    qps += client_stat.latencies().count() / client_stat.time_elapse();
     client_system_cpu_load +=
         client_stat.time_system() / client_stat.time_elapsed();
     client_user_cpu_load +=

From 60f6689103a6b89da8f3b653d727e5b59dd9f5d6 Mon Sep 17 00:00:00 2001
From: Na-Na Pang <ladynana@google.com>
Date: Mon, 14 Oct 2019 13:54:44 -0700
Subject: [PATCH 08/26] Fix typo

---
 test/cpp/qps/driver.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc
index 7cf7a312e78..71a03fb73c5 100644
--- a/test/cpp/qps/driver.cc
+++ b/test/cpp/qps/driver.cc
@@ -151,7 +151,7 @@ static void postprocess_scenario_result(ScenarioResult* result) {
   double server_system_cpu_load = 0, server_user_cpu_load = 0;
   for (size_t i = 0; i < result->client_stats_size(); i++) {
     auto client_stat = result->client_stats(i);
-    qps += client_stat.latencies().count() / client_stat.time_elapse();
+    qps += client_stat.latencies().count() / client_stat.time_elapsed();
     client_system_cpu_load +=
         client_stat.time_system() / client_stat.time_elapsed();
     client_user_cpu_load +=

From 1a374bc98435814f90a9423e404222a6136da93a Mon Sep 17 00:00:00 2001
From: Na-Na Pang <ladynana@google.com>
Date: Mon, 21 Oct 2019 18:21:08 -0700
Subject: [PATCH 09/26] Add explanation to the comments

---
 src/proto/grpc/testing/control.proto | 4 +++-
 test/cpp/qps/driver.cc               | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto
index e699ea9c4d5..734a38df343 100644
--- a/src/proto/grpc/testing/control.proto
+++ b/src/proto/grpc/testing/control.proto
@@ -220,7 +220,9 @@ message Scenarios {
 // once the scenario has finished.
 message ScenarioResultSummary
 {
-  // Total number of queries per second over all clients.
+  // Total number of oeprations per second over all clients. What is counted as 1 'operation' depends on the benchmark scenarios:
+  // For unary benchmarks, an operation is processing of a single unary RPC. 
+  // For streaming benchmarks, an operation is processing of a single ping pong of request and response. 
   double qps = 1;
   // QPS per server core.
   double qps_per_server_core = 2;
diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc
index 71a03fb73c5..94c2feffc19 100644
--- a/test/cpp/qps/driver.cc
+++ b/test/cpp/qps/driver.cc
@@ -148,7 +148,6 @@ static void postprocess_scenario_result(ScenarioResult* result) {
   // all clients
   double qps = 0;
   double client_system_cpu_load = 0, client_user_cpu_load = 0;
-  double server_system_cpu_load = 0, server_user_cpu_load = 0;
   for (size_t i = 0; i < result->client_stats_size(); i++) {
     auto client_stat = result->client_stats(i);
     qps += client_stat.latencies().count() / client_stat.time_elapsed();
@@ -159,6 +158,7 @@ static void postprocess_scenario_result(ScenarioResult* result) {
   }
   // Calculate cpu load for each server and then aggregate results for all
   // servers
+  double server_system_cpu_load = 0, server_user_cpu_load = 0;
   for (size_t i = 0; i < result->server_stats_size(); i++) {
     auto server_stat = result->server_stats(i);
     server_system_cpu_load +=

From ca0329cf3d2ed89fd8f76150b07bf6842aaca1d8 Mon Sep 17 00:00:00 2001
From: Na-Na Pang <ladynana@google.com>
Date: Mon, 21 Oct 2019 18:28:19 -0700
Subject: [PATCH 10/26] Add explanation to the comments

---
 src/proto/grpc/testing/control.proto | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto
index 734a38df343..eaed74ddfab 100644
--- a/src/proto/grpc/testing/control.proto
+++ b/src/proto/grpc/testing/control.proto
@@ -226,13 +226,15 @@ message ScenarioResultSummary
   double qps = 1;
   // QPS per server core.
   double qps_per_server_core = 2;
-  // The percentage of server cpu load based on system time used by the server process (85 => 85%)
+  // The overall percentage of server cpu load based on system time across all the server processes.
+  // For example, 85 implies 85% of a cpu core, 125 implies 125% of a cpu core. Since we are accumulating the cpu load across all the server 
+  // processes, the value could > 100 when there are multiple servers. 
   double server_system_time = 3;
-  // The percentage of server cpu load based on user time used by the server process (85 => 85%)
+  // The overall percentage of server cpu load based on user time across all the server processes (85 => 85%, 125 => 125%)
   double server_user_time = 4;
-  // The percentage of client cpu load based on system time used by the client process (85 => 85%)
+  // The overall percentage of client cpu load based on system time across all the client processes (85 => 85%, 125 => 125%)
   double client_system_time = 5;
-  // The percentage of client cpu load based on user time used by the client process (85 => 85%)
+  // The overall percentage of client cpu load based on user time across all the client processes (85 => 85%, 125 => 125%)
   double client_user_time = 6;
 
   // X% latency percentiles (in nanoseconds)

From c85964edfa474c5b5ca8487f3442f8bc14f6c8cc Mon Sep 17 00:00:00 2001
From: Na-Na Pang <ladynana@google.com>
Date: Fri, 25 Oct 2019 16:51:49 -0700
Subject: [PATCH 11/26] Modify comments

---
 src/proto/grpc/testing/control.proto | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto
index eaed74ddfab..2214538ce99 100644
--- a/src/proto/grpc/testing/control.proto
+++ b/src/proto/grpc/testing/control.proto
@@ -220,7 +220,7 @@ message Scenarios {
 // once the scenario has finished.
 message ScenarioResultSummary
 {
-  // Total number of oeprations per second over all clients. What is counted as 1 'operation' depends on the benchmark scenarios:
+  // Total number of operations per second over all clients. What is counted as 1 'operation' depends on the benchmark scenarios:
   // For unary benchmarks, an operation is processing of a single unary RPC. 
   // For streaming benchmarks, an operation is processing of a single ping pong of request and response. 
   double qps = 1;

From a7b4d4588e217f6c5aac6a121fed149b3227700d Mon Sep 17 00:00:00 2001
From: Na-Na Pang <ladynana@google.com>
Date: Mon, 4 Nov 2019 17:19:43 -0800
Subject: [PATCH 12/26] Add explaination to comments

---
 src/proto/grpc/testing/control.proto | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto
index 2214538ce99..91255d14415 100644
--- a/src/proto/grpc/testing/control.proto
+++ b/src/proto/grpc/testing/control.proto
@@ -226,15 +226,16 @@ message ScenarioResultSummary
   double qps = 1;
   // QPS per server core.
   double qps_per_server_core = 2;
-  // The overall percentage of server cpu load based on system time across all the server processes.
+  // The total server cpu load based on system time across all server processes, expressed as percentage of a single cpu core.
   // For example, 85 implies 85% of a cpu core, 125 implies 125% of a cpu core. Since we are accumulating the cpu load across all the server 
-  // processes, the value could > 100 when there are multiple servers. 
+  // processes, the value could > 100 when there are multiple servers or a single server using multiple threads and cores. 
+  // Same explanation for the total client cpu load below.
   double server_system_time = 3;
-  // The overall percentage of server cpu load based on user time across all the server processes (85 => 85%, 125 => 125%)
+  // The total server cpu load based on user time across all server processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%)
   double server_user_time = 4;
-  // The overall percentage of client cpu load based on system time across all the client processes (85 => 85%, 125 => 125%)
+  // The total client cpu load based on system time across all client processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%)
   double client_system_time = 5;
-  // The overall percentage of client cpu load based on user time across all the client processes (85 => 85%, 125 => 125%)
+  // The total client cpu load based on user time across all client processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%)
   double client_user_time = 6;
 
   // X% latency percentiles (in nanoseconds)

From 0e487a7405cbe35bf6fc0105fadf27146bc35672 Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Thu, 14 Nov 2019 09:45:17 -0800
Subject: [PATCH 13/26] Convert grpc_connector to C++.

---
 BUILD                                         |   1 -
 BUILD.gn                                      |   2 -
 CMakeLists.txt                                |   7 -
 Makefile                                      |   7 -
 build.yaml                                    |   1 -
 config.m4                                     |   1 -
 config.w32                                    |   1 -
 gRPC-Core.podspec                             |   1 -
 grpc.gemspec                                  |   1 -
 grpc.gyp                                      |   6 -
 package.xml                                   |   1 -
 .../ext/filters/client_channel/connector.cc   |  41 ---
 .../ext/filters/client_channel/connector.h    |  85 +++--
 .../ext/filters/client_channel/subchannel.cc  |  27 +-
 .../ext/filters/client_channel/subchannel.h   |   8 +-
 .../chttp2/client/chttp2_connector.cc         | 308 ++++++++----------
 .../chttp2/client/chttp2_connector.h          |  32 +-
 .../chttp2/client/insecure/channel_create.cc  |   5 +-
 .../client/secure/secure_channel_create.cc    |   5 +-
 src/python/grpcio/grpc_core_dependencies.py   |   1 -
 tools/doxygen/Doxyfile.c++.internal           |   1 -
 tools/doxygen/Doxyfile.core.internal          |   1 -
 22 files changed, 226 insertions(+), 317 deletions(-)
 delete mode 100644 src/core/ext/filters/client_channel/connector.cc

diff --git a/BUILD b/BUILD
index 7cc070e4ba0..281f56cbcb4 100644
--- a/BUILD
+++ b/BUILD
@@ -1020,7 +1020,6 @@ grpc_cc_library(
         "src/core/ext/filters/client_channel/client_channel_channelz.cc",
         "src/core/ext/filters/client_channel/client_channel_factory.cc",
         "src/core/ext/filters/client_channel/client_channel_plugin.cc",
-        "src/core/ext/filters/client_channel/connector.cc",
         "src/core/ext/filters/client_channel/global_subchannel_pool.cc",
         "src/core/ext/filters/client_channel/health/health_check_client.cc",
         "src/core/ext/filters/client_channel/http_connect_handshaker.cc",
diff --git a/BUILD.gn b/BUILD.gn
index a69a8e0940a..c17f339f241 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -218,7 +218,6 @@ config("grpc_config") {
         "src/core/ext/filters/client_channel/client_channel_factory.cc",
         "src/core/ext/filters/client_channel/client_channel_factory.h",
         "src/core/ext/filters/client_channel/client_channel_plugin.cc",
-        "src/core/ext/filters/client_channel/connector.cc",
         "src/core/ext/filters/client_channel/connector.h",
         "src/core/ext/filters/client_channel/global_subchannel_pool.cc",
         "src/core/ext/filters/client_channel/global_subchannel_pool.h",
@@ -1202,7 +1201,6 @@ config("grpc_config") {
         "src/core/ext/filters/client_channel/client_channel_factory.cc",
         "src/core/ext/filters/client_channel/client_channel_factory.h",
         "src/core/ext/filters/client_channel/client_channel_plugin.cc",
-        "src/core/ext/filters/client_channel/connector.cc",
         "src/core/ext/filters/client_channel/connector.h",
         "src/core/ext/filters/client_channel/global_subchannel_pool.cc",
         "src/core/ext/filters/client_channel/global_subchannel_pool.h",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6c370c3ebd8..b000dde6cde 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1360,7 +1360,6 @@ add_library(grpc
   src/core/ext/filters/client_channel/client_channel_channelz.cc
   src/core/ext/filters/client_channel/client_channel_factory.cc
   src/core/ext/filters/client_channel/client_channel_plugin.cc
-  src/core/ext/filters/client_channel/connector.cc
   src/core/ext/filters/client_channel/global_subchannel_pool.cc
   src/core/ext/filters/client_channel/health/health_check_client.cc
   src/core/ext/filters/client_channel/http_connect_handshaker.cc
@@ -1766,7 +1765,6 @@ add_library(grpc_cronet
   src/core/ext/filters/client_channel/client_channel_channelz.cc
   src/core/ext/filters/client_channel/client_channel_factory.cc
   src/core/ext/filters/client_channel/client_channel_plugin.cc
-  src/core/ext/filters/client_channel/connector.cc
   src/core/ext/filters/client_channel/global_subchannel_pool.cc
   src/core/ext/filters/client_channel/health/health_check_client.cc
   src/core/ext/filters/client_channel/http_connect_handshaker.cc
@@ -2164,7 +2162,6 @@ add_library(grpc_test_util
   src/core/ext/filters/client_channel/client_channel_channelz.cc
   src/core/ext/filters/client_channel/client_channel_factory.cc
   src/core/ext/filters/client_channel/client_channel_plugin.cc
-  src/core/ext/filters/client_channel/connector.cc
   src/core/ext/filters/client_channel/global_subchannel_pool.cc
   src/core/ext/filters/client_channel/health/health_check_client.cc
   src/core/ext/filters/client_channel/http_connect_handshaker.cc
@@ -2508,7 +2505,6 @@ add_library(grpc_test_util_unsecure
   src/core/ext/filters/client_channel/client_channel_channelz.cc
   src/core/ext/filters/client_channel/client_channel_factory.cc
   src/core/ext/filters/client_channel/client_channel_plugin.cc
-  src/core/ext/filters/client_channel/connector.cc
   src/core/ext/filters/client_channel/global_subchannel_pool.cc
   src/core/ext/filters/client_channel/health/health_check_client.cc
   src/core/ext/filters/client_channel/http_connect_handshaker.cc
@@ -2863,7 +2859,6 @@ add_library(grpc_unsecure
   src/core/ext/filters/client_channel/client_channel_channelz.cc
   src/core/ext/filters/client_channel/client_channel_factory.cc
   src/core/ext/filters/client_channel/client_channel_plugin.cc
-  src/core/ext/filters/client_channel/connector.cc
   src/core/ext/filters/client_channel/global_subchannel_pool.cc
   src/core/ext/filters/client_channel/health/health_check_client.cc
   src/core/ext/filters/client_channel/http_connect_handshaker.cc
@@ -3301,7 +3296,6 @@ add_library(grpc++
   src/core/ext/filters/client_channel/client_channel_channelz.cc
   src/core/ext/filters/client_channel/client_channel_factory.cc
   src/core/ext/filters/client_channel/client_channel_plugin.cc
-  src/core/ext/filters/client_channel/connector.cc
   src/core/ext/filters/client_channel/global_subchannel_pool.cc
   src/core/ext/filters/client_channel/health/health_check_client.cc
   src/core/ext/filters/client_channel/http_connect_handshaker.cc
@@ -4563,7 +4557,6 @@ add_library(grpc++_unsecure
   src/core/ext/filters/client_channel/client_channel_channelz.cc
   src/core/ext/filters/client_channel/client_channel_factory.cc
   src/core/ext/filters/client_channel/client_channel_plugin.cc
-  src/core/ext/filters/client_channel/connector.cc
   src/core/ext/filters/client_channel/global_subchannel_pool.cc
   src/core/ext/filters/client_channel/health/health_check_client.cc
   src/core/ext/filters/client_channel/http_connect_handshaker.cc
diff --git a/Makefile b/Makefile
index ed34f3e950a..4d398c4477f 100644
--- a/Makefile
+++ b/Makefile
@@ -3848,7 +3848,6 @@ LIBGRPC_SRC = \
     src/core/ext/filters/client_channel/client_channel_channelz.cc \
     src/core/ext/filters/client_channel/client_channel_factory.cc \
     src/core/ext/filters/client_channel/client_channel_plugin.cc \
-    src/core/ext/filters/client_channel/connector.cc \
     src/core/ext/filters/client_channel/global_subchannel_pool.cc \
     src/core/ext/filters/client_channel/health/health_check_client.cc \
     src/core/ext/filters/client_channel/http_connect_handshaker.cc \
@@ -4246,7 +4245,6 @@ LIBGRPC_CRONET_SRC = \
     src/core/ext/filters/client_channel/client_channel_channelz.cc \
     src/core/ext/filters/client_channel/client_channel_factory.cc \
     src/core/ext/filters/client_channel/client_channel_plugin.cc \
-    src/core/ext/filters/client_channel/connector.cc \
     src/core/ext/filters/client_channel/global_subchannel_pool.cc \
     src/core/ext/filters/client_channel/health/health_check_client.cc \
     src/core/ext/filters/client_channel/http_connect_handshaker.cc \
@@ -4635,7 +4633,6 @@ LIBGRPC_TEST_UTIL_SRC = \
     src/core/ext/filters/client_channel/client_channel_channelz.cc \
     src/core/ext/filters/client_channel/client_channel_factory.cc \
     src/core/ext/filters/client_channel/client_channel_plugin.cc \
-    src/core/ext/filters/client_channel/connector.cc \
     src/core/ext/filters/client_channel/global_subchannel_pool.cc \
     src/core/ext/filters/client_channel/health/health_check_client.cc \
     src/core/ext/filters/client_channel/http_connect_handshaker.cc \
@@ -4965,7 +4962,6 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \
     src/core/ext/filters/client_channel/client_channel_channelz.cc \
     src/core/ext/filters/client_channel/client_channel_factory.cc \
     src/core/ext/filters/client_channel/client_channel_plugin.cc \
-    src/core/ext/filters/client_channel/connector.cc \
     src/core/ext/filters/client_channel/global_subchannel_pool.cc \
     src/core/ext/filters/client_channel/health/health_check_client.cc \
     src/core/ext/filters/client_channel/http_connect_handshaker.cc \
@@ -5293,7 +5289,6 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/ext/filters/client_channel/client_channel_channelz.cc \
     src/core/ext/filters/client_channel/client_channel_factory.cc \
     src/core/ext/filters/client_channel/client_channel_plugin.cc \
-    src/core/ext/filters/client_channel/connector.cc \
     src/core/ext/filters/client_channel/global_subchannel_pool.cc \
     src/core/ext/filters/client_channel/health/health_check_client.cc \
     src/core/ext/filters/client_channel/http_connect_handshaker.cc \
@@ -5696,7 +5691,6 @@ LIBGRPC++_SRC = \
     src/core/ext/filters/client_channel/client_channel_channelz.cc \
     src/core/ext/filters/client_channel/client_channel_factory.cc \
     src/core/ext/filters/client_channel/client_channel_plugin.cc \
-    src/core/ext/filters/client_channel/connector.cc \
     src/core/ext/filters/client_channel/global_subchannel_pool.cc \
     src/core/ext/filters/client_channel/health/health_check_client.cc \
     src/core/ext/filters/client_channel/http_connect_handshaker.cc \
@@ -6924,7 +6918,6 @@ LIBGRPC++_UNSECURE_SRC = \
     src/core/ext/filters/client_channel/client_channel_channelz.cc \
     src/core/ext/filters/client_channel/client_channel_factory.cc \
     src/core/ext/filters/client_channel/client_channel_plugin.cc \
-    src/core/ext/filters/client_channel/connector.cc \
     src/core/ext/filters/client_channel/global_subchannel_pool.cc \
     src/core/ext/filters/client_channel/health/health_check_client.cc \
     src/core/ext/filters/client_channel/http_connect_handshaker.cc \
diff --git a/build.yaml b/build.yaml
index 364f3bfcb50..97cd943fdee 100644
--- a/build.yaml
+++ b/build.yaml
@@ -996,7 +996,6 @@ filegroups:
   - src/core/ext/filters/client_channel/client_channel_channelz.cc
   - src/core/ext/filters/client_channel/client_channel_factory.cc
   - src/core/ext/filters/client_channel/client_channel_plugin.cc
-  - src/core/ext/filters/client_channel/connector.cc
   - src/core/ext/filters/client_channel/global_subchannel_pool.cc
   - src/core/ext/filters/client_channel/health/health_check_client.cc
   - src/core/ext/filters/client_channel/http_connect_handshaker.cc
diff --git a/config.m4 b/config.m4
index 322029f0c72..933e940b948 100644
--- a/config.m4
+++ b/config.m4
@@ -45,7 +45,6 @@ if test "$PHP_GRPC" != "no"; then
     src/core/ext/filters/client_channel/client_channel_channelz.cc \
     src/core/ext/filters/client_channel/client_channel_factory.cc \
     src/core/ext/filters/client_channel/client_channel_plugin.cc \
-    src/core/ext/filters/client_channel/connector.cc \
     src/core/ext/filters/client_channel/global_subchannel_pool.cc \
     src/core/ext/filters/client_channel/health/health_check_client.cc \
     src/core/ext/filters/client_channel/http_connect_handshaker.cc \
diff --git a/config.w32 b/config.w32
index 0062eafa8b6..934f7c63240 100644
--- a/config.w32
+++ b/config.w32
@@ -325,7 +325,6 @@ if (PHP_GRPC != "no") {
     "src\\core\\ext\\filters\\client_channel\\client_channel_channelz.cc " +
     "src\\core\\ext\\filters\\client_channel\\client_channel_factory.cc " +
     "src\\core\\ext\\filters\\client_channel\\client_channel_plugin.cc " +
-    "src\\core\\ext\\filters\\client_channel\\connector.cc " +
     "src\\core\\ext\\filters\\client_channel\\global_subchannel_pool.cc " +
     "src\\core\\ext\\filters\\client_channel\\health\\health_check_client.cc " +
     "src\\core\\ext\\filters\\client_channel\\http_connect_handshaker.cc " +
diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec
index 04e30e692cd..8989626e319 100644
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@@ -201,7 +201,6 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/client_channel/client_channel_factory.cc',
                       'src/core/ext/filters/client_channel/client_channel_factory.h',
                       'src/core/ext/filters/client_channel/client_channel_plugin.cc',
-                      'src/core/ext/filters/client_channel/connector.cc',
                       'src/core/ext/filters/client_channel/connector.h',
                       'src/core/ext/filters/client_channel/global_subchannel_pool.cc',
                       'src/core/ext/filters/client_channel/global_subchannel_pool.h',
diff --git a/grpc.gemspec b/grpc.gemspec
index d3354b1de7e..ed711536378 100644
--- a/grpc.gemspec
+++ b/grpc.gemspec
@@ -113,7 +113,6 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/filters/client_channel/client_channel_factory.cc )
   s.files += %w( src/core/ext/filters/client_channel/client_channel_factory.h )
   s.files += %w( src/core/ext/filters/client_channel/client_channel_plugin.cc )
-  s.files += %w( src/core/ext/filters/client_channel/connector.cc )
   s.files += %w( src/core/ext/filters/client_channel/connector.h )
   s.files += %w( src/core/ext/filters/client_channel/global_subchannel_pool.cc )
   s.files += %w( src/core/ext/filters/client_channel/global_subchannel_pool.h )
diff --git a/grpc.gyp b/grpc.gyp
index 3d62307f057..58de98e7131 100644
--- a/grpc.gyp
+++ b/grpc.gyp
@@ -494,7 +494,6 @@
         'src/core/ext/filters/client_channel/client_channel_channelz.cc',
         'src/core/ext/filters/client_channel/client_channel_factory.cc',
         'src/core/ext/filters/client_channel/client_channel_plugin.cc',
-        'src/core/ext/filters/client_channel/connector.cc',
         'src/core/ext/filters/client_channel/global_subchannel_pool.cc',
         'src/core/ext/filters/client_channel/health/health_check_client.cc',
         'src/core/ext/filters/client_channel/http_connect_handshaker.cc',
@@ -807,7 +806,6 @@
         'src/core/ext/filters/client_channel/client_channel_channelz.cc',
         'src/core/ext/filters/client_channel/client_channel_factory.cc',
         'src/core/ext/filters/client_channel/client_channel_plugin.cc',
-        'src/core/ext/filters/client_channel/connector.cc',
         'src/core/ext/filters/client_channel/global_subchannel_pool.cc',
         'src/core/ext/filters/client_channel/health/health_check_client.cc',
         'src/core/ext/filters/client_channel/http_connect_handshaker.cc',
@@ -1071,7 +1069,6 @@
         'src/core/ext/filters/client_channel/client_channel_channelz.cc',
         'src/core/ext/filters/client_channel/client_channel_factory.cc',
         'src/core/ext/filters/client_channel/client_channel_plugin.cc',
-        'src/core/ext/filters/client_channel/connector.cc',
         'src/core/ext/filters/client_channel/global_subchannel_pool.cc',
         'src/core/ext/filters/client_channel/health/health_check_client.cc',
         'src/core/ext/filters/client_channel/http_connect_handshaker.cc',
@@ -1346,7 +1343,6 @@
         'src/core/ext/filters/client_channel/client_channel_channelz.cc',
         'src/core/ext/filters/client_channel/client_channel_factory.cc',
         'src/core/ext/filters/client_channel/client_channel_plugin.cc',
-        'src/core/ext/filters/client_channel/connector.cc',
         'src/core/ext/filters/client_channel/global_subchannel_pool.cc',
         'src/core/ext/filters/client_channel/health/health_check_client.cc',
         'src/core/ext/filters/client_channel/http_connect_handshaker.cc',
@@ -1557,7 +1553,6 @@
         'src/core/ext/filters/client_channel/client_channel_channelz.cc',
         'src/core/ext/filters/client_channel/client_channel_factory.cc',
         'src/core/ext/filters/client_channel/client_channel_plugin.cc',
-        'src/core/ext/filters/client_channel/connector.cc',
         'src/core/ext/filters/client_channel/global_subchannel_pool.cc',
         'src/core/ext/filters/client_channel/health/health_check_client.cc',
         'src/core/ext/filters/client_channel/http_connect_handshaker.cc',
@@ -1911,7 +1906,6 @@
         'src/core/ext/filters/client_channel/client_channel_channelz.cc',
         'src/core/ext/filters/client_channel/client_channel_factory.cc',
         'src/core/ext/filters/client_channel/client_channel_plugin.cc',
-        'src/core/ext/filters/client_channel/connector.cc',
         'src/core/ext/filters/client_channel/global_subchannel_pool.cc',
         'src/core/ext/filters/client_channel/health/health_check_client.cc',
         'src/core/ext/filters/client_channel/http_connect_handshaker.cc',
diff --git a/package.xml b/package.xml
index fc63e5772b2..b5c5f312825 100644
--- a/package.xml
+++ b/package.xml
@@ -96,7 +96,6 @@
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/client_channel_factory.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/client_channel_factory.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/client_channel_plugin.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/ext/filters/client_channel/connector.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/connector.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/global_subchannel_pool.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/global_subchannel_pool.h" role="src" />
diff --git a/src/core/ext/filters/client_channel/connector.cc b/src/core/ext/filters/client_channel/connector.cc
deleted file mode 100644
index 5e04b3b4539..00000000000
--- a/src/core/ext/filters/client_channel/connector.cc
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- *
- * Copyright 2015 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <grpc/support/port_platform.h>
-
-#include "src/core/ext/filters/client_channel/connector.h"
-
-grpc_connector* grpc_connector_ref(grpc_connector* connector) {
-  connector->vtable->ref(connector);
-  return connector;
-}
-
-void grpc_connector_unref(grpc_connector* connector) {
-  connector->vtable->unref(connector);
-}
-
-void grpc_connector_connect(grpc_connector* connector,
-                            const grpc_connect_in_args* in_args,
-                            grpc_connect_out_args* out_args,
-                            grpc_closure* notify) {
-  connector->vtable->connect(connector, in_args, out_args, notify);
-}
-
-void grpc_connector_shutdown(grpc_connector* connector, grpc_error* why) {
-  connector->vtable->shutdown(connector, why);
-}
diff --git a/src/core/ext/filters/client_channel/connector.h b/src/core/ext/filters/client_channel/connector.h
index 2bd5ff26e3e..256471456e3 100644
--- a/src/core/ext/filters/client_channel/connector.h
+++ b/src/core/ext/filters/client_channel/connector.h
@@ -23,62 +23,57 @@
 
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/channelz.h"
+#include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/transport/transport.h"
 
-typedef struct grpc_connector grpc_connector;
-typedef struct grpc_connector_vtable grpc_connector_vtable;
+namespace grpc_core {
 
-struct grpc_connector {
-  const grpc_connector_vtable* vtable;
-};
+// Interface for connection-establishment functionality.
+// Each transport that supports client channels (e.g., not inproc) must
+// supply an implementation of this.
+class SubchannelConnector : public InternallyRefCounted<SubchannelConnector> {
+ public:
+  struct Args {
+    // Set of pollsets interested in this connection.
+    grpc_pollset_set* interested_parties;
+    // Deadline for connection.
+    grpc_millis deadline;
+    // Channel args to be passed to handshakers and transport.
+    const grpc_channel_args* channel_args;
+  };
 
-typedef struct {
-  /** set of pollsets interested in this connection */
-  grpc_pollset_set* interested_parties;
-  /** deadline for connection */
-  grpc_millis deadline;
-  /** channel arguments (to be passed to transport) */
-  const grpc_channel_args* channel_args;
-} grpc_connect_in_args;
+  struct Result {
+    // The connected transport.
+    grpc_transport* transport = nullptr;
+    // Channel args to be passed to filters.
+    const grpc_channel_args* channel_args = nullptr;
+    // Channelz socket node of the connected transport, if any.
+    RefCountedPtr<channelz::SocketNode> socket_node;
 
-typedef struct {
-  /** the connected transport */
-  grpc_transport* transport;
+    void Reset() {
+      transport = nullptr;
+      channel_args = nullptr;
+      socket_node.reset();
+    }
+  };
 
-  /** channel arguments (to be passed to the filters) */
-  grpc_channel_args* channel_args;
+  // Attempts to connect.
+  // When complete, populates *result and invokes notify.
+  // Only one connection attempt may be in progress at any one time.
+  virtual void Connect(const Args& args, Result* result,
+                       grpc_closure* notify) = 0;
 
-  /** channelz socket node of the connected transport. nullptr if not available
-   */
-  grpc_core::RefCountedPtr<grpc_core::channelz::SocketNode> socket;
+  // Cancels any in-flight connection attempt and shuts down the
+  // connector.
+  virtual void Shutdown(grpc_error* error) = 0;
 
-  void reset() {
-    transport = nullptr;
-    channel_args = nullptr;
-    socket = nullptr;
+  void Orphan() override {
+    Shutdown(GRPC_ERROR_CREATE_FROM_STATIC_STRING("Subchannel disconnected"));
+    Unref();
   }
-} grpc_connect_out_args;
-
-struct grpc_connector_vtable {
-  void (*ref)(grpc_connector* connector);
-  void (*unref)(grpc_connector* connector);
-  /** Implementation of grpc_connector_shutdown */
-  void (*shutdown)(grpc_connector* connector, grpc_error* why);
-  /** Implementation of grpc_connector_connect */
-  void (*connect)(grpc_connector* connector,
-                  const grpc_connect_in_args* in_args,
-                  grpc_connect_out_args* out_args, grpc_closure* notify);
 };
 
-grpc_connector* grpc_connector_ref(grpc_connector* connector);
-void grpc_connector_unref(grpc_connector* connector);
-/** Connect using the connector: max one outstanding call at a time */
-void grpc_connector_connect(grpc_connector* connector,
-                            const grpc_connect_in_args* in_args,
-                            grpc_connect_out_args* out_args,
-                            grpc_closure* notify);
-/** Cancel any pending connection */
-void grpc_connector_shutdown(grpc_connector* connector, grpc_error* why);
+}  // namespace grpc_core
 
 #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_CONNECTOR_H */
diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc
index 4213b7ae086..26769e861e7 100644
--- a/src/core/ext/filters/client_channel/subchannel.cc
+++ b/src/core/ext/filters/client_channel/subchannel.cc
@@ -613,14 +613,14 @@ BackOff::Options ParseArgsForBackoffValues(
 
 }  // namespace
 
-Subchannel::Subchannel(SubchannelKey* key, grpc_connector* connector,
+Subchannel::Subchannel(SubchannelKey* key,
+                       OrphanablePtr<SubchannelConnector> connector,
                        const grpc_channel_args* args)
     : key_(key),
-      connector_(connector),
+      connector_(std::move(connector)),
       backoff_(ParseArgsForBackoffValues(args, &min_connect_timeout_ms_)) {
   GRPC_STATS_INC_CLIENT_SUBCHANNELS_CREATED();
   gpr_atm_no_barrier_store(&ref_pair_, 1 << INTERNAL_REF_BITS);
-  grpc_connector_ref(connector_);
   pollset_set_ = grpc_pollset_set_create();
   grpc_resolved_address* addr =
       static_cast<grpc_resolved_address*>(gpr_malloc(sizeof(*addr)));
@@ -668,12 +668,12 @@ Subchannel::~Subchannel() {
     channelz_node_->UpdateConnectivityState(GRPC_CHANNEL_SHUTDOWN);
   }
   grpc_channel_args_destroy(args_);
-  grpc_connector_unref(connector_);
+  connector_.reset();
   grpc_pollset_set_destroy(pollset_set_);
   delete key_;
 }
 
-Subchannel* Subchannel::Create(grpc_connector* connector,
+Subchannel* Subchannel::Create(OrphanablePtr<SubchannelConnector> connector,
                                const grpc_channel_args* args) {
   SubchannelKey* key = new SubchannelKey(args);
   SubchannelPoolInterface* subchannel_pool =
@@ -684,7 +684,7 @@ Subchannel* Subchannel::Create(grpc_connector* connector,
     delete key;
     return c;
   }
-  c = new Subchannel(key, connector, args);
+  c = new Subchannel(key, std::move(connector), args);
   // Try to register the subchannel before setting the subchannel pool.
   // Otherwise, in case of a registration race, unreffing c in
   // RegisterSubchannel() will cause c to be tried to be unregistered, while
@@ -975,7 +975,7 @@ void Subchannel::OnRetryAlarm(void* arg, grpc_error* error) {
 }
 
 void Subchannel::ContinueConnectingLocked() {
-  grpc_connect_in_args args;
+  SubchannelConnector::Args args;
   args.interested_parties = pollset_set_;
   const grpc_millis min_deadline =
       min_connect_timeout_ms_ + ExecCtx::Get()->Now();
@@ -983,13 +983,13 @@ void Subchannel::ContinueConnectingLocked() {
   args.deadline = std::max(next_attempt_deadline_, min_deadline);
   args.channel_args = args_;
   SetConnectivityStateLocked(GRPC_CHANNEL_CONNECTING);
-  grpc_connector_connect(connector_, &args, &connecting_result_,
-                         &on_connecting_finished_);
+  connector_->Connect(args, &connecting_result_, &on_connecting_finished_);
 }
 
 void Subchannel::OnConnectingFinished(void* arg, grpc_error* error) {
   auto* c = static_cast<Subchannel*>(arg);
-  grpc_channel_args* delete_channel_args = c->connecting_result_.channel_args;
+  const grpc_channel_args* delete_channel_args =
+      c->connecting_result_.channel_args;
   GRPC_SUBCHANNEL_WEAK_REF(c, "on_connecting_finished");
   {
     MutexLock lock(&c->mu_);
@@ -1042,8 +1042,8 @@ bool Subchannel::PublishTransportLocked() {
     return false;
   }
   RefCountedPtr<channelz::SocketNode> socket =
-      std::move(connecting_result_.socket);
-  connecting_result_.reset();
+      std::move(connecting_result_.socket_node);
+  connecting_result_.Reset();
   if (disconnected_) {
     grpc_channel_stack_destroy(stk);
     gpr_free(stk);
@@ -1075,8 +1075,7 @@ void Subchannel::Disconnect() {
   MutexLock lock(&mu_);
   GPR_ASSERT(!disconnected_);
   disconnected_ = true;
-  grpc_connector_shutdown(connector_, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                                          "Subchannel disconnected"));
+  connector_.reset();
   connected_subchannel_.reset();
   health_watcher_map_.ShutdownLocked();
 }
diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h
index ba4706e3177..cf4ddafb45e 100644
--- a/src/core/ext/filters/client_channel/subchannel.h
+++ b/src/core/ext/filters/client_channel/subchannel.h
@@ -197,12 +197,12 @@ class Subchannel {
   };
 
   // The ctor and dtor are not intended to use directly.
-  Subchannel(SubchannelKey* key, grpc_connector* connector,
+  Subchannel(SubchannelKey* key, OrphanablePtr<SubchannelConnector> connector,
              const grpc_channel_args* args);
   ~Subchannel();
 
   // Creates a subchannel given \a connector and \a args.
-  static Subchannel* Create(grpc_connector* connector,
+  static Subchannel* Create(OrphanablePtr<SubchannelConnector> connector,
                             const grpc_channel_args* args);
 
   // Strong and weak refcounting.
@@ -365,9 +365,9 @@ class Subchannel {
   gpr_atm ref_pair_;
 
   // Connection states.
-  grpc_connector* connector_ = nullptr;
+  OrphanablePtr<SubchannelConnector> connector_;
   // Set during connection.
-  grpc_connect_out_args connecting_result_;
+  SubchannelConnector::Result connecting_result_;
   grpc_closure on_connecting_finished_;
   // Active connection, or null.
   RefCountedPtr<ConnectedSubchannel> connected_subchannel_;
diff --git a/src/core/ext/transport/chttp2/client/chttp2_connector.cc b/src/core/ext/transport/chttp2/client/chttp2_connector.cc
index 18a2df8a2d0..431748be46f 100644
--- a/src/core/ext/transport/chttp2/client/chttp2_connector.cc
+++ b/src/core/ext/transport/chttp2/client/chttp2_connector.cc
@@ -38,202 +38,162 @@
 #include "src/core/lib/iomgr/tcp_client.h"
 #include "src/core/lib/slice/slice_internal.h"
 
-typedef struct {
-  grpc_connector base;
+namespace grpc_core {
 
-  gpr_mu mu;
-  gpr_refcount refs;
-
-  bool shutdown;
-  bool connecting;
-
-  grpc_closure* notify;
-  grpc_connect_in_args args;
-  grpc_connect_out_args* result;
-
-  grpc_endpoint* endpoint;  // Non-NULL until handshaking starts.
-
-  grpc_closure connected;
-
-  grpc_core::RefCountedPtr<grpc_core::HandshakeManager> handshake_mgr;
-} chttp2_connector;
+Chttp2Connector::Chttp2Connector() {
+  GRPC_CLOSURE_INIT(&connected_, Connected, this, grpc_schedule_on_exec_ctx);
+}
 
-static void chttp2_connector_ref(grpc_connector* con) {
-  chttp2_connector* c = reinterpret_cast<chttp2_connector*>(con);
-  gpr_ref(&c->refs);
+Chttp2Connector::~Chttp2Connector() {
+  if (endpoint_ != nullptr) grpc_endpoint_destroy(endpoint_);
 }
 
-static void chttp2_connector_unref(grpc_connector* con) {
-  chttp2_connector* c = reinterpret_cast<chttp2_connector*>(con);
-  if (gpr_unref(&c->refs)) {
-    gpr_mu_destroy(&c->mu);
-    // If handshaking is not yet in progress, destroy the endpoint.
-    // Otherwise, the handshaker will do this for us.
-    if (c->endpoint != nullptr) grpc_endpoint_destroy(c->endpoint);
-    gpr_free(c);
+void Chttp2Connector::Connect(const Args& args, Result* result,
+                              grpc_closure* notify) {
+  grpc_resolved_address addr;
+  Subchannel::GetAddressFromSubchannelAddressArg(args.channel_args, &addr);
+  grpc_endpoint** ep;
+  {
+    MutexLock lock(&mu_);
+    GPR_ASSERT(notify_ == nullptr);
+    args_ = args;
+    result_ = result;
+    notify_ = notify;
+    GPR_ASSERT(!connecting_);
+    connecting_ = true;
+    GPR_ASSERT(endpoint_ == nullptr);
+    ep = &endpoint_;
   }
+  // In some implementations, the closure can be flushed before
+  // grpc_tcp_client_connect() returns, and since the closure requires access
+  // to mu_, this can result in a deadlock (see
+  // https://github.com/grpc/grpc/issues/16427 for details).
+  // grpc_tcp_client_connect() will fill endpoint_ with proper contents, and we
+  // make sure that we still exist at that point by taking a ref.
+  Ref().release();  // Ref held by callback.
+  grpc_tcp_client_connect(&connected_, ep, args.interested_parties,
+                          args.channel_args, &addr, args.deadline);
 }
 
-static void chttp2_connector_shutdown(grpc_connector* con, grpc_error* why) {
-  chttp2_connector* c = reinterpret_cast<chttp2_connector*>(con);
-  gpr_mu_lock(&c->mu);
-  c->shutdown = true;
-  if (c->handshake_mgr != nullptr) {
-    c->handshake_mgr->Shutdown(GRPC_ERROR_REF(why));
+void Chttp2Connector::Shutdown(grpc_error* error) {
+  MutexLock lock(&mu_);
+  shutdown_ = true;
+  if (handshake_mgr_ != nullptr) {
+    handshake_mgr_->Shutdown(GRPC_ERROR_REF(error));
   }
   // If handshaking is not yet in progress, shutdown the endpoint.
   // Otherwise, the handshaker will do this for us.
-  if (!c->connecting && c->endpoint != nullptr) {
-    grpc_endpoint_shutdown(c->endpoint, GRPC_ERROR_REF(why));
+  if (!connecting_ && endpoint_ != nullptr) {
+    grpc_endpoint_shutdown(endpoint_, GRPC_ERROR_REF(error));
   }
-  gpr_mu_unlock(&c->mu);
-  GRPC_ERROR_UNREF(why);
+  GRPC_ERROR_UNREF(error);
 }
 
-static void on_handshake_done(void* arg, grpc_error* error) {
-  auto* args = static_cast<grpc_core::HandshakerArgs*>(arg);
-  chttp2_connector* c = static_cast<chttp2_connector*>(args->user_data);
-  gpr_mu_lock(&c->mu);
-  if (error != GRPC_ERROR_NONE || c->shutdown) {
-    if (error == GRPC_ERROR_NONE) {
-      error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("connector shutdown");
-      // We were shut down after handshaking completed successfully, so
-      // destroy the endpoint here.
-      // TODO(ctiller): It is currently necessary to shutdown endpoints
-      // before destroying them, even if we know that there are no
-      // pending read/write callbacks.  This should be fixed, at which
-      // point this can be removed.
-      grpc_endpoint_shutdown(args->endpoint, GRPC_ERROR_REF(error));
-      grpc_endpoint_destroy(args->endpoint);
-      grpc_channel_args_destroy(args->args);
-      grpc_slice_buffer_destroy_internal(args->read_buffer);
-      gpr_free(args->read_buffer);
+void Chttp2Connector::Connected(void* arg, grpc_error* error) {
+  Chttp2Connector* self = static_cast<Chttp2Connector*>(arg);
+  bool unref = false;
+  {
+    MutexLock lock(&self->mu_);
+    GPR_ASSERT(self->connecting_);
+    self->connecting_ = false;
+    if (error != GRPC_ERROR_NONE || self->shutdown_) {
+      if (error == GRPC_ERROR_NONE) {
+        error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("connector shutdown");
+      } else {
+        error = GRPC_ERROR_REF(error);
+      }
+      if (self->endpoint_ != nullptr) {
+        grpc_endpoint_shutdown(self->endpoint_, GRPC_ERROR_REF(error));
+      }
+      self->result_->Reset();
+      grpc_closure* notify = self->notify_;
+      self->notify_ = nullptr;
+      ExecCtx::Run(DEBUG_LOCATION, notify, error);
+      unref = true;
     } else {
-      error = GRPC_ERROR_REF(error);
+      GPR_ASSERT(self->endpoint_ != nullptr);
+      self->StartHandshakeLocked();
     }
-    c->result->reset();
-  } else {
-    grpc_endpoint_delete_from_pollset_set(args->endpoint,
-                                          c->args.interested_parties);
-    c->result->transport =
-        grpc_create_chttp2_transport(args->args, args->endpoint, true);
-    c->result->socket =
-        grpc_chttp2_transport_get_socket_node(c->result->transport);
-    GPR_ASSERT(c->result->transport);
-    // TODO(roth): We ideally want to wait until we receive HTTP/2
-    // settings from the server before we consider the connection
-    // established.  If that doesn't happen before the connection
-    // timeout expires, then we should consider the connection attempt a
-    // failure and feed that information back into the backoff code.
-    // We could pass a notify_on_receive_settings callback to
-    // grpc_chttp2_transport_start_reading() to let us know when
-    // settings are received, but we would need to figure out how to use
-    // that information here.
-    //
-    // Unfortunately, we don't currently have a way to split apart the two
-    // effects of scheduling c->notify: we start sending RPCs immediately
-    // (which we want to do) and we consider the connection attempt successful
-    // (which we don't want to do until we get the notify_on_receive_settings
-    // callback from the transport).  If we could split those things
-    // apart, then we could start sending RPCs but then wait for our
-    // timeout before deciding if the connection attempt is successful.
-    // If the attempt is not successful, then we would tear down the
-    // transport and feed the failure back into the backoff code.
-    //
-    // In addition, even if we did that, we would probably not want to do
-    // so until after transparent retries is implemented.  Otherwise, any
-    // RPC that we attempt to send on the connection before the timeout
-    // would fail instead of being retried on a subsequent attempt.
-    grpc_chttp2_transport_start_reading(c->result->transport, args->read_buffer,
-                                        nullptr);
-    c->result->channel_args = args->args;
   }
-  grpc_closure* notify = c->notify;
-  c->notify = nullptr;
-  grpc_core::ExecCtx::Run(DEBUG_LOCATION, notify, error);
-  c->handshake_mgr.reset();
-  gpr_mu_unlock(&c->mu);
-  chttp2_connector_unref(reinterpret_cast<grpc_connector*>(c));
+  if (unref) self->Unref();
 }
 
-static void start_handshake_locked(chttp2_connector* c) {
-  c->handshake_mgr = grpc_core::MakeRefCounted<grpc_core::HandshakeManager>();
-  grpc_core::HandshakerRegistry::AddHandshakers(
-      grpc_core::HANDSHAKER_CLIENT, c->args.channel_args,
-      c->args.interested_parties, c->handshake_mgr.get());
-  grpc_endpoint_add_to_pollset_set(c->endpoint, c->args.interested_parties);
-  c->handshake_mgr->DoHandshake(c->endpoint, c->args.channel_args,
-                                c->args.deadline, nullptr /* acceptor */,
-                                on_handshake_done, c);
-  c->endpoint = nullptr;  // Endpoint handed off to handshake manager.
+void Chttp2Connector::StartHandshakeLocked() {
+  handshake_mgr_ = MakeRefCounted<HandshakeManager>();
+  HandshakerRegistry::AddHandshakers(HANDSHAKER_CLIENT, args_.channel_args,
+                                     args_.interested_parties,
+                                     handshake_mgr_.get());
+  grpc_endpoint_add_to_pollset_set(endpoint_, args_.interested_parties);
+  handshake_mgr_->DoHandshake(endpoint_, args_.channel_args, args_.deadline,
+                              nullptr /* acceptor */, OnHandshakeDone, this);
+  endpoint_ = nullptr;  // Endpoint handed off to handshake manager.
 }
 
-static void connected(void* arg, grpc_error* error) {
-  chttp2_connector* c = static_cast<chttp2_connector*>(arg);
-  gpr_mu_lock(&c->mu);
-  GPR_ASSERT(c->connecting);
-  c->connecting = false;
-  if (error != GRPC_ERROR_NONE || c->shutdown) {
-    if (error == GRPC_ERROR_NONE) {
-      error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("connector shutdown");
+void Chttp2Connector::OnHandshakeDone(void* arg, grpc_error* error) {
+  auto* args = static_cast<HandshakerArgs*>(arg);
+  Chttp2Connector* self = static_cast<Chttp2Connector*>(args->user_data);
+  {
+    MutexLock lock(&self->mu_);
+    if (error != GRPC_ERROR_NONE || self->shutdown_) {
+      if (error == GRPC_ERROR_NONE) {
+        error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("connector shutdown");
+        // We were shut down after handshaking completed successfully, so
+        // destroy the endpoint here.
+        // TODO(ctiller): It is currently necessary to shutdown endpoints
+        // before destroying them, even if we know that there are no
+        // pending read/write callbacks.  This should be fixed, at which
+        // point this can be removed.
+        grpc_endpoint_shutdown(args->endpoint, GRPC_ERROR_REF(error));
+        grpc_endpoint_destroy(args->endpoint);
+        grpc_channel_args_destroy(args->args);
+        grpc_slice_buffer_destroy_internal(args->read_buffer);
+        gpr_free(args->read_buffer);
+      } else {
+        error = GRPC_ERROR_REF(error);
+      }
+      self->result_->Reset();
     } else {
-      error = GRPC_ERROR_REF(error);
+      grpc_endpoint_delete_from_pollset_set(args->endpoint,
+                                            self->args_.interested_parties);
+      self->result_->transport =
+          grpc_create_chttp2_transport(args->args, args->endpoint, true);
+      self->result_->socket_node =
+          grpc_chttp2_transport_get_socket_node(self->result_->transport);
+      GPR_ASSERT(self->result_->transport != nullptr);
+      // TODO(roth): We ideally want to wait until we receive HTTP/2
+      // settings from the server before we consider the connection
+      // established.  If that doesn't happen before the connection
+      // timeout expires, then we should consider the connection attempt a
+      // failure and feed that information back into the backoff code.
+      // We could pass a notify_on_receive_settings callback to
+      // grpc_chttp2_transport_start_reading() to let us know when
+      // settings are received, but we would need to figure out how to use
+      // that information here.
+      //
+      // Unfortunately, we don't currently have a way to split apart the two
+      // effects of scheduling c->notify: we start sending RPCs immediately
+      // (which we want to do) and we consider the connection attempt successful
+      // (which we don't want to do until we get the notify_on_receive_settings
+      // callback from the transport).  If we could split those things
+      // apart, then we could start sending RPCs but then wait for our
+      // timeout before deciding if the connection attempt is successful.
+      // If the attempt is not successful, then we would tear down the
+      // transport and feed the failure back into the backoff code.
+      //
+      // In addition, even if we did that, we would probably not want to do
+      // so until after transparent retries is implemented.  Otherwise, any
+      // RPC that we attempt to send on the connection before the timeout
+      // would fail instead of being retried on a subsequent attempt.
+      grpc_chttp2_transport_start_reading(self->result_->transport,
+                                          args->read_buffer, nullptr);
+      self->result_->channel_args = args->args;
     }
-    c->result->reset();
-    grpc_closure* notify = c->notify;
-    c->notify = nullptr;
-    grpc_core::ExecCtx::Run(DEBUG_LOCATION, notify, error);
-    if (c->endpoint != nullptr) {
-      grpc_endpoint_shutdown(c->endpoint, GRPC_ERROR_REF(error));
-    }
-    gpr_mu_unlock(&c->mu);
-    chttp2_connector_unref(static_cast<grpc_connector*>(arg));
-  } else {
-    GPR_ASSERT(c->endpoint != nullptr);
-    start_handshake_locked(c);
-    gpr_mu_unlock(&c->mu);
+    grpc_closure* notify = self->notify_;
+    self->notify_ = nullptr;
+    ExecCtx::Run(DEBUG_LOCATION, notify, error);
+    self->handshake_mgr_.reset();
   }
+  self->Unref();
 }
 
-static void chttp2_connector_connect(grpc_connector* con,
-                                     const grpc_connect_in_args* args,
-                                     grpc_connect_out_args* result,
-                                     grpc_closure* notify) {
-  chttp2_connector* c = reinterpret_cast<chttp2_connector*>(con);
-  grpc_resolved_address addr;
-  grpc_core::Subchannel::GetAddressFromSubchannelAddressArg(args->channel_args,
-                                                            &addr);
-  gpr_mu_lock(&c->mu);
-  GPR_ASSERT(c->notify == nullptr);
-  c->notify = notify;
-  c->args = *args;
-  c->result = result;
-  GPR_ASSERT(c->endpoint == nullptr);
-  chttp2_connector_ref(con);  // Ref taken for callback.
-  GRPC_CLOSURE_INIT(&c->connected, connected, c, grpc_schedule_on_exec_ctx);
-  GPR_ASSERT(!c->connecting);
-  c->connecting = true;
-  grpc_closure* closure = &c->connected;
-  grpc_endpoint** ep = &c->endpoint;
-  gpr_mu_unlock(&c->mu);
-  // In some implementations, the closure can be flushed before
-  // grpc_tcp_client_connect and since the closure requires access to c->mu,
-  // this can result in a deadlock. Refer
-  // https://github.com/grpc/grpc/issues/16427
-  // grpc_tcp_client_connect would fill c->endpoint with proper contents and we
-  // make sure that we would still exist at that point by taking a ref.
-  grpc_tcp_client_connect(closure, ep, args->interested_parties,
-                          args->channel_args, &addr, args->deadline);
-}
-
-static const grpc_connector_vtable chttp2_connector_vtable = {
-    chttp2_connector_ref, chttp2_connector_unref, chttp2_connector_shutdown,
-    chttp2_connector_connect};
-
-grpc_connector* grpc_chttp2_connector_create() {
-  chttp2_connector* c = static_cast<chttp2_connector*>(gpr_zalloc(sizeof(*c)));
-  c->base.vtable = &chttp2_connector_vtable;
-  gpr_mu_init(&c->mu);
-  gpr_ref_init(&c->refs, 1);
-  return &c->base;
-}
+}  // namespace grpc_core
diff --git a/src/core/ext/transport/chttp2/client/chttp2_connector.h b/src/core/ext/transport/chttp2/client/chttp2_connector.h
index 04da4413095..1ecd172bba1 100644
--- a/src/core/ext/transport/chttp2/client/chttp2_connector.h
+++ b/src/core/ext/transport/chttp2/client/chttp2_connector.h
@@ -22,7 +22,37 @@
 #include <grpc/support/port_platform.h>
 
 #include "src/core/ext/filters/client_channel/connector.h"
+#include "src/core/lib/channel/handshaker.h"
+#include "src/core/lib/channel/handshaker_registry.h"
 
-grpc_connector* grpc_chttp2_connector_create();
+namespace grpc_core {
+
+class Chttp2Connector : public SubchannelConnector {
+ public:
+  Chttp2Connector();
+  ~Chttp2Connector();
+
+  void Connect(const Args& args, Result* result, grpc_closure* notify) override;
+  void Shutdown(grpc_error* error) override;
+
+ private:
+  static void Connected(void* arg, grpc_error* error);
+  void StartHandshakeLocked();
+  static void OnHandshakeDone(void* arg, grpc_error* error);
+
+  Mutex mu_;
+  Args args_;
+  Result* result_ = nullptr;
+  grpc_closure* notify_ = nullptr;
+  bool shutdown_ = false;
+  bool connecting_ = false;
+  // Holds the endpoint when first created before being handed off to
+  // the handshake manager.
+  grpc_endpoint* endpoint_ = nullptr;
+  grpc_closure connected_;
+  RefCountedPtr<HandshakeManager> handshake_mgr_;
+};
+
+}  // namespace grpc_core
 
 #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_CLIENT_CHTTP2_CONNECTOR_H */
diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.cc b/src/core/ext/transport/chttp2/client/insecure/channel_create.cc
index bd7bad72f0a..189ed7d1ce5 100644
--- a/src/core/ext/transport/chttp2/client/insecure/channel_create.cc
+++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.cc
@@ -40,9 +40,8 @@ class Chttp2InsecureClientChannelFactory : public ClientChannelFactory {
   Subchannel* CreateSubchannel(const grpc_channel_args* args) override {
     grpc_channel_args* new_args =
         grpc_default_authority_add_if_not_present(args);
-    grpc_connector* connector = grpc_chttp2_connector_create();
-    Subchannel* s = Subchannel::Create(connector, new_args);
-    grpc_connector_unref(connector);
+    Subchannel* s =
+        Subchannel::Create(MakeOrphanable<Chttp2Connector>(), new_args);
     grpc_channel_args_destroy(new_args);
     return s;
   }
diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc
index 91f7c13a335..33aa0cd0808 100644
--- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc
+++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc
@@ -51,9 +51,8 @@ class Chttp2SecureClientChannelFactory : public ClientChannelFactory {
               "Failed to create channel args during subchannel creation.");
       return nullptr;
     }
-    grpc_connector* connector = grpc_chttp2_connector_create();
-    Subchannel* s = Subchannel::Create(connector, new_args);
-    grpc_connector_unref(connector);
+    Subchannel* s =
+        Subchannel::Create(MakeOrphanable<Chttp2Connector>(), new_args);
     grpc_channel_args_destroy(new_args);
     return s;
   }
diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py
index 556912a7708..103bcf83cb6 100644
--- a/src/python/grpcio/grpc_core_dependencies.py
+++ b/src/python/grpcio/grpc_core_dependencies.py
@@ -24,7 +24,6 @@ CORE_SOURCE_FILES = [
     'src/core/ext/filters/client_channel/client_channel_channelz.cc',
     'src/core/ext/filters/client_channel/client_channel_factory.cc',
     'src/core/ext/filters/client_channel/client_channel_plugin.cc',
-    'src/core/ext/filters/client_channel/connector.cc',
     'src/core/ext/filters/client_channel/global_subchannel_pool.cc',
     'src/core/ext/filters/client_channel/health/health_check_client.cc',
     'src/core/ext/filters/client_channel/http_connect_handshaker.cc',
diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal
index 1d69aa232d8..9a75e179261 100644
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@@ -1074,7 +1074,6 @@ src/core/ext/filters/client_channel/client_channel_channelz.h \
 src/core/ext/filters/client_channel/client_channel_factory.cc \
 src/core/ext/filters/client_channel/client_channel_factory.h \
 src/core/ext/filters/client_channel/client_channel_plugin.cc \
-src/core/ext/filters/client_channel/connector.cc \
 src/core/ext/filters/client_channel/connector.h \
 src/core/ext/filters/client_channel/global_subchannel_pool.cc \
 src/core/ext/filters/client_channel/global_subchannel_pool.h \
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index 1b0ff5c5698..bec0ab35f8e 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -888,7 +888,6 @@ src/core/ext/filters/client_channel/client_channel_channelz.h \
 src/core/ext/filters/client_channel/client_channel_factory.cc \
 src/core/ext/filters/client_channel/client_channel_factory.h \
 src/core/ext/filters/client_channel/client_channel_plugin.cc \
-src/core/ext/filters/client_channel/connector.cc \
 src/core/ext/filters/client_channel/connector.h \
 src/core/ext/filters/client_channel/global_subchannel_pool.cc \
 src/core/ext/filters/client_channel/global_subchannel_pool.h \

From 315147dcae03fde6915a6f713e68f5b45dc27128 Mon Sep 17 00:00:00 2001
From: Lidi Zheng <lidiz@google.com>
Date: Thu, 21 Nov 2019 16:36:52 -0800
Subject: [PATCH 14/26] Add Python3.8 to our macOS CI

---
 tools/internal_ci/helper_scripts/prepare_build_macos_rc | 4 ++++
 tools/run_tests/artifacts/artifact_targets.py           | 3 +--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
index 413ed49adc3..43a34d51c41 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
@@ -58,6 +58,10 @@ export PYTHONPATH=/Library/Python/3.4/site-packages
 time curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg
 time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target /
 
+# Install Python 3.8
+time curl -O https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg
+time sudo installer -pkg ./python-3.8.0-macosx10.9.pkg -target /
+
 # set xcode version for Obj-C tests
 sudo xcode-select -switch /Applications/Xcode_9.2.app/Contents/Developer/
 
diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py
index 62dd3e12f8f..0709db94696 100644
--- a/tools/run_tests/artifacts/artifact_targets.py
+++ b/tools/run_tests/artifacts/artifact_targets.py
@@ -406,8 +406,7 @@ def targets():
         PythonArtifact('macos', 'x64', 'python3.5'),
         PythonArtifact('macos', 'x64', 'python3.6'),
         PythonArtifact('macos', 'x64', 'python3.7'),
-        # TODO(https://github.com/grpc/grpc/issues/20615) Enable this artifact
-        # PythonArtifact('macos', 'x64', 'python3.8'),
+        PythonArtifact('macos', 'x64', 'python3.8'),
         PythonArtifact('windows', 'x86', 'Python27_32bits'),
         PythonArtifact('windows', 'x86', 'Python35_32bits'),
         PythonArtifact('windows', 'x86', 'Python36_32bits'),

From 3d3b6d8a8aaecb207620e9051cacf4387add8b6e Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Fri, 22 Nov 2019 10:36:10 -0800
Subject: [PATCH 15/26] shard xds_end2end_test

---
 bazel/grpc_build_system.bzl | 3 ++-
 test/cpp/end2end/BUILD      | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
index 3522728eca3..033a0051cce 100644
--- a/bazel/grpc_build_system.bzl
+++ b/bazel/grpc_build_system.bzl
@@ -189,7 +189,7 @@ def ios_cc_test(
             deps = ios_test_deps,
         )
 
-def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data = [], uses_polling = True, language = "C++", size = "medium", timeout = None, tags = [], exec_compatible_with = [], exec_properties = {}):
+def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data = [], uses_polling = True, language = "C++", size = "medium", timeout = None, tags = [], exec_compatible_with = [], exec_properties = {}, shard_count = None):
     copts = if_mac(["-DGRPC_CFSTREAM"])
     if language.upper() == "C":
         copts = copts + if_not_windows(["-std=c99"])
@@ -204,6 +204,7 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data
         "timeout": timeout,
         "exec_compatible_with": exec_compatible_with,
         "exec_properties": exec_properties,
+        "shard_count": shard_count,
     }
     if uses_polling:
         # the vanilla version of the test should run on platforms that only
diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD
index 962f91b8119..c0178b1fafc 100644
--- a/test/cpp/end2end/BUILD
+++ b/test/cpp/end2end/BUILD
@@ -518,6 +518,7 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
+    shard_count = 10,
 )
 
 grpc_cc_test(

From 63363aaf187539e3311590ff137446f2ccc6115c Mon Sep 17 00:00:00 2001
From: Lidi Zheng <lidiz@google.com>
Date: Fri, 22 Nov 2019 13:52:46 -0800
Subject: [PATCH 16/26] Install setup essentials for Python3.8

---
 tools/internal_ci/macos/grpc_build_artifacts.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh
index 7b9c5eb673d..f4a7708e71e 100755
--- a/tools/internal_ci/macos/grpc_build_artifacts.sh
+++ b/tools/internal_ci/macos/grpc_build_artifacts.sh
@@ -25,6 +25,7 @@ python2.7 -m pip install cython setuptools wheel
 python3.5 -m pip install cython setuptools wheel
 python3.6 -m pip install cython setuptools wheel
 python3.7 -m pip install cython setuptools wheel
+python3.8 -m pip install cython setuptools wheel
 
 # needed to build ruby artifacts
 time bash tools/distrib/build_ruby_environment_macos.sh

From c711e5d4a6252a329e0b407adbf4155ae4c0e044 Mon Sep 17 00:00:00 2001
From: Lidi Zheng <lidiz@google.com>
Date: Mon, 25 Nov 2019 10:59:51 -0800
Subject: [PATCH 17/26] Validate checksum for Python installers

---
 tools/internal_ci/helper_scripts/prepare_build_macos_rc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
index 43a34d51c41..6747508b036 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 # Source this rc script to prepare the environment for macos builds
 
 # show original open file limit values
@@ -56,10 +55,14 @@ export PYTHONPATH=/Library/Python/3.4/site-packages
 
 # Install Python 3.7
 time curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg
+download_hash=$(md5sum python-3.7.0-macosx10.9.pkg | awk '{print $1}')
+if [[ download_hash != 'ae0717a02efea3b0eb34aadc680dc498' ]]; then echo "Incorrect hash!"; exit 1; fi
 time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target /
 
 # Install Python 3.8
 time curl -O https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg
+download_hash=$(md5sum python-3.8.0-macosx10.9.pkg | awk '{print $1}')
+if [[ download_hash != 'f5f9ae9f416170c6355cab7256bb75b5' ]]; then echo "Incorrect hash!"; exit 1; fi
 time sudo installer -pkg ./python-3.8.0-macosx10.9.pkg -target /
 
 # set xcode version for Obj-C tests
@@ -85,3 +88,4 @@ ln -s /tmpfs/DerivedData ~/Library/Developer/Xcode/DerivedData
 
 # PHP tests currently require using an older version of PHPUnit
 ln -sf /usr/local/bin/phpunit-5.7 /usr/local/bin/phpunit
+

From f0623730ef767a9612a8e0a965aebed650648c05 Mon Sep 17 00:00:00 2001
From: Lidi Zheng <lidiz@google.com>
Date: Mon, 25 Nov 2019 13:24:26 -0800
Subject: [PATCH 18/26] Validate md5sum v2

---
 .../helper_scripts/prepare_build_macos_rc         | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
index 6747508b036..1b856f78b03 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -ex
 # Copyright 2017 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -53,16 +53,13 @@ time pip install --user virtualenv
 time pip install --user --upgrade Mako six tox setuptools twisted pyyaml pyjwt cryptography requests
 export PYTHONPATH=/Library/Python/3.4/site-packages
 
-# Install Python 3.7
+# Install Python 3.7 and Python 3.8
 time curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg
-download_hash=$(md5sum python-3.7.0-macosx10.9.pkg | awk '{print $1}')
-if [[ download_hash != 'ae0717a02efea3b0eb34aadc680dc498' ]]; then echo "Incorrect hash!"; exit 1; fi
-time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target /
-
-# Install Python 3.8
 time curl -O https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg
-download_hash=$(md5sum python-3.8.0-macosx10.9.pkg | awk '{print $1}')
-if [[ download_hash != 'f5f9ae9f416170c6355cab7256bb75b5' ]]; then echo "Incorrect hash!"; exit 1; fi
+echo "ae0717a02efea3b0eb34aadc680dc498 python-3.7.0-macosx10.9.pkg" > python_intaller_checksum.md5
+echo "f5f9ae9f416170c6355cab7256bb75b5 python-3.8.0-macosx10.9.pkg" >> python_intaller_checksum.md5
+md5sum -c python_installer_checksum.md5
+time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target /
 time sudo installer -pkg ./python-3.8.0-macosx10.9.pkg -target /
 
 # set xcode version for Obj-C tests

From 606516346605aa7bb0d811450fa3e3835f420024 Mon Sep 17 00:00:00 2001
From: Lidi Zheng <lidiz@google.com>
Date: Mon, 25 Nov 2019 14:27:27 -0800
Subject: [PATCH 19/26] Fix typo

---
 tools/internal_ci/helper_scripts/prepare_build_macos_rc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
index 1b856f78b03..a13c29ea7f5 100644
--- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
@@ -56,9 +56,9 @@ export PYTHONPATH=/Library/Python/3.4/site-packages
 # Install Python 3.7 and Python 3.8
 time curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg
 time curl -O https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg
-echo "ae0717a02efea3b0eb34aadc680dc498 python-3.7.0-macosx10.9.pkg" > python_intaller_checksum.md5
-echo "f5f9ae9f416170c6355cab7256bb75b5 python-3.8.0-macosx10.9.pkg" >> python_intaller_checksum.md5
-md5sum -c python_installer_checksum.md5
+echo "ae0717a02efea3b0eb34aadc680dc498 python-3.7.0-macosx10.9.pkg" > /tmp/python_installer_checksum.md5
+echo "f5f9ae9f416170c6355cab7256bb75b5 python-3.8.0-macosx10.9.pkg" >> /tmp/python_installer_checksum.md5
+md5sum -c /tmp/python_installer_checksum.md5
 time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target /
 time sudo installer -pkg ./python-3.8.0-macosx10.9.pkg -target /
 

From fe9576c86129632051bb92ebb99a7ed294a586ba Mon Sep 17 00:00:00 2001
From: Zhen Lian <zhenlian@google.com>
Date: Mon, 11 Nov 2019 18:51:51 -0800
Subject: [PATCH 20/26] [Exposing ALTS Context 2/2] Provide a Utility Function
 to Get Context

---
 BUILD                                |  16 +++
 CMakeLists.txt                       | 119 ++++++++++++++++++
 Makefile                             | 174 ++++++++++++++++++++++++++-
 build.yaml                           |  26 ++++
 grpc.gyp                             |  12 ++
 include/grpcpp/alts_context.h        |  39 ++++++
 src/cpp/common/alts_context.cc       |  44 +++++++
 src/proto/grpc/gcp/BUILD             |   8 ++
 test/cpp/common/BUILD                |  13 ++
 test/cpp/common/alts_context_test.cc |  96 +++++++++++++++
 tools/run_tests/generated/tests.json |  24 ++++
 11 files changed, 569 insertions(+), 2 deletions(-)
 create mode 100644 include/grpcpp/alts_context.h
 create mode 100644 src/cpp/common/alts_context.cc
 create mode 100644 test/cpp/common/alts_context_test.cc

diff --git a/BUILD b/BUILD
index 54d653f98cb..1f2a4e6b08d 100644
--- a/BUILD
+++ b/BUILD
@@ -428,6 +428,22 @@ grpc_cc_library(
     ],
 )
 
+grpc_cc_library(
+    name = "grpc++_alts",
+    srcs = [
+        "src/cpp/common/alts_context.cc",
+    ],
+    hdrs = [
+        "include/grpcpp/alts_context.h",
+    ],
+    language = "c++",
+    standalone = True,
+    deps = [
+        "grpc++",
+        "//src/proto/grpc/gcp:alts_handshaker_cc_proto",
+    ],
+)
+
 grpc_cc_library(
     name = "grpc_csharp_ext",
     srcs = [
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0b242b8121a..a1340f51cf9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -286,6 +286,12 @@ protobuf_generate_grpc_cpp(
 protobuf_generate_grpc_cpp(
   src/proto/grpc/core/stats.proto
 )
+protobuf_generate_grpc_cpp(
+  src/proto/grpc/gcp/altscontext.proto
+)
+protobuf_generate_grpc_cpp(
+  src/proto/grpc/gcp/transport_security_common.proto
+)
 protobuf_generate_grpc_cpp(
   src/proto/grpc/health/v1/health.proto
 )
@@ -629,6 +635,7 @@ if(gRPC_BUILD_TESTS)
   if(_gRPC_PLATFORM_LINUX)
     add_dependencies(buildtests_cxx alts_concurrent_connectivity_test)
   endif()
+  add_dependencies(buildtests_cxx alts_context_test)
   add_dependencies(buildtests_cxx alts_counter_test)
   add_dependencies(buildtests_cxx alts_crypt_test)
   add_dependencies(buildtests_cxx alts_crypter_test)
@@ -3825,6 +3832,78 @@ if(gRPC_INSTALL)
   )
 endif()
 
+
+if(gRPC_BUILD_CODEGEN)
+add_library(grpc++_alts
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/gcp/altscontext.pb.cc
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/gcp/altscontext.grpc.pb.cc
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/gcp/altscontext.pb.h
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/gcp/altscontext.grpc.pb.h
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/gcp/transport_security_common.pb.cc
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/gcp/transport_security_common.grpc.pb.cc
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/gcp/transport_security_common.pb.h
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/gcp/transport_security_common.grpc.pb.h
+  src/cpp/common/alts_context.cc
+)
+
+set_target_properties(grpc++_alts PROPERTIES
+  VERSION ${gRPC_CPP_VERSION}
+  SOVERSION ${gRPC_CPP_SOVERSION}
+)
+
+if(WIN32 AND MSVC)
+  set_target_properties(grpc++_alts PROPERTIES COMPILE_PDB_NAME "grpc++_alts"
+    COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}"
+  )
+  if(gRPC_INSTALL)
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++_alts.pdb
+      DESTINATION ${gRPC_INSTALL_LIBDIR} OPTIONAL
+    )
+  endif()
+endif()
+
+target_include_directories(grpc++_alts
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
+    ${_gRPC_SSL_INCLUDE_DIR}
+    ${_gRPC_UPB_GENERATED_DIR}
+    ${_gRPC_UPB_GRPC_GENERATED_DIR}
+    ${_gRPC_UPB_INCLUDE_DIR}
+    ${_gRPC_ZLIB_INCLUDE_DIR}
+    ${_gRPC_PROTO_GENS_DIR}
+)
+target_link_libraries(grpc++_alts
+  ${_gRPC_BASELIB_LIBRARIES}
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++
+)
+
+foreach(_hdr
+  include/grpcpp/alts_context.h
+  include/grpcpp/impl/codegen/security/auth_context.h
+)
+  string(REPLACE "include/" "" _path ${_hdr})
+  get_filename_component(_path ${_path} PATH)
+  install(FILES ${_hdr}
+    DESTINATION "${gRPC_INSTALL_INCLUDEDIR}/${_path}"
+  )
+endforeach()
+endif()
+
+if(gRPC_BUILD_CODEGEN)
+
+if(gRPC_INSTALL)
+  install(TARGETS grpc++_alts EXPORT gRPCTargets
+    RUNTIME DESTINATION ${gRPC_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${gRPC_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${gRPC_INSTALL_LIBDIR}
+  )
+endif()
+
+endif()
 if(gRPC_BUILD_TESTS)
 
 if(gRPC_BUILD_CODEGEN)
@@ -9930,6 +10009,46 @@ endif()
 endif()
 if(gRPC_BUILD_TESTS)
 
+add_executable(alts_context_test
+  test/cpp/common/alts_context_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+target_include_directories(alts_context_test
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
+    ${_gRPC_SSL_INCLUDE_DIR}
+    ${_gRPC_UPB_GENERATED_DIR}
+    ${_gRPC_UPB_GRPC_GENERATED_DIR}
+    ${_gRPC_UPB_INCLUDE_DIR}
+    ${_gRPC_ZLIB_INCLUDE_DIR}
+    third_party/googletest/googletest/include
+    third_party/googletest/googletest
+    third_party/googletest/googlemock/include
+    third_party/googletest/googlemock
+    ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(alts_context_test
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_test_util
+  grpc_test_util
+  grpc++_alts
+  grpc++
+  grpc
+  gpr
+  grpc++_test_config
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+
+endif()
+if(gRPC_BUILD_TESTS)
+
 add_executable(alts_counter_test
   test/core/tsi/alts/frame_protector/alts_counter_test.cc
   third_party/googletest/googletest/src/gtest-all.cc
diff --git a/Makefile b/Makefile
index 0c4e5920ff2..1392a7bd17e 100644
--- a/Makefile
+++ b/Makefile
@@ -1154,6 +1154,7 @@ uri_fuzzer_test: $(BINDIR)/$(CONFIG)/uri_fuzzer_test
 uri_parser_test: $(BINDIR)/$(CONFIG)/uri_parser_test
 alarm_test: $(BINDIR)/$(CONFIG)/alarm_test
 alts_concurrent_connectivity_test: $(BINDIR)/$(CONFIG)/alts_concurrent_connectivity_test
+alts_context_test: $(BINDIR)/$(CONFIG)/alts_context_test
 alts_counter_test: $(BINDIR)/$(CONFIG)/alts_counter_test
 alts_crypt_test: $(BINDIR)/$(CONFIG)/alts_crypt_test
 alts_crypter_test: $(BINDIR)/$(CONFIG)/alts_crypter_test
@@ -1412,14 +1413,14 @@ static: static_c static_cxx
 
 static_c: pc_c pc_c_unsecure cache.mk  $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgrpc_cronet.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libupb.a
 
-static_cxx: pc_cxx pc_cxx_unsecure cache.mk  $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc++_error_details.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a
+static_cxx: pc_cxx pc_cxx_unsecure cache.mk  $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc++_alts.a $(LIBDIR)/$(CONFIG)/libgrpc++_error_details.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a
 
 static_csharp: static_c  $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a
 
 shared: shared_c shared_cxx
 
 shared_c: pc_c pc_c_unsecure cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)
-shared_cxx: pc_cxx pc_cxx_unsecure cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)
+shared_cxx: pc_cxx pc_cxx_unsecure cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)
 
 shared_csharp: shared_c  $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CSHARP).$(SHARED_EXT_CSHARP)
 grpc_csharp_ext: shared_csharp
@@ -1634,6 +1635,7 @@ ifeq ($(EMBED_OPENSSL),true)
 buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/alarm_test \
   $(BINDIR)/$(CONFIG)/alts_concurrent_connectivity_test \
+  $(BINDIR)/$(CONFIG)/alts_context_test \
   $(BINDIR)/$(CONFIG)/alts_counter_test \
   $(BINDIR)/$(CONFIG)/alts_crypt_test \
   $(BINDIR)/$(CONFIG)/alts_crypter_test \
@@ -1806,6 +1808,7 @@ else
 buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/alarm_test \
   $(BINDIR)/$(CONFIG)/alts_concurrent_connectivity_test \
+  $(BINDIR)/$(CONFIG)/alts_context_test \
   $(BINDIR)/$(CONFIG)/alts_counter_test \
   $(BINDIR)/$(CONFIG)/alts_crypt_test \
   $(BINDIR)/$(CONFIG)/alts_crypter_test \
@@ -2244,6 +2247,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/alarm_test || ( echo test alarm_test failed ; exit 1 )
 	$(E) "[RUN]     Testing alts_concurrent_connectivity_test"
 	$(Q) $(BINDIR)/$(CONFIG)/alts_concurrent_connectivity_test || ( echo test alts_concurrent_connectivity_test failed ; exit 1 )
+	$(E) "[RUN]     Testing alts_context_test"
+	$(Q) $(BINDIR)/$(CONFIG)/alts_context_test || ( echo test alts_context_test failed ; exit 1 )
 	$(E) "[RUN]     Testing alts_counter_test"
 	$(Q) $(BINDIR)/$(CONFIG)/alts_counter_test || ( echo test alts_counter_test failed ; exit 1 )
 	$(E) "[RUN]     Testing alts_crypt_test"
@@ -2590,6 +2595,8 @@ strip-static_cxx: static_cxx
 ifeq ($(CONFIG),opt)
 	$(E) "[STRIP]   Stripping libgrpc++.a"
 	$(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libgrpc++.a
+	$(E) "[STRIP]   Stripping libgrpc++_alts.a"
+	$(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libgrpc++_alts.a
 	$(E) "[STRIP]   Stripping libgrpc++_error_details.a"
 	$(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libgrpc++_error_details.a
 	$(E) "[STRIP]   Stripping libgrpc++_reflection.a"
@@ -2620,6 +2627,8 @@ strip-shared_cxx: shared_cxx
 ifeq ($(CONFIG),opt)
 	$(E) "[STRIP]   Stripping $(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)"
 	$(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)
+	$(E) "[STRIP]   Stripping $(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)"
+	$(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)
 	$(E) "[STRIP]   Stripping $(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)"
 	$(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)
 	$(E) "[STRIP]   Stripping $(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)"
@@ -2697,6 +2706,38 @@ $(GENDIR)/src/proto/grpc/core/stats.grpc.pb.cc: src/proto/grpc/core/stats.proto
 	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin$(EXECUTABLE_SUFFIX) $<
 endif
 
+ifeq ($(NO_PROTOC),true)
+$(GENDIR)/src/proto/grpc/gcp/altscontext.pb.cc: protoc_dep_error
+$(GENDIR)/src/proto/grpc/gcp/altscontext.grpc.pb.cc: protoc_dep_error
+else
+
+$(GENDIR)/src/proto/grpc/gcp/altscontext.pb.cc: src/proto/grpc/gcp/altscontext.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/gcp/transport_security_common.pb.cc
+	$(E) "[PROTOC]  Generating protobuf CC file from $<"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --cpp_out=$(GENDIR) $<
+
+$(GENDIR)/src/proto/grpc/gcp/altscontext.grpc.pb.cc: src/proto/grpc/gcp/altscontext.proto $(GENDIR)/src/proto/grpc/gcp/altscontext.pb.cc $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/gcp/transport_security_common.pb.cc $(GENDIR)/src/proto/grpc/gcp/transport_security_common.grpc.pb.cc
+	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin$(EXECUTABLE_SUFFIX) $<
+endif
+
+ifeq ($(NO_PROTOC),true)
+$(GENDIR)/src/proto/grpc/gcp/transport_security_common.pb.cc: protoc_dep_error
+$(GENDIR)/src/proto/grpc/gcp/transport_security_common.grpc.pb.cc: protoc_dep_error
+else
+
+$(GENDIR)/src/proto/grpc/gcp/transport_security_common.pb.cc: src/proto/grpc/gcp/transport_security_common.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
+	$(E) "[PROTOC]  Generating protobuf CC file from $<"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --cpp_out=$(GENDIR) $<
+
+$(GENDIR)/src/proto/grpc/gcp/transport_security_common.grpc.pb.cc: src/proto/grpc/gcp/transport_security_common.proto $(GENDIR)/src/proto/grpc/gcp/transport_security_common.pb.cc $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
+	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin$(EXECUTABLE_SUFFIX) $<
+endif
+
 ifeq ($(NO_PROTOC),true)
 $(GENDIR)/src/proto/grpc/health/v1/health.pb.cc: protoc_dep_error
 $(GENDIR)/src/proto/grpc/health/v1/health.grpc.pb.cc: protoc_dep_error
@@ -3195,6 +3236,9 @@ install-static_cxx: static_cxx strip-static_cxx install-pkg-config_cxx
 	$(E) "[INSTALL] Installing libgrpc++.a"
 	$(Q) $(INSTALL) -d $(prefix)/lib
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++.a $(prefix)/lib/libgrpc++.a
+	$(E) "[INSTALL] Installing libgrpc++_alts.a"
+	$(Q) $(INSTALL) -d $(prefix)/lib
+	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_alts.a $(prefix)/lib/libgrpc++_alts.a
 	$(E) "[INSTALL] Installing libgrpc++_error_details.a"
 	$(Q) $(INSTALL) -d $(prefix)/lib
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_error_details.a $(prefix)/lib/libgrpc++_error_details.a
@@ -3281,6 +3325,15 @@ ifeq ($(SYSTEM),MINGW32)
 else ifneq ($(SYSTEM),Darwin)
 	$(Q) ln -sf $(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++.so.1
 	$(Q) ln -sf $(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++.so
+endif
+	$(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)"
+	$(Q) $(INSTALL) -d $(prefix)/lib
+	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/$(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)
+ifeq ($(SYSTEM),MINGW32)
+	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_alts$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpc++_alts.a
+else ifneq ($(SYSTEM),Darwin)
+	$(Q) ln -sf $(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_alts.so.1
+	$(Q) ln -sf $(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_alts.so
 endif
 	$(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)"
 	$(Q) $(INSTALL) -d $(prefix)/lib
@@ -6225,6 +6278,79 @@ endif
 endif
 
 
+LIBGRPC++_ALTS_SRC = \
+    $(GENDIR)/src/proto/grpc/gcp/altscontext.pb.cc $(GENDIR)/src/proto/grpc/gcp/altscontext.grpc.pb.cc \
+    $(GENDIR)/src/proto/grpc/gcp/transport_security_common.pb.cc $(GENDIR)/src/proto/grpc/gcp/transport_security_common.grpc.pb.cc \
+    src/cpp/common/alts_context.cc \
+
+PUBLIC_HEADERS_CXX += \
+    include/grpcpp/alts_context.h \
+    include/grpcpp/impl/codegen/security/auth_context.h \
+
+LIBGRPC++_ALTS_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_ALTS_SRC))))
+
+
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure libraries if you don't have OpenSSL.
+
+$(LIBDIR)/$(CONFIG)/libgrpc++_alts.a: openssl_dep_error
+
+$(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP): openssl_dep_error
+
+else
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay.
+
+$(LIBDIR)/$(CONFIG)/libgrpc++_alts.a: protobuf_dep_error
+
+$(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP): protobuf_dep_error
+
+else
+
+$(LIBDIR)/$(CONFIG)/libgrpc++_alts.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP)  $(PROTOBUF_DEP) $(LIBGRPC++_ALTS_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS)  $(CARES_MERGE_OBJS)  $(ADDRESS_SORTING_MERGE_OBJS)  $(UPB_MERGE_OBJS) 
+	$(E) "[AR]      Creating $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_alts.a
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc++_alts.a $(LIBGRPC++_ALTS_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS)  $(CARES_MERGE_OBJS)  $(ADDRESS_SORTING_MERGE_OBJS)  $(UPB_MERGE_OBJS) 
+ifeq ($(SYSTEM),Darwin)
+	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc++_alts.a
+endif
+
+
+
+ifeq ($(SYSTEM),MINGW32)
+$(LIBDIR)/$(CONFIG)/grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP): $(LIBGRPC++_ALTS_OBJS)  $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(PROTOBUF_DEP) $(LIBDIR)/$(CONFIG)/grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(OPENSSL_DEP)
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/grpc++_alts$(SHARED_VERSION_CPP).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libgrpc++_alts$(SHARED_VERSION_CPP)-dll.a -o $(LIBDIR)/$(CONFIG)/grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBGRPC++_ALTS_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) -lgrpc++$(SHARED_VERSION_CPP)-dll
+else
+$(LIBDIR)/$(CONFIG)/libgrpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP): $(LIBGRPC++_ALTS_OBJS)  $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(UPB_DEP) $(PROTOBUF_DEP) $(LIBDIR)/$(CONFIG)/libgrpc++.$(SHARED_EXT_CPP) $(OPENSSL_DEP)
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+ifeq ($(SYSTEM),Darwin)
+	$(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBGRPC++_ALTS_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) -lgrpc++
+else
+	$(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc++_alts.so.1 -o $(LIBDIR)/$(CONFIG)/libgrpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBGRPC++_ALTS_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) -lgrpc++
+	$(Q) ln -sf $(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/libgrpc++_alts$(SHARED_VERSION_CPP).so.1
+	$(Q) ln -sf $(SHARED_PREFIX)grpc++_alts$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/libgrpc++_alts$(SHARED_VERSION_CPP).so
+endif
+endif
+
+endif
+
+endif
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(LIBGRPC++_ALTS_OBJS:.o=.dep)
+endif
+endif
+$(OBJDIR)/$(CONFIG)/src/cpp/common/alts_context.o: $(GENDIR)/src/proto/grpc/gcp/altscontext.pb.cc $(GENDIR)/src/proto/grpc/gcp/altscontext.grpc.pb.cc $(GENDIR)/src/proto/grpc/gcp/transport_security_common.pb.cc $(GENDIR)/src/proto/grpc/gcp/transport_security_common.grpc.pb.cc
+
+
 LIBGRPC++_CORE_STATS_SRC = \
     $(GENDIR)/src/proto/grpc/core/stats.pb.cc $(GENDIR)/src/proto/grpc/core/stats.grpc.pb.cc \
     src/cpp/util/core_stats.cc \
@@ -13815,6 +13941,49 @@ $(OBJDIR)/$(CONFIG)/test/core/tsi/alts/fake_handshaker/fake_handshaker_server.o:
 $(OBJDIR)/$(CONFIG)/test/core/tsi/alts/handshaker/alts_concurrent_connectivity_test.o: $(GENDIR)/test/core/tsi/alts/fake_handshaker/handshaker.pb.cc $(GENDIR)/test/core/tsi/alts/fake_handshaker/handshaker.grpc.pb.cc $(GENDIR)/test/core/tsi/alts/fake_handshaker/transport_security_common.pb.cc $(GENDIR)/test/core/tsi/alts/fake_handshaker/transport_security_common.grpc.pb.cc
 
 
+ALTS_CONTEXT_TEST_SRC = \
+    test/cpp/common/alts_context_test.cc \
+
+ALTS_CONTEXT_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(ALTS_CONTEXT_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/alts_context_test: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+.
+
+$(BINDIR)/$(CONFIG)/alts_context_test: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/alts_context_test: $(PROTOBUF_DEP) $(ALTS_CONTEXT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_alts.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(ALTS_CONTEXT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_alts.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/alts_context_test
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/cpp/common/alts_context_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_alts.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+
+deps_alts_context_test: $(ALTS_CONTEXT_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(ALTS_CONTEXT_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 ALTS_COUNTER_TEST_SRC = \
     test/core/tsi/alts/frame_protector/alts_counter_test.cc \
 
@@ -23232,6 +23401,7 @@ src/core/tsi/ssl_transport_security.cc: $(OPENSSL_DEP)
 src/core/tsi/transport_security.cc: $(OPENSSL_DEP)
 src/core/tsi/transport_security_grpc.cc: $(OPENSSL_DEP)
 src/cpp/client/secure_credentials.cc: $(OPENSSL_DEP)
+src/cpp/common/alts_context.cc: $(OPENSSL_DEP)
 src/cpp/common/auth_property_iterator.cc: $(OPENSSL_DEP)
 src/cpp/common/secure_auth_context.cc: $(OPENSSL_DEP)
 src/cpp/common/secure_channel_arguments.cc: $(OPENSSL_DEP)
diff --git a/build.yaml b/build.yaml
index 7b35307df7e..b26ed6eddc7 100644
--- a/build.yaml
+++ b/build.yaml
@@ -1881,6 +1881,19 @@ libs:
   - grpc++_codegen_proto
   - grpc++_codegen_base_src
   secure: check
+- name: grpc++_alts
+  build: all
+  language: c++
+  public_headers:
+  - include/grpcpp/alts_context.h
+  - include/grpcpp/impl/codegen/security/auth_context.h
+  src:
+  - src/proto/grpc/gcp/altscontext.proto
+  - src/proto/grpc/gcp/transport_security_common.proto
+  - src/cpp/common/alts_context.cc
+  deps:
+  - grpc++
+  baselib: true
 - name: grpc++_core_stats
   build: private
   language: c++
@@ -3937,6 +3950,19 @@ targets:
   - grpc++_test_config
   platforms:
   - linux
+- name: alts_context_test
+  build: test
+  language: c++
+  src:
+  - test/cpp/common/alts_context_test.cc
+  deps:
+  - grpc++_test_util
+  - grpc_test_util
+  - grpc++_alts
+  - grpc++
+  - grpc
+  - gpr
+  - grpc++_test_config
 - name: alts_counter_test
   build: test
   language: c++
diff --git a/grpc.gyp b/grpc.gyp
index 5b6dbd3d44b..cf41f2dc1c8 100644
--- a/grpc.gyp
+++ b/grpc.gyp
@@ -1754,6 +1754,18 @@
         'src/cpp/codegen/codegen_init.cc',
       ],
     },
+    {
+      'target_name': 'grpc++_alts',
+      'type': 'static_library',
+      'dependencies': [
+        'grpc++',
+      ],
+      'sources': [
+        'src/proto/grpc/gcp/altscontext.proto',
+        'src/proto/grpc/gcp/transport_security_common.proto',
+        'src/cpp/common/alts_context.cc',
+      ],
+    },
     {
       'target_name': 'grpc++_core_stats',
       'type': 'static_library',
diff --git a/include/grpcpp/alts_context.h b/include/grpcpp/alts_context.h
new file mode 100644
index 00000000000..ae4ce2bf69a
--- /dev/null
+++ b/include/grpcpp/alts_context.h
@@ -0,0 +1,39 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPCPP_ALTS_CONTEXT_H
+#define GRPCPP_ALTS_CONTEXT_H
+
+#include <grpcpp/impl/codegen/security/auth_context.h>
+#include <memory>
+
+namespace grpc {
+
+namespace gcp {
+class AltsContext;
+}  // namespace gcp
+
+// GetAltsContextFromAuthContext helps to get the AltsContext from AuthContext.
+// Please make sure the underlying protocol is ALTS before calling this
+// function. Otherwise a nullptr will be returned.
+std::unique_ptr<gcp::AltsContext> GetAltsContextFromAuthContext(
+    const AuthContext& auth_context);
+
+}  // namespace grpc
+
+#endif  // GRPCPP_ALTS_CONTEXT_H
diff --git a/src/cpp/common/alts_context.cc b/src/cpp/common/alts_context.cc
new file mode 100644
index 00000000000..5e41a8797a9
--- /dev/null
+++ b/src/cpp/common/alts_context.cc
@@ -0,0 +1,44 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/grpc_security.h>
+#include <grpcpp/alts_context.h>
+
+#include "src/core/tsi/alts/handshaker/alts_tsi_handshaker.h"
+#include "src/cpp/common/secure_auth_context.h"
+#include "src/proto/grpc/gcp/altscontext.pb.h"
+
+namespace grpc {
+std::unique_ptr<gcp::AltsContext> GetAltsContextFromAuthContext(
+    const AuthContext& auth_context) {
+  std::vector<string_ref> ctx_vector =
+      auth_context.FindPropertyValues(TSI_ALTS_CONTEXT);
+  if (ctx_vector.size() != 1) {
+    gpr_log(GPR_ERROR, "contains zero or more than one ALTS context.");
+    return nullptr;
+  }
+  std::unique_ptr<gcp::AltsContext> uniq_ctx(new gcp::AltsContext());
+  bool success = uniq_ctx.get()->ParseFromArray(ctx_vector[0].data(),
+                                                ctx_vector[0].size());
+  if (!success) {
+    gpr_log(GPR_ERROR, "fails to parse ALTS context.");
+    return nullptr;
+  }
+  return uniq_ctx;
+}
+}  // namespace grpc
diff --git a/src/proto/grpc/gcp/BUILD b/src/proto/grpc/gcp/BUILD
index 1c22d89e464..f93de9c7a1b 100644
--- a/src/proto/grpc/gcp/BUILD
+++ b/src/proto/grpc/gcp/BUILD
@@ -23,3 +23,11 @@ proto_library(
     ],
     visibility = ["//visibility:public"],
 )
+
+cc_proto_library(
+    name = "alts_handshaker_cc_proto",
+    visibility = ["//visibility:public"],
+    deps = [
+        "alts_handshaker_proto",
+    ],
+)
diff --git a/test/cpp/common/BUILD b/test/cpp/common/BUILD
index 05e7bda330a..c6fc9a8ed91 100644
--- a/test/cpp/common/BUILD
+++ b/test/cpp/common/BUILD
@@ -108,3 +108,16 @@ grpc_cc_test(
         "//test/cpp/util:test_util",
     ],
 )
+
+grpc_cc_test(
+    name = "alts_context_test",
+    srcs = ["alts_context_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
+    deps = [
+        "//:grpc++_alts",
+        "//test/core/util:grpc_test_util",
+        "//test/cpp/util:test_util",
+    ],
+)
diff --git a/test/cpp/common/alts_context_test.cc b/test/cpp/common/alts_context_test.cc
new file mode 100644
index 00000000000..a3b77e2f1aa
--- /dev/null
+++ b/test/cpp/common/alts_context_test.cc
@@ -0,0 +1,96 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpcpp/alts_context.h>
+#include <grpcpp/security/auth_context.h>
+
+#include "src/core/tsi/alts/handshaker/alts_tsi_handshaker.h"
+#include "src/cpp/common/secure_auth_context.h"
+#include "src/proto/grpc/gcp/altscontext.pb.h"
+#include "test/cpp/util/string_ref_helper.h"
+
+#include <gtest/gtest.h>
+
+using grpc::testing::ToString;
+
+namespace grpc {
+namespace {
+
+TEST(AltsContextTest, EmptyAuthContext) {
+  SecureAuthContext context(nullptr);
+  std::unique_ptr<gcp::AltsContext> alts_context =
+      GetAltsContextFromAuthContext(context);
+  EXPECT_EQ(alts_context.get(), nullptr);
+}
+
+TEST(AltsContextTest, AuthContextWithMoreThanOneAltsContext) {
+  grpc_core::RefCountedPtr<grpc_auth_context> ctx =
+      grpc_core::MakeRefCounted<grpc_auth_context>(nullptr);
+  SecureAuthContext context(ctx.get());
+  ctx.reset();
+  context.AddProperty(TSI_ALTS_CONTEXT, "context1");
+  context.AddProperty(TSI_ALTS_CONTEXT, "context2");
+  std::unique_ptr<gcp::AltsContext> alts_context =
+      GetAltsContextFromAuthContext(context);
+  EXPECT_EQ(alts_context.get(), nullptr);
+}
+
+TEST(AltsContextTest, AuthContextWithBadAltsContext) {
+  grpc_core::RefCountedPtr<grpc_auth_context> ctx =
+      grpc_core::MakeRefCounted<grpc_auth_context>(nullptr);
+  SecureAuthContext context(ctx.get());
+  ctx.reset();
+  context.AddProperty(TSI_ALTS_CONTEXT, "bad context string serialization");
+  std::unique_ptr<gcp::AltsContext> alts_context =
+      GetAltsContextFromAuthContext(context);
+  EXPECT_EQ(alts_context.get(), nullptr);
+}
+
+TEST(AltsContextTest, AuthContextWithGoodAltsContext) {
+  grpc_core::RefCountedPtr<grpc_auth_context> ctx =
+      grpc_core::MakeRefCounted<grpc_auth_context>(nullptr);
+  SecureAuthContext context(ctx.get());
+  ctx.reset();
+  std::string expected_application_protocol("application protocol");
+  std::string expected_record_protocol("record protocol");
+  std::string expected_peer_id("peer");
+  std::string expected_local_id("local");
+  gcp::AltsContext expect_alts_ctx;
+  expect_alts_ctx.set_application_protocol(expected_application_protocol);
+  expect_alts_ctx.set_record_protocol(expected_record_protocol);
+  expect_alts_ctx.set_peer_service_account(expected_peer_id);
+  expect_alts_ctx.set_local_service_account(expected_local_id);
+  std::string serialized_alts_ctx;
+  bool success = expect_alts_ctx.SerializeToString(&serialized_alts_ctx);
+  EXPECT_TRUE(success);
+  context.AddProperty(TSI_ALTS_CONTEXT, serialized_alts_ctx);
+  std::unique_ptr<gcp::AltsContext> alts_ctx =
+      GetAltsContextFromAuthContext(context);
+  EXPECT_EQ(expected_application_protocol, alts_ctx->application_protocol());
+  EXPECT_EQ(expected_record_protocol, alts_ctx->record_protocol());
+  EXPECT_EQ(expected_peer_id, alts_ctx->peer_service_account());
+  EXPECT_EQ(expected_local_id, alts_ctx->local_service_account());
+}
+
+}  // namespace
+}  // namespace grpc
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index 0f3920e95fc..52a49d900bf 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -3031,6 +3031,30 @@
     ], 
     "uses_polling": true
   }, 
+  {
+    "args": [], 
+    "benchmark": false, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": false, 
+    "language": "c++", 
+    "name": "alts_context_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "uses_polling": true
+  }, 
   {
     "args": [], 
     "benchmark": false, 

From aa156984b164738203951ed6d2f71f65491675c2 Mon Sep 17 00:00:00 2001
From: Richard Belleville <rbellevi@google.com>
Date: Mon, 25 Nov 2019 17:41:16 -0800
Subject: [PATCH 21/26] Support mutating a value used for a response in
 grpcio_testing

---
 .../grpc_testing/_server/_service.py          |  3 ++-
 .../tests/testing/_application_common.py      |  2 ++
 .../tests/testing/_server_application.py      | 12 +++++++--
 .../tests/testing/_server_test.py             | 25 +++++++++++++++++++
 4 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/src/python/grpcio_testing/grpc_testing/_server/_service.py b/src/python/grpcio_testing/grpc_testing/_server/_service.py
index 36b0a2f7fff..97b3ffa4229 100644
--- a/src/python/grpcio_testing/grpc_testing/_server/_service.py
+++ b/src/python/grpcio_testing/grpc_testing/_server/_service.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import grpc
 
 
@@ -59,7 +60,7 @@ def _stream_response(argument, implementation, rpc, servicer_context):
     else:
         while True:
             try:
-                response = next(response_iterator)
+                response = copy.deepcopy(next(response_iterator))
             except StopIteration:
                 rpc.stream_response_complete()
                 break
diff --git a/src/python/grpcio_tests/tests/testing/_application_common.py b/src/python/grpcio_tests/tests/testing/_application_common.py
index b7d5b236051..3226d1fb020 100644
--- a/src/python/grpcio_tests/tests/testing/_application_common.py
+++ b/src/python/grpcio_tests/tests/testing/_application_common.py
@@ -37,5 +37,7 @@ ABORT_SUCCESS_QUERY = requests_pb2.Up(first_up_field=43)
 ABORT_NO_STATUS_RESPONSE = services_pb2.Down(first_down_field=50)
 ABORT_SUCCESS_RESPONSE = services_pb2.Down(first_down_field=51)
 ABORT_FAILURE_RESPONSE = services_pb2.Down(first_down_field=52)
+STREAM_STREAM_MUTATING_REQUEST = requests_pb2.Top(first_top_field=24601)
+STREAM_STREAM_MUTATING_COUNT = 2
 
 INFINITE_REQUEST_STREAM_TIMEOUT = 0.2
diff --git a/src/python/grpcio_tests/tests/testing/_server_application.py b/src/python/grpcio_tests/tests/testing/_server_application.py
index 1dc5e8f3917..51ed977b8fe 100644
--- a/src/python/grpcio_tests/tests/testing/_server_application.py
+++ b/src/python/grpcio_tests/tests/testing/_server_application.py
@@ -75,13 +75,21 @@ class FirstServiceServicer(services_pb2_grpc.FirstServiceServicer):
             return _application_common.STREAM_UNARY_RESPONSE
 
     def StreStre(self, request_iterator, context):
+        valid_requests = (_application_common.STREAM_STREAM_REQUEST,
+                          _application_common.STREAM_STREAM_MUTATING_REQUEST)
         for request in request_iterator:
-            if request != _application_common.STREAM_STREAM_REQUEST:
+            if request not in valid_requests:
                 context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
                 context.set_details('Something is wrong with your request!')
                 return
             elif not context.is_active():
                 return
-            else:
+            elif request == _application_common.STREAM_STREAM_REQUEST:
                 yield _application_common.STREAM_STREAM_RESPONSE
                 yield _application_common.STREAM_STREAM_RESPONSE
+            elif request == _application_common.STREAM_STREAM_MUTATING_REQUEST:
+                response = services_pb2.Bottom()
+                for i in range(
+                        _application_common.STREAM_STREAM_MUTATING_COUNT):
+                    response.first_bottom_field = i
+                    yield response
diff --git a/src/python/grpcio_tests/tests/testing/_server_test.py b/src/python/grpcio_tests/tests/testing/_server_test.py
index 45975f229bc..617a41b7e54 100644
--- a/src/python/grpcio_tests/tests/testing/_server_test.py
+++ b/src/python/grpcio_tests/tests/testing/_server_test.py
@@ -21,6 +21,7 @@ import grpc_testing
 from tests.testing import _application_common
 from tests.testing import _application_testing_common
 from tests.testing import _server_application
+from tests.testing.proto import services_pb2
 
 
 class FirstServiceServicerTest(unittest.TestCase):
@@ -94,6 +95,30 @@ class FirstServiceServicerTest(unittest.TestCase):
                              response)
         self.assertIs(code, grpc.StatusCode.OK)
 
+    def test_mutating_stream_stream(self):
+        rpc = self._real_time_server.invoke_stream_stream(
+            _application_testing_common.FIRST_SERVICE_STRESTRE, (), None)
+        rpc.send_request(_application_common.STREAM_STREAM_MUTATING_REQUEST)
+        initial_metadata = rpc.initial_metadata()
+        responses = [
+            rpc.take_response()
+            for _ in range(_application_common.STREAM_STREAM_MUTATING_COUNT)
+        ]
+        rpc.send_request(_application_common.STREAM_STREAM_MUTATING_REQUEST)
+        responses.extend([
+            rpc.take_response()
+            for _ in range(_application_common.STREAM_STREAM_MUTATING_COUNT)
+        ])
+        rpc.requests_closed()
+        _, _, _ = rpc.termination()
+        expected_responses = (
+            services_pb2.Bottom(first_bottom_field=0),
+            services_pb2.Bottom(first_bottom_field=1),
+            services_pb2.Bottom(first_bottom_field=0),
+            services_pb2.Bottom(first_bottom_field=1),
+        )
+        self.assertSequenceEqual(expected_responses, responses)
+
     def test_server_rpc_idempotence(self):
         rpc = self._real_time_server.invoke_unary_unary(
             _application_testing_common.FIRST_SERVICE_UNUN, (),

From d023aba60eaf7143187c6461c72100e761facc10 Mon Sep 17 00:00:00 2001
From: Prashant Jaikumar <pjaikumar@google.com>
Date: Mon, 25 Nov 2019 14:36:26 -0800
Subject: [PATCH 22/26] Sync time before running time_jump_test

---
 tools/internal_ci/macos/grpc_run_bazel_isolated_tests.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tools/internal_ci/macos/grpc_run_bazel_isolated_tests.sh b/tools/internal_ci/macos/grpc_run_bazel_isolated_tests.sh
index aaa2dc3f63c..8ca34d19fe3 100644
--- a/tools/internal_ci/macos/grpc_run_bazel_isolated_tests.sh
+++ b/tools/internal_ci/macos/grpc_run_bazel_isolated_tests.sh
@@ -23,6 +23,10 @@ cd $(dirname $0)/../../..
 # run cfstream_test separately because it messes with the network
 tools/bazel test $RUN_TESTS_FLAGS --spawn_strategy=standalone --genrule_strategy=standalone --test_output=all //test/cpp/end2end:cfstream_test
 
+# Make sure time is in sync before running time_jump_test because the test does
+# NTP sync before exiting. Bazel gets confused if test end time < start time.
+sudo sntp -sS pool.ntp.org
+
 # run time_jump_test separately because it changes system time
 tools/bazel test $RUN_TESTS_FLAGS --spawn_strategy=standalone --genrule_strategy=standalone --test_output=all //test/cpp/common:time_jump_test
 

From 39f6bb152ee01cc9663aad36517952847bcb11f2 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Tue, 26 Nov 2019 06:22:00 -0500
Subject: [PATCH 23/26] make test sharding work for poller-specific tests

---
 bazel/grpc_build_system.bzl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
index 033a0051cce..7bb6b8bdb98 100644
--- a/bazel/grpc_build_system.bzl
+++ b/bazel/grpc_build_system.bzl
@@ -193,6 +193,8 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data
     copts = if_mac(["-DGRPC_CFSTREAM"])
     if language.upper() == "C":
         copts = copts + if_not_windows(["-std=c99"])
+    # NOTE: these attributes won't be used for the poller-specific versions of a test
+    # automatically, you need to set them explicitly (if applicable)
     args = {
         "srcs": srcs,
         "args": args,
@@ -235,6 +237,7 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data
                 tags = (tags + ["no_windows", "no_mac"]),
                 exec_compatible_with = exec_compatible_with,
                 exec_properties = exec_properties,
+                shard_count = shard_count,
             )
     else:
         # the test behavior doesn't depend on polling, just generate the test

From 7e6defd394c996d794a26ef8ac006589ae7f6fbb Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Fri, 15 Nov 2019 08:31:52 -0500
Subject: [PATCH 24/26] use default test timeouts for RBE opt and dbg tests

---
 tools/remote_build/rbe_common.bazelrc | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tools/remote_build/rbe_common.bazelrc b/tools/remote_build/rbe_common.bazelrc
index 78f29206ced..d1a1140f3af 100644
--- a/tools/remote_build/rbe_common.bazelrc
+++ b/tools/remote_build/rbe_common.bazelrc
@@ -49,10 +49,6 @@ test --test_env=GRPC_VERBOSITY=debug
 build --test_tag_filters=-no_linux
 build --build_tag_filters=-no_linux
 
-# Default test timeouts for all RBE tests (sanitizers override these values)
-# TODO(jtattermusch): revisit the non-standard test timeout values
-build --test_timeout=300,450,1200,3600
-
 # address sanitizer: most settings are already in %workspace%/.bazelrc
 # we only need a few additional ones that are Foundry specific
 build:asan --copt=-gmlt

From 674b1f6f525f750146ce04fb9ef0f61263d455ce Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Fri, 15 Nov 2019 09:08:09 -0500
Subject: [PATCH 25/26] use more reasonable timeouts for sanitizer tests

---
 test/cpp/end2end/BUILD                |  1 +
 tools/remote_build/rbe_common.bazelrc | 19 +++++++++++--------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD
index 450ce3416cc..55c594c7f15 100644
--- a/test/cpp/end2end/BUILD
+++ b/test/cpp/end2end/BUILD
@@ -724,6 +724,7 @@ grpc_cc_test(
         "//test/core/util:grpc_test_util",
         "//test/cpp/util:test_util",
     ],
+    shard_count = 5,
 )
 
 grpc_cc_test(
diff --git a/tools/remote_build/rbe_common.bazelrc b/tools/remote_build/rbe_common.bazelrc
index d1a1140f3af..af024760cd4 100644
--- a/tools/remote_build/rbe_common.bazelrc
+++ b/tools/remote_build/rbe_common.bazelrc
@@ -52,15 +52,16 @@ build --build_tag_filters=-no_linux
 # address sanitizer: most settings are already in %workspace%/.bazelrc
 # we only need a few additional ones that are Foundry specific
 build:asan --copt=-gmlt
-# TODO(jtattermusch): use more reasonable test timeout
-build:asan --test_timeout=3600
+# use double the default value for "moderate" and "long" timeout as sanitizer
+# tests tend to be slower
+build:asan --test_timeout=60,600,1800,3600
 build:asan --test_tag_filters=-no_linux,-qps_json_driver
 
 # memory sanitizer: most settings are already in %workspace%/.bazelrc
 # we only need a few additional ones that are Foundry specific
 build:msan --copt=-gmlt
-# TODO(jtattermusch): use more reasonable test timeout
-build:msan --test_timeout=3600
+# "moderate" timeout 3x the default, "long" timeout 2x the default
+build:msan --test_timeout=60,900,1800,3600
 # TODO(jtattermusch): revisit the disabled tests
 build:msan --test_tag_filters=-no_linux,-nomsan,-json_run_localhost
 build:msan --cxxopt=--stdlib=libc++
@@ -74,16 +75,18 @@ build:msan --crosstool_top=@rbe_msan//cc:toolchain
 # thread sanitizer: most settings are already in %workspace%/.bazelrc
 # we only need a few additional ones that are Foundry specific
 build:tsan --copt=-gmlt
-# TODO(jtattermusch): use more reasonable test timeout
-build:tsan --test_timeout=3600
+# use double the default value for "moderate" and "long" timeout as sanitizer
+# tests tend to be slower
+build:tsan --test_timeout=60,600,1800,3600
 build:tsan --test_tag_filters=-no_linux,-qps_json_driver
 build:tsan --extra_execution_platforms=@rbe_default//config:platform
 
 # undefined behavior sanitizer: most settings are already in %workspace%/.bazelrc
 # we only need a few additional ones that are Foundry specific
 build:ubsan --copt=-gmlt
-# TODO(jtattermusch): use more reasonable test timeout
-build:ubsan --test_timeout=3600
+# use double the default value for "moderate" and "long" timeout as sanitizer
+# tests tend to be slower
+build:ubsan --test_timeout=60,600,1800,3600
 # override the config-agnostic crosstool_top
 # how to update the bazel toolchain for ubsan:
 # - check for the latest released version in https://github.com/bazelbuild/bazel-toolchains/tree/master/configs/experimental/ubuntu16_04_clang

From 649e706a12ff50dcc5406700a98929386e274cbc Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Tue, 26 Nov 2019 08:53:36 -0500
Subject: [PATCH 26/26] print test case name and speedup under MSAN

---
 test/core/tsi/ssl_transport_security_test.cc | 30 ++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/test/core/tsi/ssl_transport_security_test.cc b/test/core/tsi/ssl_transport_security_test.cc
index c5e6e839b18..8a5dd6e4a0a 100644
--- a/test/core/tsi/ssl_transport_security_test.cc
+++ b/test/core/tsi/ssl_transport_security_test.cc
@@ -457,6 +457,7 @@ static tsi_test_fixture* ssl_tsi_test_fixture_create() {
 }
 
 void ssl_tsi_test_do_handshake_tiny_handshake_buffer() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake_tiny_handshake_buffer");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   fixture->handshake_buffer_size = TSI_TEST_TINY_HANDSHAKE_BUFFER_SIZE;
   tsi_test_do_handshake(fixture);
@@ -464,6 +465,7 @@ void ssl_tsi_test_do_handshake_tiny_handshake_buffer() {
 }
 
 void ssl_tsi_test_do_handshake_small_handshake_buffer() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake_small_handshake_buffer");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   fixture->handshake_buffer_size = TSI_TEST_SMALL_HANDSHAKE_BUFFER_SIZE;
   tsi_test_do_handshake(fixture);
@@ -471,12 +473,14 @@ void ssl_tsi_test_do_handshake_small_handshake_buffer() {
 }
 
 void ssl_tsi_test_do_handshake() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   tsi_test_do_handshake(fixture);
   tsi_test_fixture_destroy(fixture);
 }
 
 void ssl_tsi_test_do_handshake_with_root_store() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake_with_root_store");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   ssl_tsi_test_fixture* ssl_fixture =
       reinterpret_cast<ssl_tsi_test_fixture*>(fixture);
@@ -486,6 +490,7 @@ void ssl_tsi_test_do_handshake_with_root_store() {
 }
 
 void ssl_tsi_test_do_handshake_with_client_authentication() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake_with_client_authentication");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   ssl_tsi_test_fixture* ssl_fixture =
       reinterpret_cast<ssl_tsi_test_fixture*>(fixture);
@@ -495,6 +500,9 @@ void ssl_tsi_test_do_handshake_with_client_authentication() {
 }
 
 void ssl_tsi_test_do_handshake_with_client_authentication_and_root_store() {
+  gpr_log(
+      GPR_INFO,
+      "ssl_tsi_test_do_handshake_with_client_authentication_and_root_store");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   ssl_tsi_test_fixture* ssl_fixture =
       reinterpret_cast<ssl_tsi_test_fixture*>(fixture);
@@ -505,6 +513,8 @@ void ssl_tsi_test_do_handshake_with_client_authentication_and_root_store() {
 }
 
 void ssl_tsi_test_do_handshake_with_server_name_indication_exact_domain() {
+  gpr_log(GPR_INFO,
+          "ssl_tsi_test_do_handshake_with_server_name_indication_exact_domain");
   /* server1 cert contains "waterzooi.test.google.be" in SAN. */
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   ssl_tsi_test_fixture* ssl_fixture =
@@ -516,6 +526,9 @@ void ssl_tsi_test_do_handshake_with_server_name_indication_exact_domain() {
 }
 
 void ssl_tsi_test_do_handshake_with_server_name_indication_wild_star_domain() {
+  gpr_log(
+      GPR_INFO,
+      "ssl_tsi_test_do_handshake_with_server_name_indication_wild_star_domain");
   /* server1 cert contains "*.test.google.fr" in SAN. */
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   ssl_tsi_test_fixture* ssl_fixture =
@@ -527,6 +540,7 @@ void ssl_tsi_test_do_handshake_with_server_name_indication_wild_star_domain() {
 }
 
 void ssl_tsi_test_do_handshake_with_bad_server_cert() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake_with_bad_server_cert");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   ssl_tsi_test_fixture* ssl_fixture =
       reinterpret_cast<ssl_tsi_test_fixture*>(fixture);
@@ -536,6 +550,7 @@ void ssl_tsi_test_do_handshake_with_bad_server_cert() {
 }
 
 void ssl_tsi_test_do_handshake_with_bad_client_cert() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake_with_bad_client_cert");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   ssl_tsi_test_fixture* ssl_fixture =
       reinterpret_cast<ssl_tsi_test_fixture*>(fixture);
@@ -546,6 +561,7 @@ void ssl_tsi_test_do_handshake_with_bad_client_cert() {
 }
 
 void ssl_tsi_test_do_handshake_alpn_client_no_server() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake_alpn_client_no_server");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   ssl_tsi_test_fixture* ssl_fixture =
       reinterpret_cast<ssl_tsi_test_fixture*>(fixture);
@@ -555,6 +571,7 @@ void ssl_tsi_test_do_handshake_alpn_client_no_server() {
 }
 
 void ssl_tsi_test_do_handshake_alpn_server_no_client() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake_alpn_server_no_client");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   ssl_tsi_test_fixture* ssl_fixture =
       reinterpret_cast<ssl_tsi_test_fixture*>(fixture);
@@ -564,6 +581,7 @@ void ssl_tsi_test_do_handshake_alpn_server_no_client() {
 }
 
 void ssl_tsi_test_do_handshake_alpn_client_server_mismatch() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake_alpn_server_no_client");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   ssl_tsi_test_fixture* ssl_fixture =
       reinterpret_cast<ssl_tsi_test_fixture*>(fixture);
@@ -573,6 +591,7 @@ void ssl_tsi_test_do_handshake_alpn_client_server_mismatch() {
 }
 
 void ssl_tsi_test_do_handshake_alpn_client_server_ok() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake_alpn_client_server_ok");
   tsi_test_fixture* fixture = ssl_tsi_test_fixture_create();
   ssl_tsi_test_fixture* ssl_fixture =
       reinterpret_cast<ssl_tsi_test_fixture*>(fixture);
@@ -582,6 +601,7 @@ void ssl_tsi_test_do_handshake_alpn_client_server_ok() {
 }
 
 void ssl_tsi_test_do_round_trip_for_all_configs() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_round_trip_for_all_configs");
   unsigned int* bit_array = static_cast<unsigned int*>(
       gpr_zalloc(sizeof(unsigned int) * TSI_TEST_NUM_OF_ARGUMENTS));
   const unsigned int mask = 1U << (TSI_TEST_NUM_OF_ARGUMENTS - 1);
@@ -605,7 +625,13 @@ void ssl_tsi_test_do_round_trip_for_all_configs() {
 }
 
 void ssl_tsi_test_do_round_trip_odd_buffer_size() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_round_trip_odd_buffer_size");
+#ifndef MEMORY_SANITIZER
   const size_t odd_sizes[] = {1025, 2051, 4103, 8207, 16409};
+#else
+  // avoid test being extremely slow under MSAN
+  const size_t odd_sizes[] = {1025};
+#endif
   const size_t size = sizeof(odd_sizes) / sizeof(size_t);
   for (size_t ind1 = 0; ind1 < size; ind1++) {
     for (size_t ind2 = 0; ind2 < size; ind2++) {
@@ -628,6 +654,7 @@ void ssl_tsi_test_do_round_trip_odd_buffer_size() {
 }
 
 void ssl_tsi_test_do_handshake_session_cache() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_do_handshake_session_cache");
   tsi_ssl_session_cache* session_cache = tsi_ssl_session_cache_create_lru(16);
   char session_ticket_key[kSessionTicketEncryptionKeySize];
   auto do_handshake = [&session_ticket_key,
@@ -770,12 +797,14 @@ void test_tsi_ssl_client_handshaker_factory_bad_params() {
 }
 
 void ssl_tsi_test_handshaker_factory_internals() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_handshaker_factory_internals");
   test_tsi_ssl_client_handshaker_factory_refcounting();
   test_tsi_ssl_server_handshaker_factory_refcounting();
   test_tsi_ssl_client_handshaker_factory_bad_params();
 }
 
 void ssl_tsi_test_duplicate_root_certificates() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_duplicate_root_certificates");
   char* root_cert = load_file(SSL_TSI_TEST_CREDENTIALS_DIR, "ca.pem");
   char* dup_root_cert = static_cast<char*>(
       gpr_zalloc(sizeof(char) * (strlen(root_cert) * 2 + 1)));
@@ -791,6 +820,7 @@ void ssl_tsi_test_duplicate_root_certificates() {
 }
 
 void ssl_tsi_test_extract_x509_subject_names() {
+  gpr_log(GPR_INFO, "ssl_tsi_test_extract_x509_subject_names");
   char* cert = load_file(SSL_TSI_TEST_CREDENTIALS_DIR, "multi-domain.pem");
   tsi_peer peer;
   GPR_ASSERT(tsi_ssl_extract_x509_subject_names_from_pem_cert(cert, &peer) ==