Update TSI with new non-blocking TSI interfaces.

Makefile

@@ -2987,6 +2987,7 @@ LIBGRPC_SRC = \
     src/core/tsi/fake_transport_security.c \
     src/core/tsi/ssl_transport_security.c \
     src/core/tsi/transport_security.c \
+    src/core/tsi/transport_security_adapter.c \
     src/core/ext/transport/chttp2/server/chttp2_server.c \
     src/core/ext/transport/chttp2/client/secure/secure_channel_create.c \
     src/core/ext/filters/client_channel/channel_connectivity.c \
@@ -3330,6 +3331,7 @@ LIBGRPC_CRONET_SRC = \
     src/core/tsi/fake_transport_security.c \
     src/core/tsi/ssl_transport_security.c \
     src/core/tsi/transport_security.c \
+    src/core/tsi/transport_security_adapter.c \
     src/core/ext/transport/chttp2/client/chttp2_connector.c \
     src/core/ext/filters/load_reporting/load_reporting.c \
     src/core/ext/filters/load_reporting/load_reporting_filter.c \
@@ -19257,6 +19259,7 @@ src/core/plugin_registry/grpc_plugin_registry.c: $(OPENSSL_DEP)
 src/core/tsi/fake_transport_security.c: $(OPENSSL_DEP)
 src/core/tsi/ssl_transport_security.c: $(OPENSSL_DEP)
 src/core/tsi/transport_security.c: $(OPENSSL_DEP)
+src/core/tsi/transport_security_adapter.c: $(OPENSSL_DEP)
 src/cpp/client/cronet_credentials.cc: $(OPENSSL_DEP)
 src/cpp/client/secure_credentials.cc: $(OPENSSL_DEP)
 src/cpp/common/auth_property_iterator.cc: $(OPENSSL_DEP)

src/core/tsi/fake_transport_security.c

@@ -499,6 +499,7 @@ static const tsi_handshaker_vtable handshaker_vtable = {
     fake_handshaker_extract_peer,
     fake_handshaker_create_frame_protector,
     fake_handshaker_destroy,
+    NULL,
 };
 
 tsi_handshaker *tsi_create_fake_handshaker(int is_client) {

src/core/tsi/ssl_transport_security.c

@@ -1049,6 +1049,7 @@ static const tsi_handshaker_vtable handshaker_vtable = {
     ssl_handshaker_extract_peer,
     ssl_handshaker_create_frame_protector,
     ssl_handshaker_destroy,
+    NULL,
 };
 
 /* --- tsi_ssl_handshaker_factory common methods. --- */

src/core/tsi/transport_security.c

@@ -73,6 +73,8 @@ const char *tsi_result_to_string(tsi_result result) {
       return "TSI_HANDSHAKE_IN_PROGRESS";
     case TSI_OUT_OF_RESOURCES:
       return "TSI_OUT_OF_RESOURCES";
+    case TSI_ASYNC:
+      return "TSI_ASYNC";
     default:
       return "UNKNOWN";
   }
@@ -185,11 +187,53 @@ tsi_result tsi_handshaker_create_frame_protector(
   return result;
 }
 
+tsi_result tsi_handshaker_next(
+    tsi_handshaker *self, const unsigned char *received_bytes,
+    size_t received_bytes_size, unsigned char **bytes_to_send,
+    size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result,
+    tsi_handshaker_on_next_done_cb cb, void *user_data) {
+  if (self == NULL) return TSI_INVALID_ARGUMENT;
+  if (self->handshaker_result_created) return TSI_FAILED_PRECONDITION;
+  return self->vtable->next(self, received_bytes, received_bytes_size,
+                            bytes_to_send, bytes_to_send_size,
+                            handshaker_result, cb, user_data);
+}
+
 void tsi_handshaker_destroy(tsi_handshaker *self) {
   if (self == NULL) return;
   self->vtable->destroy(self);
 }
 
+/* --- tsi_handshaker_result implementation. --- */
+
+tsi_result tsi_handshaker_result_extract_peer(tsi_handshaker_result *self,
+                                              tsi_peer *peer) {
+  if (self == NULL || peer == NULL) return TSI_INVALID_ARGUMENT;
+  memset(peer, 0, sizeof(tsi_peer));
+  return self->vtable->extract_peer(self, peer);
+}
+
+tsi_result tsi_handshaker_result_create_frame_protector(
+    tsi_handshaker_result *self, size_t *max_protected_frame_size,
+    tsi_frame_protector **protector) {
+  if (self == NULL || protector == NULL) return TSI_INVALID_ARGUMENT;
+  return self->vtable->create_frame_protector(self, max_protected_frame_size,
+                                              protector);
+}
+
+tsi_result tsi_handshaker_result_get_unused_bytes(
+    tsi_handshaker_result *self, unsigned char **bytes, size_t *bytes_size) {
+  if (self == NULL || bytes == NULL || bytes_size == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
+  return self->vtable->get_unused_bytes(self, bytes, bytes_size);
+}
+
+void tsi_handshaker_result_destroy(tsi_handshaker_result *self) {
+  if (self == NULL) return;
+  self->vtable->destroy(self);
+}
+
 /* --- tsi_peer implementation. --- */
 
 tsi_peer_property tsi_init_peer_property(void) {

src/core/tsi/transport_security.h

@@ -81,11 +81,33 @@ typedef struct {
                                        size_t *max_protected_frame_size,
                                        tsi_frame_protector **protector);
   void (*destroy)(tsi_handshaker *self);
+  tsi_result (*next)(tsi_handshaker *self, const unsigned char *received_bytes,
+                     size_t received_bytes_size, unsigned char **bytes_to_send,
+                     size_t *bytes_to_send_size,
+                     tsi_handshaker_result **handshaker_result,
+                     tsi_handshaker_on_next_done_cb cb, void *user_data);
 } tsi_handshaker_vtable;
 
 struct tsi_handshaker {
   const tsi_handshaker_vtable *vtable;
   int frame_protector_created;
+  int handshaker_result_created;
+};
+
+/* Base for tsi_handshaker_result implementations.
+   See transport_security_interface.h for documentation. */
+typedef struct {
+  tsi_result (*extract_peer)(tsi_handshaker_result *self, tsi_peer *peer);
+  tsi_result (*create_frame_protector)(tsi_handshaker_result *self,
+                                       size_t *max_output_protected_frame_size,
+                                       tsi_frame_protector **protector);
+  tsi_result (*get_unused_bytes)(tsi_handshaker_result *self,
+                                 unsigned char **bytes, size_t *bytes_size);
+  void (*destroy)(tsi_handshaker_result *self);
+} tsi_handshaker_result_vtable;
+
+struct tsi_handshaker_result {
+  const tsi_handshaker_result_vtable *vtable;
 };
 
 /* Peer and property construction/destruction functions. */

src/core/tsi/transport_security_adapter.c

@@ -0,0 +1,230 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/tsi/transport_security_adapter.h"
+
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include "src/core/tsi/transport_security.h"
+
+#define TSI_ADAPTER_INITIAL_BUFFER_SIZE 256
+
+/* --- tsi_adapter_handshaker_result implementation ---*/
+
+typedef struct {
+  tsi_handshaker_result base;
+  tsi_handshaker *handshaker;
+  unsigned char *unused_bytes;
+  size_t unused_bytes_size;
+} tsi_adapter_handshaker_result;
+
+static tsi_result tsi_adapter_result_extract_peer(tsi_handshaker_result *self,
+                                                  tsi_peer *peer) {
+  tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self;
+  return tsi_handshaker_extract_peer(impl->handshaker, peer);
+}
+
+static tsi_result tsi_adapter_result_create_frame_protector(
+    tsi_handshaker_result *self, size_t *max_output_protected_frame_size,
+    tsi_frame_protector **protector) {
+  tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self;
+  return tsi_handshaker_create_frame_protector(
+      impl->handshaker, max_output_protected_frame_size, protector);
+}
+
+static tsi_result tsi_adapter_result_get_unused_bytes(
+    tsi_handshaker_result *self, unsigned char **bytes, size_t *byte_size) {
+  tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self;
+  *bytes = impl->unused_bytes;
+  *byte_size = impl->unused_bytes_size;
+  return TSI_OK;
+}
+
+static void tsi_adapter_result_destroy(tsi_handshaker_result *self) {
+  tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self;
+  if (impl->unused_bytes != NULL) {
+    gpr_free(impl->unused_bytes);
+  }
+  gpr_free(self);
+}
+
+static const tsi_handshaker_result_vtable result_vtable = {
+    tsi_adapter_result_extract_peer, tsi_adapter_result_create_frame_protector,
+    tsi_adapter_result_get_unused_bytes, tsi_adapter_result_destroy,
+};
+
+tsi_result tsi_adapter_create_handshaker_result(
+    tsi_handshaker *handshaker, const unsigned char *unused_bytes,
+    size_t unused_bytes_size, tsi_handshaker_result **handshaker_result) {
+  if (handshaker == NULL || (unused_bytes_size > 0 && unused_bytes == NULL)) {
+    return TSI_INVALID_ARGUMENT;
+  }
+  tsi_adapter_handshaker_result *impl = gpr_zalloc(sizeof(*impl));
+  impl->base.vtable = &result_vtable;
+  impl->handshaker = handshaker;
+  impl->unused_bytes_size = unused_bytes_size;
+  if (unused_bytes_size > 0) {
+    impl->unused_bytes = gpr_malloc(unused_bytes_size);
+    memcpy(impl->unused_bytes, unused_bytes, unused_bytes_size);
+  } else {
+    impl->unused_bytes = NULL;
+  }
+  *handshaker_result = &impl->base;
+  return TSI_OK;
+}
+
+/* --- tsi_adapter_handshaker implementation ---*/
+
+typedef struct {
+  tsi_handshaker base;
+  tsi_handshaker *wrapped;
+  unsigned char *adapter_buffer;
+  size_t adapter_buffer_size;
+} tsi_adapter_handshaker;
+
+tsi_result tsi_adapter_get_bytes_to_send_to_peer(tsi_handshaker *self,
+                                                 unsigned char *bytes,
+                                                 size_t *bytes_size) {
+  return tsi_handshaker_get_bytes_to_send_to_peer(
+      tsi_adapter_handshaker_get_wrapped(self), bytes, bytes_size);
+}
+
+tsi_result tsi_adapter_process_bytes_from_peer(tsi_handshaker *self,
+                                               const unsigned char *bytes,
+                                               size_t *bytes_size) {
+  return tsi_handshaker_process_bytes_from_peer(
+      tsi_adapter_handshaker_get_wrapped(self), bytes, bytes_size);
+}
+
+tsi_result tsi_adapter_get_result(tsi_handshaker *self) {
+  return tsi_handshaker_get_result(tsi_adapter_handshaker_get_wrapped(self));
+}
+
+tsi_result tsi_adapter_extract_peer(tsi_handshaker *self, tsi_peer *peer) {
+  return tsi_handshaker_extract_peer(tsi_adapter_handshaker_get_wrapped(self),
+                                     peer);
+}
+
+tsi_result tsi_adapter_create_frame_protector(tsi_handshaker *self,
+                                              size_t *max_protected_frame_size,
+                                              tsi_frame_protector **protector) {
+  return tsi_handshaker_create_frame_protector(
+      tsi_adapter_handshaker_get_wrapped(self), max_protected_frame_size,
+      protector);
+}
+
+void tsi_adapter_destroy(tsi_handshaker *self) {
+  tsi_adapter_handshaker *impl = (tsi_adapter_handshaker *)self;
+  tsi_handshaker_destroy(impl->wrapped);
+  gpr_free(impl->adapter_buffer);
+  gpr_free(self);
+}
+
+tsi_result tsi_adapter_next(
+    tsi_handshaker *self, const unsigned char *received_bytes,
+    size_t received_bytes_size, unsigned char **bytes_to_send,
+    size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result,
+    tsi_handshaker_on_next_done_cb cb, void *user_data) {
+  /* Input sanity check.  */
+  if ((received_bytes_size > 0 && received_bytes == NULL) ||
+      bytes_to_send == NULL || bytes_to_send_size == NULL ||
+      handshaker_result == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
+
+  /* If there are received bytes, process them first.  */
+  tsi_adapter_handshaker *impl = (tsi_adapter_handshaker *)self;
+  tsi_result status = TSI_OK;
+  size_t bytes_consumed = received_bytes_size;
+  if (received_bytes_size > 0) {
+    status = tsi_handshaker_process_bytes_from_peer(
+        impl->wrapped, received_bytes, &bytes_consumed);
+    if (status != TSI_OK) return status;
+  }
+
+  /* Get bytes to send to the peer, if available.  */
+  size_t offset = 0;
+  do {
+    size_t to_send_size = impl->adapter_buffer_size - offset;
+    status = tsi_handshaker_get_bytes_to_send_to_peer(
+        impl->wrapped, impl->adapter_buffer + offset, &to_send_size);
+    offset += to_send_size;
+    if (status == TSI_INCOMPLETE_DATA) {
+      impl->adapter_buffer_size *= 2;
+      impl->adapter_buffer =
+          gpr_realloc(impl->adapter_buffer, impl->adapter_buffer_size);
+    }
+  } while (status == TSI_INCOMPLETE_DATA);
+  if (status != TSI_OK) return status;
+  *bytes_to_send = impl->adapter_buffer;
+  *bytes_to_send_size = offset;
+
+  /* If handshake completes, create tsi_handshaker_result.  */
+  if (!tsi_handshaker_is_in_progress(impl->wrapped)) {
+    size_t unused_bytes_size = received_bytes_size - bytes_consumed;
+    const unsigned char *unused_bytes =
+        unused_bytes_size == 0 ? NULL : received_bytes + bytes_consumed;
+    return tsi_adapter_create_handshaker_result(
+        impl->wrapped, unused_bytes, unused_bytes_size, handshaker_result);
+  }
+  *handshaker_result = NULL;
+  return TSI_OK;
+}
+
+static const tsi_handshaker_vtable handshaker_vtable = {
+    tsi_adapter_get_bytes_to_send_to_peer,
+    tsi_adapter_process_bytes_from_peer,
+    tsi_adapter_get_result,
+    tsi_adapter_extract_peer,
+    tsi_adapter_create_frame_protector,
+    tsi_adapter_destroy,
+    tsi_adapter_next,
+};
+
+tsi_handshaker *tsi_create_adapter_handshaker(tsi_handshaker *wrapped) {
+  GPR_ASSERT(wrapped != NULL);
+  tsi_adapter_handshaker *impl = gpr_zalloc(sizeof(*impl));
+  impl->base.vtable = &handshaker_vtable;
+  impl->wrapped = wrapped;
+  impl->adapter_buffer_size = TSI_ADAPTER_INITIAL_BUFFER_SIZE;
+  impl->adapter_buffer = gpr_malloc(impl->adapter_buffer_size);
+  return &impl->base;
+}
+
+tsi_handshaker *tsi_adapter_handshaker_get_wrapped(tsi_handshaker *adapter) {
+  if (adapter == NULL) return NULL;
+  tsi_adapter_handshaker *impl = (tsi_adapter_handshaker *)adapter;
+  return impl->wrapped;
+}

src/core/tsi/transport_security_adapter.h

@@ -0,0 +1,55 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H
+#define GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H
+
+#include "src/core/tsi/transport_security_interface.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Create a tsi handshaker that takes an implementation of old interface and
+   converts into an implementation of new interface.
+   Ownership of input tsi_handshaker is transferred to this new adapter.  */
+tsi_handshaker *tsi_create_adapter_handshaker(tsi_handshaker *wrapped);
+
+/* Given a tsi adapter handshaker, return the original wrapped handshaker.  */
+tsi_handshaker *tsi_adapter_handshaker_get_wrapped(tsi_handshaker *adapter);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H

src/core/tsi/transport_security_interface.h

@@ -56,7 +56,8 @@ typedef enum {
   TSI_NOT_FOUND = 9,
   TSI_PROTOCOL_FAILURE = 10,
   TSI_HANDSHAKE_IN_PROGRESS = 11,
-  TSI_OUT_OF_RESOURCES = 12
+  TSI_OUT_OF_RESOURCES = 12,
+  TSI_ASYNC = 13
 } tsi_result;
 
 typedef enum {
@@ -208,76 +209,203 @@ typedef struct {
 /* Destructs the tsi_peer object. */
 void tsi_peer_destruct(tsi_peer *self);
 
+/*  --- tsi_handshaker_result object ---
+
+  This object contains all necessary handshake results and data such as peer
+  info, negotiated keys, unused handshake bytes, when the handshake completes.
+  Implementations of this object must be thread compatible.  */
+
+typedef struct tsi_handshaker_result tsi_handshaker_result;
+
+/* This method extracts tsi peer. It returns TSI_OK assuming there is no fatal
+   error.
+   The caller is responsible for destructing the peer.  */
+tsi_result tsi_handshaker_result_extract_peer(tsi_handshaker_result *self,
+                                              tsi_peer *peer);
+
+/* This method creates a tsi_frame_protector object. It returns TSI_OK assuming
+   there is no fatal error.
+   The caller is responsible for destroying the protector.  */
+tsi_result tsi_handshaker_result_create_frame_protector(
+    tsi_handshaker_result *self, size_t *max_output_protected_frame_size,
+    tsi_frame_protector **protector);
+
+/* This method returns the unused bytes from the handshake. It returns TSI_OK
+   assuming there is no fatal error.
+   The caller should not free the bytes.  */
+tsi_result tsi_handshaker_result_get_unused_bytes(tsi_handshaker_result *self,
+                                                  unsigned char **bytes,
+                                                  size_t *byte_size);
+
+/* This method releases the tsi_handshaker_handshaker object. After this method
+   is called, no other method can be called on the object.  */
+void tsi_handshaker_result_destroy(tsi_handshaker_result *self);
+
 /* --- tsi_handshaker objects ----
 
    Implementations of this object must be thread compatible.
 
-   A typical usage of this object would be:
+   ------------------------------------------------------------------------
+
+   A typical usage of the synchronous TSI handshaker would be:
 
    ------------------------------------------------------------------------
-   tsi_result result = TSI_OK;
+   tsi_result status = TSI_OK;
    unsigned char buf[4096];
-   size_t buf_offset;
+   const size_t buf_size = 4906;
-   size_t buf_size;
+   size_t bytes_received_size = 0;
+   unsigned char *bytes_to_send = NULL;
+   size_t bytes_to_send_size = 0;
+   tsi_handshaker_result *result = NULL;
+
    while (1) {
-     // See if we need to send some bytes to the peer.
+     status = tsi_handshaker_next(
-     do {
+         handshaker, buf, bytes_received_size,
-       size_t buf_size_to_send = sizeof(buf);
+         &bytes_to_send, &bytes_to_send_size, &result, NULL, NULL);
-       result = tsi_handshaker_get_bytes_to_send_to_peer(handshaker, buf,
+     if (status == TSI_INCOMPLETE_DATA) {
-                                                         &buf_size_to_send);
+       // Need more data from the peer.
-       if (buf_size_to_send > 0) send_bytes_to_peer(buf, buf_size_to_send);
+       bytes_received_size = buf_size;
-     } while (result == TSI_INCOMPLETE_DATA);
+       read_bytes_from_peer(buf, &bytes_received_size);
-     if (result != TSI_OK) return result;
+       continue;
-     if (!tsi_handshaker_is_in_progress(handshaker)) break;
+     }
-
+     if (status != TSI_OK) return status;
-     do {
+     if (bytes_to_send_size > 0) {
-       // Read bytes from the peer.
+       send_bytes_to_peer(bytes_to_send, bytes_to_send_size);
-       buf_size = sizeof(buf);
+     }
-       buf_offset = 0;
+     if (result != NULL) break;
-       read_bytes_from_peer(buf, &buf_size);
+     bytes_received_size = buf_size;
-       if (buf_size == 0) break;
+     read_bytes_from_peer(buf, &bytes_received_size);
-
-       // Process the bytes from the peer. We have to be careful as these bytes
-       // may contain non-handshake data (protected data). If this is the case,
-       // we will exit from the loop with buf_size > 0.
-       size_t consumed_by_handshaker = buf_size;
-       result = tsi_handshaker_process_bytes_from_peer(
-           handshaker, buf, &consumed_by_handshaker);
-       buf_size -= consumed_by_handshaker;
-       buf_offset += consumed_by_handshaker;
-     } while (result == TSI_INCOMPLETE_DATA);
-
-     if (result != TSI_OK) return result;
-     if (!tsi_handshaker_is_in_progress(handshaker)) break;
    }
 
    // Check the Peer.
    tsi_peer peer;
-   do {
+   status = tsi_handshaker_result_extract_peer(result, &peer);
-     result = tsi_handshaker_extract_peer(handshaker, &peer);
+   if (status != TSI_OK) return status;
-     if (result != TSI_OK) break;
+   status = check_peer(&peer);
-     result = check_peer(&peer);
-   } while (0);
    tsi_peer_destruct(&peer);
-   if (result != TSI_OK) return result;
+   if (status != TSI_OK) return status;
 
    // Create the protector.
    tsi_frame_protector* protector = NULL;
-   result = tsi_handshaker_create_frame_protector(handshaker, NULL,
+   status = tsi_handshaker_result_create_frame_protector(result, NULL,
-                                                  &protector);
+                                                         &protector);
-   if (result != TSI_OK) return result;
+   if (status != TSI_OK) return status;
 
    // Do not forget to unprotect outstanding data if any.
-   if (buf_size > 0) {
+   unsigned char *unused_bytes = NULL;
-     result = tsi_frame_protector_unprotect(protector, buf + buf_offset,
+   size_t unused_bytes_size = 0;
-                                            buf_size, ..., ...);
+   status = tsi_handshaker_result_get_unused_bytes(result, &unused_bytes,
+                                                   &unused_bytes_size);
+   if (status != TSI_OK) return status;
+   if (unused_bytes_size > 0) {
+     status = tsi_frame_protector_unprotect(protector, unused_bytes,
+                                            unused_bytes_size, ..., ...);
      ....
    }
    ...
+   ------------------------------------------------------------------------
+
+   A typical usage of the new TSI would be as follows, supporting both
+   synchronous and asynchrnous TSI handshaker implementations:
+
+   ------------------------------------------------------------------------
+
+   typedef struct {
+     tsi_handshaker *handshaker;
+     tsi_handshaker_result *handshaker_result;
+     unsigned char *handshake_buffer;
+     size_t handshake_buffer_size;
+     ...
+   } security_handshaker;
+
+   void do_handshake(security_handshaker *h, ...) {
+     // Start the handshake by the calling do_handshake_next.
+     do_handshake_next(h, NULL, 0);
+     ...
+   }
+
+   // This method is a wrapper of the callback function to execute when
+   // tsi_handshaker_next finishes. It is passed to tsi_handshaker_next as
+   // the callback function.
+   void on_handshake_next_done_wrapper(
+       tsi_result status, void *user_data, const unsigned char *bytes_to_send,
+       size_t bytes_to_send_size, tsi_handshaker_result *result) {
+     security_handshaker *h = (security_handshaker *)user_data;
+     on_handshake_next_done(h, status, bytes_to_send,
+                            bytes_to_send_size, result);
+   }
+
+   // This method is the callback function when there are data received from
+   // the peer. This method will read bytes into the handshake buffer and call
+   // do_handshake_next.
+   void on_handshake_data_received_from_peer(void *user_data) {
+     security_handshaker *h = (security_handshaker *)user_data;
+     size_t bytes_received_size = h->handshake_buffer_size;
+     read_bytes_from_peer(h->handshake_buffer, &bytes_received_size);
+     do_handshake_next(h, h->handshake_buffer, bytes_received_size);
+   }
+
+   // This method processes a step of handshake, calling tsi_handshaker_next.
+   void do_handshake_next(security_handshaker *h,
+                          const unsigned char* bytes_received,
+                          size_t bytes_received_size) {
+     tsi_result status = TSI_OK;
+     unsigned char *bytes_to_send = NULL;
+     size_t bytes_to_send_size = 0;
+     tsi_handshaker_result *result = NULL;
+     status = tsi_handshaker_next(
+         handshaker, bytes_received, bytes_received_size, &bytes_to_send,
+         &bytes_to_send_size, &result, on_handshake_next_done_wrapper, h);
+     // If TSI handshaker is asynchronous, on_handshake_next_done will be
+     // called during the execution of the callback function.
+     if (status == TSI_ASYNC) return;
+     on_handshake_next_done(h, status, bytes_to_send,
+                            bytes_to_send_size, result);
+   }
+
+   // This is the real function to execute after tsi_handshaker_next.
+   void on_handshake_next_done(
+       security_handshaker *h, tsi_result status,
+       const unsigned char *bytes_to_send, size_t bytes_to_send_size,
+       tsi_handshaker_result *result) {
+     if (status == TSI_INCOMPLETE_DATA) {
+       // Schedule an asynchronous read from the peer. If handshake data are
+       // received, on_handshake_data_received_from_peer will be called.
+       async_read_from_peer(..., ..., on_handshake_data_received_from_peer);
+       return;
+     }
+     if (status != TSI_OK) return;
+
+     if (bytes_to_send_size > 0) {
+       send_bytes_to_peer(bytes_to_send, bytes_to_send_size);
+     }
+
+     if (result != NULL) {
+       // Handshake completed.
+       h->result = result;
+       // Check the Peer.
+       tsi_peer peer;
+       status = tsi_handshaker_result_extract_peer(result, &peer);
+       if (status != TSI_OK) return status;
+       status = check_peer(&peer);
+       tsi_peer_destruct(&peer);
+       if (status != TSI_OK) return;
+
+       // Create the protector.
+       tsi_frame_protector* protector = NULL;
+       status = tsi_handshaker_result_create_frame_protector(result, NULL,
+                                                             &protector);
+       if (status != TSI_OK) return;
+
+       // Do not forget to unprotect outstanding data if any.
+       ....
+     }
+   }
    ------------------------------------------------------------------------   */
 typedef struct tsi_handshaker tsi_handshaker;
 
-/* Gets bytes that need to be sent to the peer.
+/* TO BE DEPRECATED SOON.
+   Gets bytes that need to be sent to the peer.
    - bytes is the buffer that will be written with the data to be sent to the
      peer.
    - bytes_size is an input/output parameter specifying the capacity of the
@@ -292,7 +420,8 @@ tsi_result tsi_handshaker_get_bytes_to_send_to_peer(tsi_handshaker *self,
                                                     unsigned char *bytes,
                                                     size_t *bytes_size);
 
-/* Processes bytes received from the peer.
+/* TO BE DEPRECATED SOON.
+   Processes bytes received from the peer.
    - bytes is the buffer containing the data.
    - bytes_size is an input/output parameter specifying the size of the data as
      input and the number of bytes consumed as output.
@@ -305,24 +434,28 @@ tsi_result tsi_handshaker_process_bytes_from_peer(tsi_handshaker *self,
                                                   const unsigned char *bytes,
                                                   size_t *bytes_size);
 
-/* Gets the result of the handshaker.
+/* TO BE DEPRECATED SOON.
+   Gets the result of the handshaker.
    Returns TSI_OK if the hanshake completed successfully and there has been no
    errors. Returns TSI_HANDSHAKE_IN_PROGRESS if the handshaker is not done yet
    but no error has been encountered so far. Otherwise the handshaker failed
    with the returned error.  */
 tsi_result tsi_handshaker_get_result(tsi_handshaker *self);
 
-/* Returns 1 if the handshake is in progress, 0 otherwise.  */
+/* TO BE DEPRECATED SOON.
+   Returns 1 if the handshake is in progress, 0 otherwise.  */
 #define tsi_handshaker_is_in_progress(h) \
   (tsi_handshaker_get_result((h)) == TSI_HANDSHAKE_IN_PROGRESS)
 
-/* This method may return TSI_FAILED_PRECONDITION if
+/* TO BE DEPRECATED SOON.
+   This method may return TSI_FAILED_PRECONDITION if
    tsi_handshaker_is_in_progress returns 1, it returns TSI_OK otherwise
    assuming the handshaker is not in a fatal error state.
    The caller is responsible for destructing the peer.  */
 tsi_result tsi_handshaker_extract_peer(tsi_handshaker *self, tsi_peer *peer);
 
-/* This method creates a tsi_frame_protector object after the handshake phase
+/* TO BE DEPRECATED SOON.
+   This method creates a tsi_frame_protector object after the handshake phase
    is done. After this method has been called successfully, the only method
    that can be called on this object is Destroy.
    - max_output_protected_frame_size is an input/output parameter specifying the
@@ -342,10 +475,52 @@ tsi_result tsi_handshaker_create_frame_protector(
     tsi_handshaker *self, size_t *max_output_protected_frame_size,
     tsi_frame_protector **protector);
 
+/* Callback function definition for tsi_handshaker_next.
+   - status indicates the status of the next operation.
+   - user_data is the argument to callback function passed from the caller.
+   - bytes_to_send is the data buffer to be sent to the peer.
+   - bytes_to_send_size is the size of data buffer to be sent to the peer.
+   - handshaker_result is the result of handshake when the handshake completes,
+     is NULL otherwise.  */
+typedef void (*tsi_handshaker_on_next_done_cb)(
+    tsi_result status, void *user_data, const unsigned char *bytes_to_send,
+    size_t bytes_to_send_size, tsi_handshaker_result *handshaker_result);
+
+/* Conduct a next step of the handshake.
+   - received_bytes is the buffer containing the data received from the peer.
+   - received_bytes_size is the size of the data received from the peer.
+   - bytes_to_send is the data buffer to be sent to the peer.
+   - bytes_to_send_size is the size of data buffer to be sent to the peer.
+   - handshaker_result is the result of handshake if the handshake completes.
+   - cb is the callback function defined above. It can be NULL for synchronous
+     TSI handshaker implementation.
+   - user_data is the argument to callback function passed from the caller.
+   This method returns TSI_ASYNC if the TSI handshaker implementation is
+   asynchronous. It returns TSI_OK if the handshake completes or if there are
+   data to send to the peer, otherwise returns TSI_INCOMPLETE_DATA which
+   indicates that this method needs to be called again with more data from the
+   peer. In case of a fatal error in the handshake, another specific error code
+   is returned.
+   The caller is responsible for destroying the handshaker_result. However, the
+   caller should not free bytes_to_send, as the buffer is owned by the
+   tsi_handshaker object.  */
+tsi_result tsi_handshaker_next(
+    tsi_handshaker *self, const unsigned char *received_bytes,
+    size_t received_bytes_size, unsigned char **bytes_to_send,
+    size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result,
+    tsi_handshaker_on_next_done_cb cb, void *user_data);
+
 /* This method releases the tsi_handshaker object. After this method is called,
    no other method can be called on the object.  */
 void tsi_handshaker_destroy(tsi_handshaker *self);
 
+/* This method initializes the necessary shared objects used for tsi
+   implementation.  */
+void tsi_init();
+
+/* This method destroys the shared objects created by tsi_init.  */
+void tsi_destroy();
+
 #ifdef __cplusplus
 }
 #endif

test/core/tsi/transport_security_test.c

@@ -376,6 +376,8 @@ static void test_handshaker_invalid_args(void) {
              TSI_INVALID_ARGUMENT);
   GPR_ASSERT(tsi_handshaker_get_bytes_to_send_to_peer(NULL, NULL, NULL) ==
              TSI_INVALID_ARGUMENT);
+  GPR_ASSERT(tsi_handshaker_next(NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL) ==
+             TSI_INVALID_ARGUMENT);
 }
 
 static void test_handshaker_invalid_state(void) {