Merge pull request #12308 from jiangtaoli2016/fake_zero_copy_protector

Fake zero copy protector

src/core/lib/security/transport/secure_endpoint.c

@@ -34,7 +34,7 @@
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
 #include "src/core/lib/support/string.h"
-#include "src/core/tsi/transport_security_interface.h"
+#include "src/core/tsi/transport_security_grpc.h"
 
 #define STAGING_BUFFER_SIZE 8192
 
@@ -42,6 +42,7 @@ typedef struct {
   grpc_endpoint base;
   grpc_endpoint *wrapped_ep;
   struct tsi_frame_protector *protector;
+  struct tsi_zero_copy_grpc_protector *zero_copy_protector;
   gpr_mu protector_mu;
   /* saved upper level callbacks and user_data. */
   grpc_closure *read_cb;
@@ -67,6 +68,7 @@ static void destroy(grpc_exec_ctx *exec_ctx, secure_endpoint *secure_ep) {
   secure_endpoint *ep = secure_ep;
   grpc_endpoint_destroy(exec_ctx, ep->wrapped_ep);
   tsi_frame_protector_destroy(ep->protector);
+  tsi_zero_copy_grpc_protector_destroy(ep->zero_copy_protector);
   grpc_slice_buffer_destroy_internal(exec_ctx, &ep->leftover_bytes);
   grpc_slice_unref_internal(exec_ctx, ep->read_staging_buffer);
   grpc_slice_unref_internal(exec_ctx, ep->write_staging_buffer);
@@ -159,6 +161,12 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data,
     return;
   }
 
+  if (ep->zero_copy_protector != NULL) {
+    // Use zero-copy grpc protector to unprotect.
+    result = tsi_zero_copy_grpc_protector_unprotect(
+        ep->zero_copy_protector, &ep->source_buffer, ep->read_buffer);
+  } else {
+    // Use frame protector to unprotect.
     /* TODO(yangg) check error, maybe bail out early */
     for (i = 0; i < ep->source_buffer.count; i++) {
       grpc_slice encrypted = ep->source_buffer.slices[i];
@@ -169,8 +177,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data,
         size_t unprotected_buffer_size_written = (size_t)(end - cur);
         size_t processed_message_size = message_size;
         gpr_mu_lock(&ep->protector_mu);
-      result = tsi_frame_protector_unprotect(ep->protector, message_bytes,
-                                             &processed_message_size, cur,
+        result = tsi_frame_protector_unprotect(
+            ep->protector, message_bytes, &processed_message_size, cur,
             &unprotected_buffer_size_written);
         gpr_mu_unlock(&ep->protector_mu);
         if (result != TSI_OK) {
@@ -184,10 +192,10 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data,
 
         if (cur == end) {
           flush_read_staging_buffer(ep, &cur, &end);
-        /* Force to enter the loop again to extract buffered bytes in protector.
-           The bytes could be buffered because of running out of staging_buffer.
-           If this happens at the end of all slices, doing another unprotect
-           avoids leaving data in the protector. */
+          /* Force to enter the loop again to extract buffered bytes in
+             protector. The bytes could be buffered because of running out of
+             staging_buffer. If this happens at the end of all slices, doing
+             another unprotect avoids leaving data in the protector. */
           keep_looping = 1;
         } else if (unprotected_buffer_size_written > 0) {
           keep_looping = 1;
@@ -205,6 +213,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data,
               &ep->read_staging_buffer,
               (size_t)(cur - GRPC_SLICE_START_PTR(ep->read_staging_buffer))));
     }
+  }
 
   /* TODO(yangg) experiment with moving this block after read_cb to see if it
      helps latency */
@@ -270,6 +279,12 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
     }
   }
 
+  if (ep->zero_copy_protector != NULL) {
+    // Use zero-copy grpc protector to protect.
+    result = tsi_zero_copy_grpc_protector_protect(ep->zero_copy_protector,
+                                                  slices, &ep->output_buffer);
+  } else {
+    // Use frame protector to protect.
     for (i = 0; i < slices->count; i++) {
       grpc_slice plain = slices->slices[i];
       uint8_t *message_bytes = GRPC_SLICE_START_PTR(plain);
@@ -302,8 +317,8 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
       do {
         size_t protected_buffer_size_to_send = (size_t)(end - cur);
         gpr_mu_lock(&ep->protector_mu);
-      result = tsi_frame_protector_protect_flush(ep->protector, cur,
-                                                 &protected_buffer_size_to_send,
+        result = tsi_frame_protector_protect_flush(
+            ep->protector, cur, &protected_buffer_size_to_send,
             &still_pending_size);
         gpr_mu_unlock(&ep->protector_mu);
         if (result != TSI_OK) break;
@@ -317,7 +332,9 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
             &ep->output_buffer,
             grpc_slice_split_head(
                 &ep->write_staging_buffer,
-              (size_t)(cur - GRPC_SLICE_START_PTR(ep->write_staging_buffer))));
+                (size_t)(cur -
+                         GRPC_SLICE_START_PTR(ep->write_staging_buffer))));
+      }
     }
   }
 
@@ -389,13 +406,16 @@ static const grpc_endpoint_vtable vtable = {endpoint_read,
                                             endpoint_get_fd};
 
 grpc_endpoint *grpc_secure_endpoint_create(
-    struct tsi_frame_protector *protector, grpc_endpoint *transport,
-    grpc_slice *leftover_slices, size_t leftover_nslices) {
+    struct tsi_frame_protector *protector,
+    struct tsi_zero_copy_grpc_protector *zero_copy_protector,
+    grpc_endpoint *transport, grpc_slice *leftover_slices,
+    size_t leftover_nslices) {
   size_t i;
   secure_endpoint *ep = (secure_endpoint *)gpr_malloc(sizeof(secure_endpoint));
   ep->base.vtable = &vtable;
   ep->wrapped_ep = transport;
   ep->protector = protector;
+  ep->zero_copy_protector = zero_copy_protector;
   grpc_slice_buffer_init(&ep->leftover_bytes);
   for (i = 0; i < leftover_nslices; i++) {
     grpc_slice_buffer_add(&ep->leftover_bytes,

src/core/lib/security/transport/secure_endpoint.h

@@ -23,12 +23,17 @@
 #include "src/core/lib/iomgr/endpoint.h"
 
 struct tsi_frame_protector;
+struct tsi_zero_copy_grpc_protector;
 
 extern grpc_tracer_flag grpc_trace_secure_endpoint;
 
-/* Takes ownership of protector and to_wrap, and refs leftover_slices. */
+/* Takes ownership of protector, zero_copy_protector, and to_wrap, and refs
+ * leftover_slices. If zero_copy_protector is not NULL, protector will never be
+ * used. */
 grpc_endpoint *grpc_secure_endpoint_create(
-    struct tsi_frame_protector *protector, grpc_endpoint *to_wrap,
-    grpc_slice *leftover_slices, size_t leftover_nslices);
+    struct tsi_frame_protector *protector,
+    struct tsi_zero_copy_grpc_protector *zero_copy_protector,
+    grpc_endpoint *to_wrap, grpc_slice *leftover_slices,
+    size_t leftover_nslices);
 
 #endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H */

src/core/lib/security/transport/security_handshaker.c

@@ -32,6 +32,7 @@
 #include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/security/transport/tsi_error.h"
 #include "src/core/lib/slice/slice_internal.h"
+#include "src/core/tsi/transport_security_grpc.h"
 
 #define GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE 256
 
@@ -135,17 +136,31 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *arg,
     security_handshake_failed_locked(exec_ctx, h, GRPC_ERROR_REF(error));
     goto done;
   }
-  // Create frame protector.
-  tsi_frame_protector *protector;
-  tsi_result result = tsi_handshaker_result_create_frame_protector(
-      h->handshaker_result, NULL, &protector);
-  if (result != TSI_OK) {
+  // Create zero-copy frame protector, if implemented.
+  tsi_zero_copy_grpc_protector *zero_copy_protector = NULL;
+  tsi_result result = tsi_handshaker_result_create_zero_copy_grpc_protector(
+      h->handshaker_result, NULL, &zero_copy_protector);
+  if (result != TSI_OK && result != TSI_UNIMPLEMENTED) {
     error = grpc_set_tsi_error_result(
-        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Frame protector creation failed"),
+        GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+            "Zero-copy frame protector creation failed"),
+        result);
+    security_handshake_failed_locked(exec_ctx, h, error);
+    goto done;
+  }
+  // Create frame protector if zero-copy frame protector is NULL.
+  tsi_frame_protector *protector = NULL;
+  if (zero_copy_protector == NULL) {
+    result = tsi_handshaker_result_create_frame_protector(h->handshaker_result,
+                                                          NULL, &protector);
+    if (result != TSI_OK) {
+      error = grpc_set_tsi_error_result(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                                            "Frame protector creation failed"),
                                         result);
       security_handshake_failed_locked(exec_ctx, h, error);
       goto done;
     }
+  }
   // Get unused bytes.
   const unsigned char *unused_bytes = NULL;
   size_t unused_bytes_size = 0;
@@ -155,12 +170,12 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *arg,
   if (unused_bytes_size > 0) {
     grpc_slice slice =
         grpc_slice_from_copied_buffer((char *)unused_bytes, unused_bytes_size);
-    h->args->endpoint =
-        grpc_secure_endpoint_create(protector, h->args->endpoint, &slice, 1);
+    h->args->endpoint = grpc_secure_endpoint_create(
+        protector, zero_copy_protector, h->args->endpoint, &slice, 1);
     grpc_slice_unref_internal(exec_ctx, slice);
   } else {
-    h->args->endpoint =
-        grpc_secure_endpoint_create(protector, h->args->endpoint, NULL, 0);
+    h->args->endpoint = grpc_secure_endpoint_create(
+        protector, zero_copy_protector, h->args->endpoint, NULL, 0);
   }
   tsi_handshaker_result_destroy(h->handshaker_result);
   h->handshaker_result = NULL;

src/core/tsi/fake_transport_security.c

@@ -25,7 +25,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/useful.h>
-#include "src/core/tsi/transport_security.h"
+#include "src/core/tsi/transport_security_grpc.h"
 
 /* --- Constants. ---*/
 #define TSI_FAKE_FRAME_HEADER_SIZE 4
@@ -74,6 +74,14 @@ typedef struct {
   size_t max_frame_size;
 } tsi_fake_frame_protector;
 
+typedef struct {
+  tsi_zero_copy_grpc_protector base;
+  grpc_slice_buffer header_sb;
+  grpc_slice_buffer protected_sb;
+  size_t max_frame_size;
+  size_t parsed_frame_size;
+} tsi_fake_zero_copy_grpc_protector;
+
 /* --- Utils. ---*/
 
 static const char *tsi_fake_handshake_message_strings[] = {
@@ -113,6 +121,28 @@ static void store32_little_endian(uint32_t value, unsigned char *buf) {
   buf[0] = (unsigned char)((value)&0xFF);
 }
 
+static uint32_t read_frame_size(const grpc_slice_buffer *sb) {
+  GPR_ASSERT(sb != NULL && sb->length >= TSI_FAKE_FRAME_HEADER_SIZE);
+  uint8_t frame_size_buffer[TSI_FAKE_FRAME_HEADER_SIZE];
+  uint8_t *buf = frame_size_buffer;
+  /* Copies the first 4 bytes to a temporary buffer.  */
+  size_t remaining = TSI_FAKE_FRAME_HEADER_SIZE;
+  for (size_t i = 0; i < sb->count; i++) {
+    size_t slice_length = GRPC_SLICE_LENGTH(sb->slices[i]);
+    if (remaining <= slice_length) {
+      memcpy(buf, GRPC_SLICE_START_PTR(sb->slices[i]), remaining);
+      remaining = 0;
+      break;
+    } else {
+      memcpy(buf, GRPC_SLICE_START_PTR(sb->slices[i]), slice_length);
+      buf += slice_length;
+      remaining -= slice_length;
+    }
+  }
+  GPR_ASSERT(remaining == 0);
+  return load32_little_endian(frame_size_buffer);
+}
+
 static void tsi_fake_frame_reset(tsi_fake_frame *frame, int needs_draining) {
   frame->offset = 0;
   frame->needs_draining = needs_draining;
@@ -363,6 +393,82 @@ static const tsi_frame_protector_vtable frame_protector_vtable = {
     fake_protector_unprotect, fake_protector_destroy,
 };
 
+/* --- tsi_zero_copy_grpc_protector methods implementation. ---*/
+
+static tsi_result fake_zero_copy_grpc_protector_protect(
+    tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *unprotected_slices,
+    grpc_slice_buffer *protected_slices) {
+  if (self == NULL || unprotected_slices == NULL || protected_slices == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
+  tsi_fake_zero_copy_grpc_protector *impl =
+      (tsi_fake_zero_copy_grpc_protector *)self;
+  /* Protects each frame. */
+  while (unprotected_slices->length > 0) {
+    size_t frame_length =
+        GPR_MIN(impl->max_frame_size,
+                unprotected_slices->length + TSI_FAKE_FRAME_HEADER_SIZE);
+    grpc_slice slice = grpc_slice_malloc(TSI_FAKE_FRAME_HEADER_SIZE);
+    store32_little_endian((uint32_t)frame_length, GRPC_SLICE_START_PTR(slice));
+    grpc_slice_buffer_add(protected_slices, slice);
+    size_t data_length = frame_length - TSI_FAKE_FRAME_HEADER_SIZE;
+    grpc_slice_buffer_move_first(unprotected_slices, data_length,
+                                 protected_slices);
+  }
+  return TSI_OK;
+}
+
+static tsi_result fake_zero_copy_grpc_protector_unprotect(
+    tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *protected_slices,
+    grpc_slice_buffer *unprotected_slices) {
+  if (self == NULL || unprotected_slices == NULL || protected_slices == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
+  tsi_fake_zero_copy_grpc_protector *impl =
+      (tsi_fake_zero_copy_grpc_protector *)self;
+  grpc_slice_buffer_move_into(protected_slices, &impl->protected_sb);
+  /* Unprotect each frame, if we get a full frame. */
+  while (impl->protected_sb.length >= TSI_FAKE_FRAME_HEADER_SIZE) {
+    if (impl->parsed_frame_size == 0) {
+      impl->parsed_frame_size = read_frame_size(&impl->protected_sb);
+      if (impl->parsed_frame_size <= 4) {
+        gpr_log(GPR_ERROR, "Invalid frame size.");
+        return TSI_DATA_CORRUPTED;
+      }
+    }
+    /* If we do not have a full frame, return with OK status. */
+    if (impl->protected_sb.length < impl->parsed_frame_size) break;
+    /* Strips header bytes. */
+    grpc_slice_buffer_move_first(&impl->protected_sb,
+                                 TSI_FAKE_FRAME_HEADER_SIZE, &impl->header_sb);
+    /* Moves data to unprotected slices. */
+    grpc_slice_buffer_move_first(
+        &impl->protected_sb,
+        impl->parsed_frame_size - TSI_FAKE_FRAME_HEADER_SIZE,
+        unprotected_slices);
+    impl->parsed_frame_size = 0;
+    grpc_slice_buffer_reset_and_unref(&impl->header_sb);
+  }
+  return TSI_OK;
+}
+
+static void fake_zero_copy_grpc_protector_destroy(
+    tsi_zero_copy_grpc_protector *self) {
+  if (self == NULL) return;
+  tsi_fake_zero_copy_grpc_protector *impl =
+      (tsi_fake_zero_copy_grpc_protector *)self;
+  grpc_slice_buffer_destroy(&impl->header_sb);
+  grpc_slice_buffer_destroy(&impl->protected_sb);
+  gpr_free(impl);
+}
+
+static const tsi_zero_copy_grpc_protector_vtable
+    zero_copy_grpc_protector_vtable = {
+        fake_zero_copy_grpc_protector_protect,
+        fake_zero_copy_grpc_protector_unprotect,
+        fake_zero_copy_grpc_protector_destroy,
+};
+
 /* --- tsi_handshaker_result methods implementation. ---*/
 
 typedef struct {
@@ -383,6 +489,14 @@ static tsi_result fake_handshaker_result_extract_peer(
   return result;
 }
 
+static tsi_result fake_handshaker_result_create_zero_copy_grpc_protector(
+    const tsi_handshaker_result *self, size_t *max_output_protected_frame_size,
+    tsi_zero_copy_grpc_protector **protector) {
+  *protector =
+      tsi_create_fake_zero_copy_grpc_protector(max_output_protected_frame_size);
+  return TSI_OK;
+}
+
 static tsi_result fake_handshaker_result_create_frame_protector(
     const tsi_handshaker_result *self, size_t *max_output_protected_frame_size,
     tsi_frame_protector **protector) {
@@ -407,7 +521,7 @@ static void fake_handshaker_result_destroy(tsi_handshaker_result *self) {
 
 static const tsi_handshaker_result_vtable handshaker_result_vtable = {
     fake_handshaker_result_extract_peer,
-    NULL, /* create_zero_copy_grpc_protector */
+    fake_handshaker_result_create_zero_copy_grpc_protector,
     fake_handshaker_result_create_frame_protector,
     fake_handshaker_result_get_unused_bytes,
     fake_handshaker_result_destroy,
@@ -631,3 +745,16 @@ tsi_frame_protector *tsi_create_fake_frame_protector(
   impl->base.vtable = &frame_protector_vtable;
   return &impl->base;
 }
+
+tsi_zero_copy_grpc_protector *tsi_create_fake_zero_copy_grpc_protector(
+    size_t *max_protected_frame_size) {
+  tsi_fake_zero_copy_grpc_protector *impl = gpr_zalloc(sizeof(*impl));
+  grpc_slice_buffer_init(&impl->header_sb);
+  grpc_slice_buffer_init(&impl->protected_sb);
+  impl->max_frame_size = (max_protected_frame_size == NULL)
+                             ? TSI_FAKE_DEFAULT_FRAME_SIZE
+                             : *max_protected_frame_size;
+  impl->parsed_frame_size = 0;
+  impl->base.vtable = &zero_copy_grpc_protector_vtable;
+  return &impl->base;
+}

src/core/tsi/fake_transport_security.h

@@ -39,6 +39,11 @@ tsi_handshaker *tsi_create_fake_handshaker(int is_client);
 tsi_frame_protector *tsi_create_fake_frame_protector(
     size_t *max_protected_frame_size);
 
+/* Creates a zero-copy protector directly without going through the handshake
+ * phase. */
+tsi_zero_copy_grpc_protector *tsi_create_fake_zero_copy_grpc_protector(
+    size_t *max_protected_frame_size);
+
 #ifdef __cplusplus
 }
 #endif

test/core/security/secure_endpoint_test.c

@@ -36,12 +36,17 @@ static gpr_mu *g_mu;
 static grpc_pollset *g_pollset;
 
 static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair(
-    size_t slice_size, grpc_slice *leftover_slices, size_t leftover_nslices) {
+    size_t slice_size, grpc_slice *leftover_slices, size_t leftover_nslices,
+    bool use_zero_copy_protector) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   tsi_frame_protector *fake_read_protector =
       tsi_create_fake_frame_protector(NULL);
   tsi_frame_protector *fake_write_protector =
       tsi_create_fake_frame_protector(NULL);
+  tsi_zero_copy_grpc_protector *fake_read_zero_copy_protector =
+      tsi_create_fake_zero_copy_grpc_protector(NULL);
+  tsi_zero_copy_grpc_protector *fake_write_zero_copy_protector =
+      tsi_create_fake_zero_copy_grpc_protector(NULL);
   grpc_endpoint_test_fixture f;
   grpc_endpoint_pair tcp;
 
@@ -55,7 +60,11 @@ static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair(
 
   if (leftover_nslices == 0) {
     f.client_ep =
-        grpc_secure_endpoint_create(fake_read_protector, tcp.client, NULL, 0);
+        use_zero_copy_protector
+            ? grpc_secure_endpoint_create(NULL, fake_read_zero_copy_protector,
+                                          tcp.client, NULL, 0)
+            : grpc_secure_endpoint_create(fake_read_protector, NULL, tcp.client,
+                                          NULL, 0);
   } else {
     unsigned i;
     tsi_result result;
@@ -96,31 +105,53 @@ static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair(
     } while (still_pending_size > 0);
     encrypted_leftover = grpc_slice_from_copied_buffer(
         (const char *)encrypted_buffer, total_buffer_size - buffer_size);
-    f.client_ep = grpc_secure_endpoint_create(fake_read_protector, tcp.client,
+    f.client_ep =
+        use_zero_copy_protector
+            ? grpc_secure_endpoint_create(NULL, fake_read_zero_copy_protector,
+                                          tcp.client, &encrypted_leftover, 1)
+            : grpc_secure_endpoint_create(fake_read_protector, NULL, tcp.client,
                                           &encrypted_leftover, 1);
     grpc_slice_unref(encrypted_leftover);
     gpr_free(encrypted_buffer);
   }
 
   f.server_ep =
-      grpc_secure_endpoint_create(fake_write_protector, tcp.server, NULL, 0);
+      use_zero_copy_protector
+          ? grpc_secure_endpoint_create(NULL, fake_write_zero_copy_protector,
+                                        tcp.server, NULL, 0)
+          : grpc_secure_endpoint_create(fake_write_protector, NULL, tcp.server,
+                                        NULL, 0);
   grpc_exec_ctx_finish(&exec_ctx);
   return f;
 }
 
 static grpc_endpoint_test_fixture
 secure_endpoint_create_fixture_tcp_socketpair_noleftover(size_t slice_size) {
-  return secure_endpoint_create_fixture_tcp_socketpair(slice_size, NULL, 0);
+  return secure_endpoint_create_fixture_tcp_socketpair(slice_size, NULL, 0,
+                                                       false);
+}
+
+static grpc_endpoint_test_fixture
+secure_endpoint_create_fixture_tcp_socketpair_noleftover_zero_copy(
+    size_t slice_size) {
+  return secure_endpoint_create_fixture_tcp_socketpair(slice_size, NULL, 0,
+                                                       true);
 }
 
 static grpc_endpoint_test_fixture
 secure_endpoint_create_fixture_tcp_socketpair_leftover(size_t slice_size) {
   grpc_slice s =
       grpc_slice_from_copied_string("hello world 12345678900987654321");
-  grpc_endpoint_test_fixture f;
+  return secure_endpoint_create_fixture_tcp_socketpair(slice_size, &s, 1,
+                                                       false);
+}
 
-  f = secure_endpoint_create_fixture_tcp_socketpair(slice_size, &s, 1);
-  return f;
+static grpc_endpoint_test_fixture
+secure_endpoint_create_fixture_tcp_socketpair_leftover_zero_copy(
+    size_t slice_size) {
+  grpc_slice s =
+      grpc_slice_from_copied_string("hello world 12345678900987654321");
+  return secure_endpoint_create_fixture_tcp_socketpair(slice_size, &s, 1, true);
 }
 
 static void clean_up(void) {}
@@ -128,8 +159,14 @@ static void clean_up(void) {}
 static grpc_endpoint_test_config configs[] = {
     {"secure_ep/tcp_socketpair",
      secure_endpoint_create_fixture_tcp_socketpair_noleftover, clean_up},
+    {"secure_ep/tcp_socketpair_zero_copy",
+     secure_endpoint_create_fixture_tcp_socketpair_noleftover_zero_copy,
+     clean_up},
     {"secure_ep/tcp_socketpair_leftover",
      secure_endpoint_create_fixture_tcp_socketpair_leftover, clean_up},
+    {"secure_ep/tcp_socketpair_leftover_zero_copy",
+     secure_endpoint_create_fixture_tcp_socketpair_leftover_zero_copy,
+     clean_up},
 };
 
 static void inc_call_ctr(grpc_exec_ctx *exec_ctx, void *arg,
@@ -184,7 +221,9 @@ int main(int argc, char **argv) {
   g_pollset = gpr_zalloc(grpc_pollset_size());
   grpc_pollset_init(g_pollset, &g_mu);
   grpc_endpoint_tests(configs[0], g_pollset, g_mu);
-  test_leftover(configs[1], 1);
+  grpc_endpoint_tests(configs[1], g_pollset, g_mu);
+  test_leftover(configs[2], 1);
+  test_leftover(configs[3], 1);
   GRPC_CLOSURE_INIT(&destroyed, destroy_pollset, g_pollset,
                     grpc_schedule_on_exec_ctx);
   grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed);