diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c index 5e41b94ff82..e0520eb72eb 100644 --- a/src/core/lib/security/transport/secure_endpoint.c +++ b/src/core/lib/security/transport/secure_endpoint.c @@ -34,7 +34,7 @@ #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" -#include "src/core/tsi/transport_security_interface.h" +#include "src/core/tsi/transport_security_grpc.h" #define STAGING_BUFFER_SIZE 8192 @@ -42,6 +42,7 @@ typedef struct { grpc_endpoint base; grpc_endpoint *wrapped_ep; struct tsi_frame_protector *protector; + struct tsi_zero_copy_grpc_protector *zero_copy_protector; gpr_mu protector_mu; /* saved upper level callbacks and user_data. */ grpc_closure *read_cb; @@ -67,6 +68,7 @@ static void destroy(grpc_exec_ctx *exec_ctx, secure_endpoint *secure_ep) { secure_endpoint *ep = secure_ep; grpc_endpoint_destroy(exec_ctx, ep->wrapped_ep); tsi_frame_protector_destroy(ep->protector); + tsi_zero_copy_grpc_protector_destroy(ep->zero_copy_protector); grpc_slice_buffer_destroy_internal(exec_ctx, &ep->leftover_bytes); grpc_slice_unref_internal(exec_ctx, ep->read_staging_buffer); grpc_slice_unref_internal(exec_ctx, ep->write_staging_buffer); @@ -159,51 +161,58 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, return; } - /* TODO(yangg) check error, maybe bail out early */ - for (i = 0; i < ep->source_buffer.count; i++) { - grpc_slice encrypted = ep->source_buffer.slices[i]; - uint8_t *message_bytes = GRPC_SLICE_START_PTR(encrypted); - size_t message_size = GRPC_SLICE_LENGTH(encrypted); - - while (message_size > 0 || keep_looping) { - size_t unprotected_buffer_size_written = (size_t)(end - cur); - size_t processed_message_size = message_size; - gpr_mu_lock(&ep->protector_mu); - result = tsi_frame_protector_unprotect(ep->protector, message_bytes, - &processed_message_size, cur, - &unprotected_buffer_size_written); - gpr_mu_unlock(&ep->protector_mu); - if (result != TSI_OK) { - gpr_log(GPR_ERROR, "Decryption error: %s", - tsi_result_to_string(result)); - break; - } - message_bytes += processed_message_size; - message_size -= processed_message_size; - cur += unprotected_buffer_size_written; - - if (cur == end) { - flush_read_staging_buffer(ep, &cur, &end); - /* Force to enter the loop again to extract buffered bytes in protector. - The bytes could be buffered because of running out of staging_buffer. - If this happens at the end of all slices, doing another unprotect - avoids leaving data in the protector. */ - keep_looping = 1; - } else if (unprotected_buffer_size_written > 0) { - keep_looping = 1; - } else { - keep_looping = 0; + if (ep->zero_copy_protector != NULL) { + // Use zero-copy grpc protector to unprotect. + result = tsi_zero_copy_grpc_protector_unprotect( + ep->zero_copy_protector, &ep->source_buffer, ep->read_buffer); + } else { + // Use frame protector to unprotect. + /* TODO(yangg) check error, maybe bail out early */ + for (i = 0; i < ep->source_buffer.count; i++) { + grpc_slice encrypted = ep->source_buffer.slices[i]; + uint8_t *message_bytes = GRPC_SLICE_START_PTR(encrypted); + size_t message_size = GRPC_SLICE_LENGTH(encrypted); + + while (message_size > 0 || keep_looping) { + size_t unprotected_buffer_size_written = (size_t)(end - cur); + size_t processed_message_size = message_size; + gpr_mu_lock(&ep->protector_mu); + result = tsi_frame_protector_unprotect( + ep->protector, message_bytes, &processed_message_size, cur, + &unprotected_buffer_size_written); + gpr_mu_unlock(&ep->protector_mu); + if (result != TSI_OK) { + gpr_log(GPR_ERROR, "Decryption error: %s", + tsi_result_to_string(result)); + break; + } + message_bytes += processed_message_size; + message_size -= processed_message_size; + cur += unprotected_buffer_size_written; + + if (cur == end) { + flush_read_staging_buffer(ep, &cur, &end); + /* Force to enter the loop again to extract buffered bytes in + protector. The bytes could be buffered because of running out of + staging_buffer. If this happens at the end of all slices, doing + another unprotect avoids leaving data in the protector. */ + keep_looping = 1; + } else if (unprotected_buffer_size_written > 0) { + keep_looping = 1; + } else { + keep_looping = 0; + } } + if (result != TSI_OK) break; } - if (result != TSI_OK) break; - } - if (cur != GRPC_SLICE_START_PTR(ep->read_staging_buffer)) { - grpc_slice_buffer_add( - ep->read_buffer, - grpc_slice_split_head( - &ep->read_staging_buffer, - (size_t)(cur - GRPC_SLICE_START_PTR(ep->read_staging_buffer)))); + if (cur != GRPC_SLICE_START_PTR(ep->read_staging_buffer)) { + grpc_slice_buffer_add( + ep->read_buffer, + grpc_slice_split_head( + &ep->read_staging_buffer, + (size_t)(cur - GRPC_SLICE_START_PTR(ep->read_staging_buffer)))); + } } /* TODO(yangg) experiment with moving this block after read_cb to see if it @@ -270,54 +279,62 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, } } - for (i = 0; i < slices->count; i++) { - grpc_slice plain = slices->slices[i]; - uint8_t *message_bytes = GRPC_SLICE_START_PTR(plain); - size_t message_size = GRPC_SLICE_LENGTH(plain); - while (message_size > 0) { - size_t protected_buffer_size_to_send = (size_t)(end - cur); - size_t processed_message_size = message_size; - gpr_mu_lock(&ep->protector_mu); - result = tsi_frame_protector_protect(ep->protector, message_bytes, - &processed_message_size, cur, - &protected_buffer_size_to_send); - gpr_mu_unlock(&ep->protector_mu); - if (result != TSI_OK) { - gpr_log(GPR_ERROR, "Encryption error: %s", - tsi_result_to_string(result)); - break; - } - message_bytes += processed_message_size; - message_size -= processed_message_size; - cur += protected_buffer_size_to_send; - - if (cur == end) { - flush_write_staging_buffer(ep, &cur, &end); + if (ep->zero_copy_protector != NULL) { + // Use zero-copy grpc protector to protect. + result = tsi_zero_copy_grpc_protector_protect(ep->zero_copy_protector, + slices, &ep->output_buffer); + } else { + // Use frame protector to protect. + for (i = 0; i < slices->count; i++) { + grpc_slice plain = slices->slices[i]; + uint8_t *message_bytes = GRPC_SLICE_START_PTR(plain); + size_t message_size = GRPC_SLICE_LENGTH(plain); + while (message_size > 0) { + size_t protected_buffer_size_to_send = (size_t)(end - cur); + size_t processed_message_size = message_size; + gpr_mu_lock(&ep->protector_mu); + result = tsi_frame_protector_protect(ep->protector, message_bytes, + &processed_message_size, cur, + &protected_buffer_size_to_send); + gpr_mu_unlock(&ep->protector_mu); + if (result != TSI_OK) { + gpr_log(GPR_ERROR, "Encryption error: %s", + tsi_result_to_string(result)); + break; + } + message_bytes += processed_message_size; + message_size -= processed_message_size; + cur += protected_buffer_size_to_send; + + if (cur == end) { + flush_write_staging_buffer(ep, &cur, &end); + } } - } - if (result != TSI_OK) break; - } - if (result == TSI_OK) { - size_t still_pending_size; - do { - size_t protected_buffer_size_to_send = (size_t)(end - cur); - gpr_mu_lock(&ep->protector_mu); - result = tsi_frame_protector_protect_flush(ep->protector, cur, - &protected_buffer_size_to_send, - &still_pending_size); - gpr_mu_unlock(&ep->protector_mu); if (result != TSI_OK) break; - cur += protected_buffer_size_to_send; - if (cur == end) { - flush_write_staging_buffer(ep, &cur, &end); + } + if (result == TSI_OK) { + size_t still_pending_size; + do { + size_t protected_buffer_size_to_send = (size_t)(end - cur); + gpr_mu_lock(&ep->protector_mu); + result = tsi_frame_protector_protect_flush( + ep->protector, cur, &protected_buffer_size_to_send, + &still_pending_size); + gpr_mu_unlock(&ep->protector_mu); + if (result != TSI_OK) break; + cur += protected_buffer_size_to_send; + if (cur == end) { + flush_write_staging_buffer(ep, &cur, &end); + } + } while (still_pending_size > 0); + if (cur != GRPC_SLICE_START_PTR(ep->write_staging_buffer)) { + grpc_slice_buffer_add( + &ep->output_buffer, + grpc_slice_split_head( + &ep->write_staging_buffer, + (size_t)(cur - + GRPC_SLICE_START_PTR(ep->write_staging_buffer)))); } - } while (still_pending_size > 0); - if (cur != GRPC_SLICE_START_PTR(ep->write_staging_buffer)) { - grpc_slice_buffer_add( - &ep->output_buffer, - grpc_slice_split_head( - &ep->write_staging_buffer, - (size_t)(cur - GRPC_SLICE_START_PTR(ep->write_staging_buffer)))); } } @@ -389,13 +406,16 @@ static const grpc_endpoint_vtable vtable = {endpoint_read, endpoint_get_fd}; grpc_endpoint *grpc_secure_endpoint_create( - struct tsi_frame_protector *protector, grpc_endpoint *transport, - grpc_slice *leftover_slices, size_t leftover_nslices) { + struct tsi_frame_protector *protector, + struct tsi_zero_copy_grpc_protector *zero_copy_protector, + grpc_endpoint *transport, grpc_slice *leftover_slices, + size_t leftover_nslices) { size_t i; secure_endpoint *ep = (secure_endpoint *)gpr_malloc(sizeof(secure_endpoint)); ep->base.vtable = &vtable; ep->wrapped_ep = transport; ep->protector = protector; + ep->zero_copy_protector = zero_copy_protector; grpc_slice_buffer_init(&ep->leftover_bytes); for (i = 0; i < leftover_nslices; i++) { grpc_slice_buffer_add(&ep->leftover_bytes, diff --git a/src/core/lib/security/transport/secure_endpoint.h b/src/core/lib/security/transport/secure_endpoint.h index 1c5555f3df9..3323a6ff42a 100644 --- a/src/core/lib/security/transport/secure_endpoint.h +++ b/src/core/lib/security/transport/secure_endpoint.h @@ -23,12 +23,17 @@ #include "src/core/lib/iomgr/endpoint.h" struct tsi_frame_protector; +struct tsi_zero_copy_grpc_protector; extern grpc_tracer_flag grpc_trace_secure_endpoint; -/* Takes ownership of protector and to_wrap, and refs leftover_slices. */ +/* Takes ownership of protector, zero_copy_protector, and to_wrap, and refs + * leftover_slices. If zero_copy_protector is not NULL, protector will never be + * used. */ grpc_endpoint *grpc_secure_endpoint_create( - struct tsi_frame_protector *protector, grpc_endpoint *to_wrap, - grpc_slice *leftover_slices, size_t leftover_nslices); + struct tsi_frame_protector *protector, + struct tsi_zero_copy_grpc_protector *zero_copy_protector, + grpc_endpoint *to_wrap, grpc_slice *leftover_slices, + size_t leftover_nslices); #endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H */ diff --git a/src/core/lib/security/transport/security_handshaker.c b/src/core/lib/security/transport/security_handshaker.c index fc9c9f980f0..ea9608f4442 100644 --- a/src/core/lib/security/transport/security_handshaker.c +++ b/src/core/lib/security/transport/security_handshaker.c @@ -32,6 +32,7 @@ #include "src/core/lib/security/transport/secure_endpoint.h" #include "src/core/lib/security/transport/tsi_error.h" #include "src/core/lib/slice/slice_internal.h" +#include "src/core/tsi/transport_security_grpc.h" #define GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE 256 @@ -135,17 +136,31 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *arg, security_handshake_failed_locked(exec_ctx, h, GRPC_ERROR_REF(error)); goto done; } - // Create frame protector. - tsi_frame_protector *protector; - tsi_result result = tsi_handshaker_result_create_frame_protector( - h->handshaker_result, NULL, &protector); - if (result != TSI_OK) { + // Create zero-copy frame protector, if implemented. + tsi_zero_copy_grpc_protector *zero_copy_protector = NULL; + tsi_result result = tsi_handshaker_result_create_zero_copy_grpc_protector( + h->handshaker_result, NULL, &zero_copy_protector); + if (result != TSI_OK && result != TSI_UNIMPLEMENTED) { error = grpc_set_tsi_error_result( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Frame protector creation failed"), + GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Zero-copy frame protector creation failed"), result); security_handshake_failed_locked(exec_ctx, h, error); goto done; } + // Create frame protector if zero-copy frame protector is NULL. + tsi_frame_protector *protector = NULL; + if (zero_copy_protector == NULL) { + result = tsi_handshaker_result_create_frame_protector(h->handshaker_result, + NULL, &protector); + if (result != TSI_OK) { + error = grpc_set_tsi_error_result(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Frame protector creation failed"), + result); + security_handshake_failed_locked(exec_ctx, h, error); + goto done; + } + } // Get unused bytes. const unsigned char *unused_bytes = NULL; size_t unused_bytes_size = 0; @@ -155,12 +170,12 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *arg, if (unused_bytes_size > 0) { grpc_slice slice = grpc_slice_from_copied_buffer((char *)unused_bytes, unused_bytes_size); - h->args->endpoint = - grpc_secure_endpoint_create(protector, h->args->endpoint, &slice, 1); + h->args->endpoint = grpc_secure_endpoint_create( + protector, zero_copy_protector, h->args->endpoint, &slice, 1); grpc_slice_unref_internal(exec_ctx, slice); } else { - h->args->endpoint = - grpc_secure_endpoint_create(protector, h->args->endpoint, NULL, 0); + h->args->endpoint = grpc_secure_endpoint_create( + protector, zero_copy_protector, h->args->endpoint, NULL, 0); } tsi_handshaker_result_destroy(h->handshaker_result); h->handshaker_result = NULL; diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c index 967126ecee7..17bbd74002f 100644 --- a/src/core/tsi/fake_transport_security.c +++ b/src/core/tsi/fake_transport_security.c @@ -25,7 +25,7 @@ #include #include #include -#include "src/core/tsi/transport_security.h" +#include "src/core/tsi/transport_security_grpc.h" /* --- Constants. ---*/ #define TSI_FAKE_FRAME_HEADER_SIZE 4 @@ -74,6 +74,14 @@ typedef struct { size_t max_frame_size; } tsi_fake_frame_protector; +typedef struct { + tsi_zero_copy_grpc_protector base; + grpc_slice_buffer header_sb; + grpc_slice_buffer protected_sb; + size_t max_frame_size; + size_t parsed_frame_size; +} tsi_fake_zero_copy_grpc_protector; + /* --- Utils. ---*/ static const char *tsi_fake_handshake_message_strings[] = { @@ -113,6 +121,28 @@ static void store32_little_endian(uint32_t value, unsigned char *buf) { buf[0] = (unsigned char)((value)&0xFF); } +static uint32_t read_frame_size(const grpc_slice_buffer *sb) { + GPR_ASSERT(sb != NULL && sb->length >= TSI_FAKE_FRAME_HEADER_SIZE); + uint8_t frame_size_buffer[TSI_FAKE_FRAME_HEADER_SIZE]; + uint8_t *buf = frame_size_buffer; + /* Copies the first 4 bytes to a temporary buffer. */ + size_t remaining = TSI_FAKE_FRAME_HEADER_SIZE; + for (size_t i = 0; i < sb->count; i++) { + size_t slice_length = GRPC_SLICE_LENGTH(sb->slices[i]); + if (remaining <= slice_length) { + memcpy(buf, GRPC_SLICE_START_PTR(sb->slices[i]), remaining); + remaining = 0; + break; + } else { + memcpy(buf, GRPC_SLICE_START_PTR(sb->slices[i]), slice_length); + buf += slice_length; + remaining -= slice_length; + } + } + GPR_ASSERT(remaining == 0); + return load32_little_endian(frame_size_buffer); +} + static void tsi_fake_frame_reset(tsi_fake_frame *frame, int needs_draining) { frame->offset = 0; frame->needs_draining = needs_draining; @@ -363,6 +393,82 @@ static const tsi_frame_protector_vtable frame_protector_vtable = { fake_protector_unprotect, fake_protector_destroy, }; +/* --- tsi_zero_copy_grpc_protector methods implementation. ---*/ + +static tsi_result fake_zero_copy_grpc_protector_protect( + tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *unprotected_slices, + grpc_slice_buffer *protected_slices) { + if (self == NULL || unprotected_slices == NULL || protected_slices == NULL) { + return TSI_INVALID_ARGUMENT; + } + tsi_fake_zero_copy_grpc_protector *impl = + (tsi_fake_zero_copy_grpc_protector *)self; + /* Protects each frame. */ + while (unprotected_slices->length > 0) { + size_t frame_length = + GPR_MIN(impl->max_frame_size, + unprotected_slices->length + TSI_FAKE_FRAME_HEADER_SIZE); + grpc_slice slice = grpc_slice_malloc(TSI_FAKE_FRAME_HEADER_SIZE); + store32_little_endian((uint32_t)frame_length, GRPC_SLICE_START_PTR(slice)); + grpc_slice_buffer_add(protected_slices, slice); + size_t data_length = frame_length - TSI_FAKE_FRAME_HEADER_SIZE; + grpc_slice_buffer_move_first(unprotected_slices, data_length, + protected_slices); + } + return TSI_OK; +} + +static tsi_result fake_zero_copy_grpc_protector_unprotect( + tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *protected_slices, + grpc_slice_buffer *unprotected_slices) { + if (self == NULL || unprotected_slices == NULL || protected_slices == NULL) { + return TSI_INVALID_ARGUMENT; + } + tsi_fake_zero_copy_grpc_protector *impl = + (tsi_fake_zero_copy_grpc_protector *)self; + grpc_slice_buffer_move_into(protected_slices, &impl->protected_sb); + /* Unprotect each frame, if we get a full frame. */ + while (impl->protected_sb.length >= TSI_FAKE_FRAME_HEADER_SIZE) { + if (impl->parsed_frame_size == 0) { + impl->parsed_frame_size = read_frame_size(&impl->protected_sb); + if (impl->parsed_frame_size <= 4) { + gpr_log(GPR_ERROR, "Invalid frame size."); + return TSI_DATA_CORRUPTED; + } + } + /* If we do not have a full frame, return with OK status. */ + if (impl->protected_sb.length < impl->parsed_frame_size) break; + /* Strips header bytes. */ + grpc_slice_buffer_move_first(&impl->protected_sb, + TSI_FAKE_FRAME_HEADER_SIZE, &impl->header_sb); + /* Moves data to unprotected slices. */ + grpc_slice_buffer_move_first( + &impl->protected_sb, + impl->parsed_frame_size - TSI_FAKE_FRAME_HEADER_SIZE, + unprotected_slices); + impl->parsed_frame_size = 0; + grpc_slice_buffer_reset_and_unref(&impl->header_sb); + } + return TSI_OK; +} + +static void fake_zero_copy_grpc_protector_destroy( + tsi_zero_copy_grpc_protector *self) { + if (self == NULL) return; + tsi_fake_zero_copy_grpc_protector *impl = + (tsi_fake_zero_copy_grpc_protector *)self; + grpc_slice_buffer_destroy(&impl->header_sb); + grpc_slice_buffer_destroy(&impl->protected_sb); + gpr_free(impl); +} + +static const tsi_zero_copy_grpc_protector_vtable + zero_copy_grpc_protector_vtable = { + fake_zero_copy_grpc_protector_protect, + fake_zero_copy_grpc_protector_unprotect, + fake_zero_copy_grpc_protector_destroy, +}; + /* --- tsi_handshaker_result methods implementation. ---*/ typedef struct { @@ -383,6 +489,14 @@ static tsi_result fake_handshaker_result_extract_peer( return result; } +static tsi_result fake_handshaker_result_create_zero_copy_grpc_protector( + const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + tsi_zero_copy_grpc_protector **protector) { + *protector = + tsi_create_fake_zero_copy_grpc_protector(max_output_protected_frame_size); + return TSI_OK; +} + static tsi_result fake_handshaker_result_create_frame_protector( const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, tsi_frame_protector **protector) { @@ -407,7 +521,7 @@ static void fake_handshaker_result_destroy(tsi_handshaker_result *self) { static const tsi_handshaker_result_vtable handshaker_result_vtable = { fake_handshaker_result_extract_peer, - NULL, /* create_zero_copy_grpc_protector */ + fake_handshaker_result_create_zero_copy_grpc_protector, fake_handshaker_result_create_frame_protector, fake_handshaker_result_get_unused_bytes, fake_handshaker_result_destroy, @@ -631,3 +745,16 @@ tsi_frame_protector *tsi_create_fake_frame_protector( impl->base.vtable = &frame_protector_vtable; return &impl->base; } + +tsi_zero_copy_grpc_protector *tsi_create_fake_zero_copy_grpc_protector( + size_t *max_protected_frame_size) { + tsi_fake_zero_copy_grpc_protector *impl = gpr_zalloc(sizeof(*impl)); + grpc_slice_buffer_init(&impl->header_sb); + grpc_slice_buffer_init(&impl->protected_sb); + impl->max_frame_size = (max_protected_frame_size == NULL) + ? TSI_FAKE_DEFAULT_FRAME_SIZE + : *max_protected_frame_size; + impl->parsed_frame_size = 0; + impl->base.vtable = &zero_copy_grpc_protector_vtable; + return &impl->base; +} diff --git a/src/core/tsi/fake_transport_security.h b/src/core/tsi/fake_transport_security.h index 934b3cbeb20..6159708a849 100644 --- a/src/core/tsi/fake_transport_security.h +++ b/src/core/tsi/fake_transport_security.h @@ -39,6 +39,11 @@ tsi_handshaker *tsi_create_fake_handshaker(int is_client); tsi_frame_protector *tsi_create_fake_frame_protector( size_t *max_protected_frame_size); +/* Creates a zero-copy protector directly without going through the handshake + * phase. */ +tsi_zero_copy_grpc_protector *tsi_create_fake_zero_copy_grpc_protector( + size_t *max_protected_frame_size); + #ifdef __cplusplus } #endif diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c index 7ecd947e1fc..2145d55da75 100644 --- a/test/core/security/secure_endpoint_test.c +++ b/test/core/security/secure_endpoint_test.c @@ -36,12 +36,17 @@ static gpr_mu *g_mu; static grpc_pollset *g_pollset; static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( - size_t slice_size, grpc_slice *leftover_slices, size_t leftover_nslices) { + size_t slice_size, grpc_slice *leftover_slices, size_t leftover_nslices, + bool use_zero_copy_protector) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; tsi_frame_protector *fake_read_protector = tsi_create_fake_frame_protector(NULL); tsi_frame_protector *fake_write_protector = tsi_create_fake_frame_protector(NULL); + tsi_zero_copy_grpc_protector *fake_read_zero_copy_protector = + tsi_create_fake_zero_copy_grpc_protector(NULL); + tsi_zero_copy_grpc_protector *fake_write_zero_copy_protector = + tsi_create_fake_zero_copy_grpc_protector(NULL); grpc_endpoint_test_fixture f; grpc_endpoint_pair tcp; @@ -55,7 +60,11 @@ static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( if (leftover_nslices == 0) { f.client_ep = - grpc_secure_endpoint_create(fake_read_protector, tcp.client, NULL, 0); + use_zero_copy_protector + ? grpc_secure_endpoint_create(NULL, fake_read_zero_copy_protector, + tcp.client, NULL, 0) + : grpc_secure_endpoint_create(fake_read_protector, NULL, tcp.client, + NULL, 0); } else { unsigned i; tsi_result result; @@ -96,31 +105,53 @@ static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( } while (still_pending_size > 0); encrypted_leftover = grpc_slice_from_copied_buffer( (const char *)encrypted_buffer, total_buffer_size - buffer_size); - f.client_ep = grpc_secure_endpoint_create(fake_read_protector, tcp.client, - &encrypted_leftover, 1); + f.client_ep = + use_zero_copy_protector + ? grpc_secure_endpoint_create(NULL, fake_read_zero_copy_protector, + tcp.client, &encrypted_leftover, 1) + : grpc_secure_endpoint_create(fake_read_protector, NULL, tcp.client, + &encrypted_leftover, 1); grpc_slice_unref(encrypted_leftover); gpr_free(encrypted_buffer); } f.server_ep = - grpc_secure_endpoint_create(fake_write_protector, tcp.server, NULL, 0); + use_zero_copy_protector + ? grpc_secure_endpoint_create(NULL, fake_write_zero_copy_protector, + tcp.server, NULL, 0) + : grpc_secure_endpoint_create(fake_write_protector, NULL, tcp.server, + NULL, 0); grpc_exec_ctx_finish(&exec_ctx); return f; } static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair_noleftover(size_t slice_size) { - return secure_endpoint_create_fixture_tcp_socketpair(slice_size, NULL, 0); + return secure_endpoint_create_fixture_tcp_socketpair(slice_size, NULL, 0, + false); +} + +static grpc_endpoint_test_fixture +secure_endpoint_create_fixture_tcp_socketpair_noleftover_zero_copy( + size_t slice_size) { + return secure_endpoint_create_fixture_tcp_socketpair(slice_size, NULL, 0, + true); } static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair_leftover(size_t slice_size) { grpc_slice s = grpc_slice_from_copied_string("hello world 12345678900987654321"); - grpc_endpoint_test_fixture f; + return secure_endpoint_create_fixture_tcp_socketpair(slice_size, &s, 1, + false); +} - f = secure_endpoint_create_fixture_tcp_socketpair(slice_size, &s, 1); - return f; +static grpc_endpoint_test_fixture +secure_endpoint_create_fixture_tcp_socketpair_leftover_zero_copy( + size_t slice_size) { + grpc_slice s = + grpc_slice_from_copied_string("hello world 12345678900987654321"); + return secure_endpoint_create_fixture_tcp_socketpair(slice_size, &s, 1, true); } static void clean_up(void) {} @@ -128,8 +159,14 @@ static void clean_up(void) {} static grpc_endpoint_test_config configs[] = { {"secure_ep/tcp_socketpair", secure_endpoint_create_fixture_tcp_socketpair_noleftover, clean_up}, + {"secure_ep/tcp_socketpair_zero_copy", + secure_endpoint_create_fixture_tcp_socketpair_noleftover_zero_copy, + clean_up}, {"secure_ep/tcp_socketpair_leftover", secure_endpoint_create_fixture_tcp_socketpair_leftover, clean_up}, + {"secure_ep/tcp_socketpair_leftover_zero_copy", + secure_endpoint_create_fixture_tcp_socketpair_leftover_zero_copy, + clean_up}, }; static void inc_call_ctr(grpc_exec_ctx *exec_ctx, void *arg, @@ -184,7 +221,9 @@ int main(int argc, char **argv) { g_pollset = gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(g_pollset, &g_mu); grpc_endpoint_tests(configs[0], g_pollset, g_mu); - test_leftover(configs[1], 1); + grpc_endpoint_tests(configs[1], g_pollset, g_mu); + test_leftover(configs[2], 1); + test_leftover(configs[3], 1); GRPC_CLOSURE_INIT(&destroyed, destroy_pollset, g_pollset, grpc_schedule_on_exec_ctx); grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed);