From 464f558a456f8dd43609df066722cc5f0a0257bc Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 1 Jul 2019 10:12:23 -0700 Subject: [PATCH 1/7] Increase the control message size --- src/core/lib/iomgr/tcp_posix.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index 889272f6299..0a2ef767598 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -435,7 +435,9 @@ static void tcp_do_read(grpc_tcp* tcp) { GPR_TIMER_SCOPE("tcp_do_read", 0); struct msghdr msg; struct iovec iov[MAX_READ_IOVEC]; - char cmsgbuf[24 /*CMSG_SPACE(sizeof(int))*/]; + char cmsgbuf + [128 /*CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) + CMSG_SPACE(sizeof(int))*/ + ]; ssize_t read_bytes; size_t total_read_bytes = 0; From 420d5413c7d9816df16cf24018c1f8b8dc2e8d09 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 1 Jul 2019 10:50:47 -0700 Subject: [PATCH 2/7] Use the actual formula --- src/core/lib/iomgr/tcp_posix.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index 0a2ef767598..606e033a21e 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -435,9 +435,9 @@ static void tcp_do_read(grpc_tcp* tcp) { GPR_TIMER_SCOPE("tcp_do_read", 0); struct msghdr msg; struct iovec iov[MAX_READ_IOVEC]; - char cmsgbuf - [128 /*CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) + CMSG_SPACE(sizeof(int))*/ - ]; + constexpr size_t cmsg_alloc_space = + CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) + CMSG_SPACE(sizeof(int)); + char cmsgbuf[cmsg_alloc_space]; ssize_t read_bytes; size_t total_read_bytes = 0; From f44e0c07a709c81211fb32d5010906a8451f63ca Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 1 Jul 2019 10:55:11 -0700 Subject: [PATCH 3/7] Reviewer comments --- src/core/lib/iomgr/tcp_posix.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index 606e033a21e..031f681baa3 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -526,6 +526,7 @@ static void tcp_do_read(grpc_tcp* tcp) { if (cmsg->cmsg_level == SOL_TCP && cmsg->cmsg_type == TCP_CM_INQ && cmsg->cmsg_len == CMSG_LEN(sizeof(int))) { tcp->inq = *reinterpret_cast(CMSG_DATA(cmsg)); + break; } } } From 7cb861ce294edae5d40e8ea0c597687e989eec17 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 1 Jul 2019 11:00:55 -0700 Subject: [PATCH 4/7] Reviewer comments --- src/core/lib/iomgr/tcp_posix.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index 031f681baa3..79c56d5f3df 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -437,12 +437,12 @@ static void tcp_do_read(grpc_tcp* tcp) { struct iovec iov[MAX_READ_IOVEC]; constexpr size_t cmsg_alloc_space = CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) + CMSG_SPACE(sizeof(int)); - char cmsgbuf[cmsg_alloc_space]; ssize_t read_bytes; size_t total_read_bytes = 0; - size_t iov_len = std::min(MAX_READ_IOVEC, tcp->incoming_buffer->count); + char cmsgbuf[cmsg_alloc_space]; + for (size_t i = 0; i < iov_len; i++) { iov[i].iov_base = GRPC_SLICE_START_PTR(tcp->incoming_buffer->slices[i]); iov[i].iov_len = GRPC_SLICE_LENGTH(tcp->incoming_buffer->slices[i]); From a94e00dccf732e19ec33e22ca184a99d44ace042 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 1 Jul 2019 11:15:40 -0700 Subject: [PATCH 5/7] Reviewer comments --- src/core/lib/iomgr/tcp_posix.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index 79c56d5f3df..94a9da2ee49 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -435,14 +435,13 @@ static void tcp_do_read(grpc_tcp* tcp) { GPR_TIMER_SCOPE("tcp_do_read", 0); struct msghdr msg; struct iovec iov[MAX_READ_IOVEC]; - constexpr size_t cmsg_alloc_space = - CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) + CMSG_SPACE(sizeof(int)); ssize_t read_bytes; size_t total_read_bytes = 0; size_t iov_len = std::min(MAX_READ_IOVEC, tcp->incoming_buffer->count); + constexpr size_t cmsg_alloc_space = + CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) + CMSG_SPACE(sizeof(int)); char cmsgbuf[cmsg_alloc_space]; - for (size_t i = 0; i < iov_len; i++) { iov[i].iov_base = GRPC_SLICE_START_PTR(tcp->incoming_buffer->slices[i]); iov[i].iov_len = GRPC_SLICE_LENGTH(tcp->incoming_buffer->slices[i]); From 02ff96bd31f33a0c193f0766d87bb57ba438b2ef Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 1 Jul 2019 11:46:52 -0700 Subject: [PATCH 6/7] No need to allocate space for receive timestamp if errqueue is not present --- src/core/lib/iomgr/tcp_posix.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index 94a9da2ee49..0e153679ad7 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -439,8 +439,12 @@ static void tcp_do_read(grpc_tcp* tcp) { size_t total_read_bytes = 0; size_t iov_len = std::min(MAX_READ_IOVEC, tcp->incoming_buffer->count); +#ifdef GRPC_LINUX_ERRQUEUE constexpr size_t cmsg_alloc_space = CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) + CMSG_SPACE(sizeof(int)); +#else + constexpr size_t cmsg_alloc_space = CMSG_SPACE(sizeof(int)); +#endif /* GRPC_LINUX_ERRQUEUE */ char cmsgbuf[cmsg_alloc_space]; for (size_t i = 0; i < iov_len; i++) { iov[i].iov_base = GRPC_SLICE_START_PTR(tcp->incoming_buffer->slices[i]); From 1487ac42cc992e5e9a9e8dfa0742a74c14593db6 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 1 Jul 2019 14:11:44 -0700 Subject: [PATCH 7/7] Remove CMSG_SPACE for macos --- src/core/lib/iomgr/tcp_posix.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index 0e153679ad7..819c5284256 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -443,7 +443,7 @@ static void tcp_do_read(grpc_tcp* tcp) { constexpr size_t cmsg_alloc_space = CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) + CMSG_SPACE(sizeof(int)); #else - constexpr size_t cmsg_alloc_space = CMSG_SPACE(sizeof(int)); + constexpr size_t cmsg_alloc_space = 24 /* CMSG_SPACE(sizeof(int)) */; #endif /* GRPC_LINUX_ERRQUEUE */ char cmsgbuf[cmsg_alloc_space]; for (size_t i = 0; i < iov_len; i++) {