From 929f4c652f5b57de016cb2cafbf81139cae30e3f Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 10 Oct 2016 16:02:43 -0700 Subject: [PATCH] Add tos configuration in channel args --- BUILD | 4 ++ CMakeLists.txt | 3 + Makefile | 40 +++++++++++ binding.gyp | 1 + build.yaml | 15 ++++ config.m4 | 1 + gRPC-Core.podspec | 1 + grpc.gemspec | 1 + include/grpc/impl/codegen/grpc_types.h | 2 + package.xml | 1 + .../chttp2/client/insecure/channel_create.c | 9 ++- .../client/secure/secure_channel_create.c | 12 +++- src/core/lib/http/httpcli.c | 11 ++- src/core/lib/iomgr/socket_utils_posix.h | 4 ++ src/core/lib/iomgr/socket_utils_tos_posix.c | 68 +++++++++++++++++++ src/core/lib/iomgr/tcp_client.h | 18 ++++- src/core/lib/iomgr/tcp_client_posix.c | 39 ++++++----- src/core/lib/iomgr/tcp_client_windows.c | 8 +-- src/python/grpcio/grpc_core_dependencies.py | 1 + test/core/end2end/fixtures/http_proxy.c | 11 ++- test/core/end2end/fuzzers/api_fuzzer.c | 14 ++-- test/core/iomgr/socket_utils_tos_test.c | 67 ++++++++++++++++++ test/core/iomgr/tcp_client_posix_test.c | 22 ++++-- tools/doxygen/Doxyfile.core.internal | 1 + tools/run_tests/sources_and_headers.json | 18 +++++ tools/run_tests/tests.json | 19 ++++++ vsprojects/vcxproj/grpc/grpc.vcxproj | 2 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 3 + .../grpc_test_util/grpc_test_util.vcxproj | 2 + .../grpc_test_util.vcxproj.filters | 3 + .../grpc_unsecure/grpc_unsecure.vcxproj | 2 + .../grpc_unsecure.vcxproj.filters | 3 + 32 files changed, 358 insertions(+), 48 deletions(-) create mode 100644 src/core/lib/iomgr/socket_utils_tos_posix.c create mode 100644 test/core/iomgr/socket_utils_tos_test.c diff --git a/BUILD b/BUILD index bad6f3f0755..d2a4e70f614 100644 --- a/BUILD +++ b/BUILD @@ -366,6 +366,7 @@ cc_library( "src/core/lib/iomgr/socket_utils_common_posix.c", "src/core/lib/iomgr/socket_utils_linux.c", "src/core/lib/iomgr/socket_utils_posix.c", + "src/core/lib/iomgr/socket_utils_tos_posix.c", "src/core/lib/iomgr/socket_windows.c", "src/core/lib/iomgr/tcp_client_posix.c", "src/core/lib/iomgr/tcp_client_windows.c", @@ -752,6 +753,7 @@ cc_library( "src/core/lib/iomgr/socket_utils_common_posix.c", "src/core/lib/iomgr/socket_utils_linux.c", "src/core/lib/iomgr/socket_utils_posix.c", + "src/core/lib/iomgr/socket_utils_tos_posix.c", "src/core/lib/iomgr/socket_windows.c", "src/core/lib/iomgr/tcp_client_posix.c", "src/core/lib/iomgr/tcp_client_windows.c", @@ -1100,6 +1102,7 @@ cc_library( "src/core/lib/iomgr/socket_utils_common_posix.c", "src/core/lib/iomgr/socket_utils_linux.c", "src/core/lib/iomgr/socket_utils_posix.c", + "src/core/lib/iomgr/socket_utils_tos_posix.c", "src/core/lib/iomgr/socket_windows.c", "src/core/lib/iomgr/tcp_client_posix.c", "src/core/lib/iomgr/tcp_client_windows.c", @@ -1863,6 +1866,7 @@ objc_library( "src/core/lib/iomgr/socket_utils_common_posix.c", "src/core/lib/iomgr/socket_utils_linux.c", "src/core/lib/iomgr/socket_utils_posix.c", + "src/core/lib/iomgr/socket_utils_tos_posix.c", "src/core/lib/iomgr/socket_windows.c", "src/core/lib/iomgr/tcp_client_posix.c", "src/core/lib/iomgr/tcp_client_windows.c", diff --git a/CMakeLists.txt b/CMakeLists.txt index c4191521bd1..d88d9177524 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -333,6 +333,7 @@ add_library(grpc src/core/lib/iomgr/socket_utils_common_posix.c src/core/lib/iomgr/socket_utils_linux.c src/core/lib/iomgr/socket_utils_posix.c + src/core/lib/iomgr/socket_utils_tos_posix.c src/core/lib/iomgr/socket_windows.c src/core/lib/iomgr/tcp_client_posix.c src/core/lib/iomgr/tcp_client_windows.c @@ -592,6 +593,7 @@ add_library(grpc_cronet src/core/lib/iomgr/socket_utils_common_posix.c src/core/lib/iomgr/socket_utils_linux.c src/core/lib/iomgr/socket_utils_posix.c + src/core/lib/iomgr/socket_utils_tos_posix.c src/core/lib/iomgr/socket_windows.c src/core/lib/iomgr/tcp_client_posix.c src/core/lib/iomgr/tcp_client_windows.c @@ -823,6 +825,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/socket_utils_common_posix.c src/core/lib/iomgr/socket_utils_linux.c src/core/lib/iomgr/socket_utils_posix.c + src/core/lib/iomgr/socket_utils_tos_posix.c src/core/lib/iomgr/socket_windows.c src/core/lib/iomgr/tcp_client_posix.c src/core/lib/iomgr/tcp_client_windows.c diff --git a/Makefile b/Makefile index 62c65822b0f..cd988b85557 100644 --- a/Makefile +++ b/Makefile @@ -1012,6 +1012,7 @@ set_initial_connect_string_test: $(BINDIR)/$(CONFIG)/set_initial_connect_string_ sockaddr_resolver_test: $(BINDIR)/$(CONFIG)/sockaddr_resolver_test sockaddr_utils_test: $(BINDIR)/$(CONFIG)/sockaddr_utils_test socket_utils_test: $(BINDIR)/$(CONFIG)/socket_utils_test +socket_utils_tos_test: $(BINDIR)/$(CONFIG)/socket_utils_tos_test tcp_client_posix_test: $(BINDIR)/$(CONFIG)/tcp_client_posix_test tcp_posix_test: $(BINDIR)/$(CONFIG)/tcp_posix_test tcp_server_posix_test: $(BINDIR)/$(CONFIG)/tcp_server_posix_test @@ -1328,6 +1329,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/sockaddr_resolver_test \ $(BINDIR)/$(CONFIG)/sockaddr_utils_test \ $(BINDIR)/$(CONFIG)/socket_utils_test \ + $(BINDIR)/$(CONFIG)/socket_utils_tos_test \ $(BINDIR)/$(CONFIG)/tcp_client_posix_test \ $(BINDIR)/$(CONFIG)/tcp_posix_test \ $(BINDIR)/$(CONFIG)/tcp_server_posix_test \ @@ -1714,6 +1716,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/sockaddr_utils_test || ( echo test sockaddr_utils_test failed ; exit 1 ) $(E) "[RUN] Testing socket_utils_test" $(Q) $(BINDIR)/$(CONFIG)/socket_utils_test || ( echo test socket_utils_test failed ; exit 1 ) + $(E) "[RUN] Testing socket_utils_tos_test" + $(Q) $(BINDIR)/$(CONFIG)/socket_utils_tos_test || ( echo test socket_utils_tos_test failed ; exit 1 ) $(E) "[RUN] Testing tcp_client_posix_test" $(Q) $(BINDIR)/$(CONFIG)/tcp_client_posix_test || ( echo test tcp_client_posix_test failed ; exit 1 ) $(E) "[RUN] Testing tcp_posix_test" @@ -2577,6 +2581,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ src/core/lib/iomgr/socket_utils_posix.c \ + src/core/lib/iomgr/socket_utils_tos_posix.c \ src/core/lib/iomgr/socket_windows.c \ src/core/lib/iomgr/tcp_client_posix.c \ src/core/lib/iomgr/tcp_client_windows.c \ @@ -2854,6 +2859,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ src/core/lib/iomgr/socket_utils_posix.c \ + src/core/lib/iomgr/socket_utils_tos_posix.c \ src/core/lib/iomgr/socket_windows.c \ src/core/lib/iomgr/tcp_client_posix.c \ src/core/lib/iomgr/tcp_client_windows.c \ @@ -3121,6 +3127,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ src/core/lib/iomgr/socket_utils_posix.c \ + src/core/lib/iomgr/socket_utils_tos_posix.c \ src/core/lib/iomgr/socket_windows.c \ src/core/lib/iomgr/tcp_client_posix.c \ src/core/lib/iomgr/tcp_client_windows.c \ @@ -3315,6 +3322,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ src/core/lib/iomgr/socket_utils_posix.c \ + src/core/lib/iomgr/socket_utils_tos_posix.c \ src/core/lib/iomgr/socket_windows.c \ src/core/lib/iomgr/tcp_client_posix.c \ src/core/lib/iomgr/tcp_client_windows.c \ @@ -10341,6 +10349,38 @@ endif endif +SOCKET_UTILS_TOS_TEST_SRC = \ + test/core/iomgr/socket_utils_tos_test.c \ + +SOCKET_UTILS_TOS_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(SOCKET_UTILS_TOS_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/socket_utils_tos_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/socket_utils_tos_test: $(SOCKET_UTILS_TOS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(SOCKET_UTILS_TOS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/socket_utils_tos_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/iomgr/socket_utils_tos_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_socket_utils_tos_test: $(SOCKET_UTILS_TOS_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(SOCKET_UTILS_TOS_TEST_OBJS:.o=.dep) +endif +endif + + TCP_CLIENT_POSIX_TEST_SRC = \ test/core/iomgr/tcp_client_posix_test.c \ diff --git a/binding.gyp b/binding.gyp index 58edda2e63e..fd2bed86b04 100644 --- a/binding.gyp +++ b/binding.gyp @@ -608,6 +608,7 @@ 'src/core/lib/iomgr/socket_utils_common_posix.c', 'src/core/lib/iomgr/socket_utils_linux.c', 'src/core/lib/iomgr/socket_utils_posix.c', + 'src/core/lib/iomgr/socket_utils_tos_posix.c', 'src/core/lib/iomgr/socket_windows.c', 'src/core/lib/iomgr/tcp_client_posix.c', 'src/core/lib/iomgr/tcp_client_windows.c', diff --git a/build.yaml b/build.yaml index 584084ff865..857ef6c8ee6 100644 --- a/build.yaml +++ b/build.yaml @@ -293,6 +293,7 @@ filegroups: - src/core/lib/iomgr/socket_utils_common_posix.c - src/core/lib/iomgr/socket_utils_linux.c - src/core/lib/iomgr/socket_utils_posix.c + - src/core/lib/iomgr/socket_utils_tos_posix.c - src/core/lib/iomgr/socket_windows.c - src/core/lib/iomgr/tcp_client_posix.c - src/core/lib/iomgr/tcp_client_windows.c @@ -2437,6 +2438,20 @@ targets: - mac - linux - posix +- name: socket_utils_tos_test + build: test + language: c + src: + - test/core/iomgr/socket_utils_tos_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: tcp_client_posix_test cpu_cost: 0.5 build: test diff --git a/config.m4 b/config.m4 index 0f103655a84..7012fb0c54d 100644 --- a/config.m4 +++ b/config.m4 @@ -127,6 +127,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ src/core/lib/iomgr/socket_utils_posix.c \ + src/core/lib/iomgr/socket_utils_tos_posix.c \ src/core/lib/iomgr/socket_windows.c \ src/core/lib/iomgr/tcp_client_posix.c \ src/core/lib/iomgr/tcp_client_windows.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 6886bdfb5aa..7b74143f415 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -457,6 +457,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/socket_utils_common_posix.c', 'src/core/lib/iomgr/socket_utils_linux.c', 'src/core/lib/iomgr/socket_utils_posix.c', + 'src/core/lib/iomgr/socket_utils_tos_posix.c', 'src/core/lib/iomgr/socket_windows.c', 'src/core/lib/iomgr/tcp_client_posix.c', 'src/core/lib/iomgr/tcp_client_windows.c', diff --git a/grpc.gemspec b/grpc.gemspec index 6079ea2aef5..23537983552 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -377,6 +377,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/socket_utils_common_posix.c ) s.files += %w( src/core/lib/iomgr/socket_utils_linux.c ) s.files += %w( src/core/lib/iomgr/socket_utils_posix.c ) + s.files += %w( src/core/lib/iomgr/socket_utils_tos_posix.c ) s.files += %w( src/core/lib/iomgr/socket_windows.c ) s.files += %w( src/core/lib/iomgr/tcp_client_posix.c ) s.files += %w( src/core/lib/iomgr/tcp_client_windows.c ) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 07e7cff2016..1b93729b721 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -201,6 +201,8 @@ typedef struct { #define GRPC_ARG_MAX_METADATA_SIZE "grpc.max_metadata_size" /** If non-zero, allow the use of SO_REUSEPORT if it's available (default 1) */ #define GRPC_ARG_ALLOW_REUSEPORT "grpc.so_reuseport" +/** The Type-Of-Service (TOS), in bytes. */ +#define GRPC_ARG_TOS "grpc.tos" /** \} */ /** Result of a grpc call. If the caller satisfies the prerequisites of a diff --git a/package.xml b/package.xml index 7bac6957871..489504c43f6 100644 --- a/package.xml +++ b/package.xml @@ -384,6 +384,7 @@ + diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c index c2b59569fd3..858b1dbee00 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c @@ -142,6 +142,7 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con, grpc_connect_out_args *result, grpc_closure *notify) { connector *c = (connector *)con; + grpc_tcp_client_connect_args tcp_client_connect_args; GPR_ASSERT(c->notify == NULL); GPR_ASSERT(notify->cb); c->notify = notify; @@ -149,9 +150,13 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con, c->result = result; c->tcp = NULL; grpc_closure_init(&c->connected, connected, c); + tcp_client_connect_args.interested_parties = args->interested_parties; + tcp_client_connect_args.addr = args->addr; + tcp_client_connect_args.addr_len = args->addr_len; + tcp_client_connect_args.deadline = args->deadline; + tcp_client_connect_args.channel_args = args->channel_args; grpc_tcp_client_connect(exec_ctx, &c->connected, &c->tcp, - args->interested_parties, args->addr, args->addr_len, - args->deadline); + &tcp_client_connect_args); } static const grpc_connector_vtable connector_vtable = { diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c index 31c54ff74c6..d71d9f2d52b 100644 --- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c +++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c @@ -200,6 +200,7 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con, grpc_connect_out_args *result, grpc_closure *notify) { connector *c = (connector *)con; + grpc_tcp_client_connect_args tcp_client_connect_args; GPR_ASSERT(c->notify == NULL); c->notify = notify; c->args = *args; @@ -208,9 +209,14 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con, GPR_ASSERT(c->connecting_endpoint == NULL); gpr_mu_unlock(&c->mu); grpc_closure_init(&c->connected_closure, connected, c); - grpc_tcp_client_connect( - exec_ctx, &c->connected_closure, &c->newly_connecting_endpoint, - args->interested_parties, args->addr, args->addr_len, args->deadline); + tcp_client_connect_args.interested_parties = args->interested_parties; + tcp_client_connect_args.addr = args->addr; + tcp_client_connect_args.addr_len = args->addr_len; + tcp_client_connect_args.deadline = args->deadline; + tcp_client_connect_args.channel_args = args->channel_args; + grpc_tcp_client_connect(exec_ctx, &c->connected_closure, + &c->newly_connecting_endpoint, + &tcp_client_connect_args); } static const grpc_connector_vtable connector_vtable = { diff --git a/src/core/lib/http/httpcli.c b/src/core/lib/http/httpcli.c index 7f3c2d120d1..c56fae6d0c1 100644 --- a/src/core/lib/http/httpcli.c +++ b/src/core/lib/http/httpcli.c @@ -213,6 +213,7 @@ static void on_connected(grpc_exec_ctx *exec_ctx, void *arg, static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req, grpc_error *error) { grpc_resolved_address *addr; + grpc_tcp_client_connect_args tcp_client_connect_args; if (error != GRPC_ERROR_NONE) { append_error(req, error); } @@ -224,9 +225,13 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req, } addr = &req->addresses->addrs[req->next_address++]; grpc_closure_init(&req->connected, on_connected, req); - grpc_tcp_client_connect( - exec_ctx, &req->connected, &req->ep, req->context->pollset_set, - (struct sockaddr *)&addr->addr, addr->len, req->deadline); + tcp_client_connect_args.interested_parties = req->context->pollset_set; + tcp_client_connect_args.addr = (struct sockaddr *)&addr->addr; + tcp_client_connect_args.addr_len = addr->len; + tcp_client_connect_args.deadline = req->deadline; + tcp_client_connect_args.channel_args = NULL; + grpc_tcp_client_connect(exec_ctx, &req->connected, &req->ep, + &tcp_client_connect_args); } static void on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { diff --git a/src/core/lib/iomgr/socket_utils_posix.h b/src/core/lib/iomgr/socket_utils_posix.h index 7bcc2219aea..bf4cc96af2b 100644 --- a/src/core/lib/iomgr/socket_utils_posix.h +++ b/src/core/lib/iomgr/socket_utils_posix.h @@ -37,6 +37,7 @@ #include #include +#include #include "src/core/lib/iomgr/error.h" /* a wrapper for accept or accept4 */ @@ -86,6 +87,9 @@ grpc_error *grpc_set_socket_sndbuf(int fd, int buffer_size_bytes); /* Tries to set the socket's receive buffer to given size. */ grpc_error *grpc_set_socket_rcvbuf(int fd, int buffer_size_bytes); +/* Tries to set the socket's type to service */ +grpc_error *grpc_set_socket_tos(int fd, grpc_arg *arg); + /* An enum to keep track of IPv4/IPv6 socket modes. Currently, this information is only used when a socket is first created, but diff --git a/src/core/lib/iomgr/socket_utils_tos_posix.c b/src/core/lib/iomgr/socket_utils_tos_posix.c new file mode 100644 index 00000000000..603ad3c421d --- /dev/null +++ b/src/core/lib/iomgr/socket_utils_tos_posix.c @@ -0,0 +1,68 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include + +#ifdef GPR_POSIX_SOCKET + +#include "src/core/lib/iomgr/socket_utils_posix.h" + +#include +#include +#include + +#include +#include + +grpc_error* grpc_set_socket_tos(int fd, grpc_arg* arg) { + int newval; + socklen_t intlen = sizeof(newval); + + GPR_ASSERT(0 == strcmp(arg->key, GRPC_ARG_TOS)); + GPR_ASSERT(arg->type == GRPC_ARG_INTEGER); + + if (0 != setsockopt(fd, IPPROTO_IP, IP_TOS, &arg->value.integer, + sizeof(arg->value.integer))) { + return GRPC_OS_ERROR(errno, "setsockopt(IP_TOS)"); + } + if (0 != getsockopt(fd, IPPROTO_IP, IP_TOS, &newval, &intlen)) { + return GRPC_OS_ERROR(errno, "getsockopt(IP_TOS)"); + } + if (newval != arg->value.integer) { + return GRPC_ERROR_CREATE("Failed to set IP_TOS"); + } + + return GRPC_ERROR_NONE; +} + +#endif diff --git a/src/core/lib/iomgr/tcp_client.h b/src/core/lib/iomgr/tcp_client.h index a07e0b9f0cb..215c2fbe5b5 100644 --- a/src/core/lib/iomgr/tcp_client.h +++ b/src/core/lib/iomgr/tcp_client.h @@ -34,11 +34,25 @@ #ifndef GRPC_CORE_LIB_IOMGR_TCP_CLIENT_H #define GRPC_CORE_LIB_IOMGR_TCP_CLIENT_H +#include #include #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/iomgr/sockaddr.h" +/** arguments for a tcp client connection */ +typedef struct { + /** set of pollsets interested in this connection */ + grpc_pollset_set *interested_parties; + /** address to connect to */ + const struct sockaddr *addr; + size_t addr_len; + /** deadline for connection */ + gpr_timespec deadline; + /** channel arguments */ + const grpc_channel_args *channel_args; +} grpc_tcp_client_connect_args; + /* Asynchronously connect to an address (specified as (addr, len)), and call cb with arg and the completed connection when done (or call cb with arg and NULL on failure). @@ -46,8 +60,6 @@ in this connection being established (in order to continue their work) */ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_connect, grpc_endpoint **endpoint, - grpc_pollset_set *interested_parties, - const struct sockaddr *addr, size_t addr_len, - gpr_timespec deadline); + const grpc_tcp_client_connect_args *args); #endif /* GRPC_CORE_LIB_IOMGR_TCP_CLIENT_H */ diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c index 3496b6094f2..1439efe706f 100644 --- a/src/core/lib/iomgr/tcp_client_posix.c +++ b/src/core/lib/iomgr/tcp_client_posix.c @@ -71,7 +71,8 @@ typedef struct { grpc_closure *closure; } async_connect; -static grpc_error *prepare_socket(const struct sockaddr *addr, int fd) { +static grpc_error *prepare_socket(const struct sockaddr *addr, int fd, + const grpc_channel_args *channel_args) { grpc_error *err = GRPC_ERROR_NONE; GPR_ASSERT(fd >= 0); @@ -86,6 +87,15 @@ static grpc_error *prepare_socket(const struct sockaddr *addr, int fd) { } err = grpc_set_socket_no_sigpipe_if_possible(fd); if (err != GRPC_ERROR_NONE) goto error; + if (channel_args) { + for (size_t i = 0; i < channel_args->num_args; i++) { + if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_TOS)) { + err = grpc_set_socket_tos(fd, &channel_args->args[i]); + if (err != GRPC_ERROR_NONE) goto error; + break; + } + } + } goto done; error: @@ -222,9 +232,7 @@ finish: static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, - grpc_pollset_set *interested_parties, - const struct sockaddr *addr, - size_t addr_len, gpr_timespec deadline) { + const grpc_tcp_client_connect_args *args) { int fd; grpc_dualstack_mode dsmode; int err; @@ -235,6 +243,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, char *name; char *addr_str; grpc_error *error; + const struct sockaddr *addr = args->addr; + size_t addr_len = args->addr_len; *ep = NULL; @@ -255,7 +265,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, addr = (struct sockaddr *)&addr4_copy; addr_len = sizeof(addr4_copy); } - if ((error = prepare_socket(addr, fd)) != GRPC_ERROR_NONE) { + if ((error = prepare_socket(addr, fd, args->channel_args)) != + GRPC_ERROR_NONE) { grpc_exec_ctx_sched(exec_ctx, closure, error, NULL); return; } @@ -283,13 +294,13 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, goto done; } - grpc_pollset_set_add_fd(exec_ctx, interested_parties, fdobj); + grpc_pollset_set_add_fd(exec_ctx, args->interested_parties, fdobj); ac = gpr_malloc(sizeof(async_connect)); ac->closure = closure; ac->ep = ep; ac->fd = fdobj; - ac->interested_parties = interested_parties; + ac->interested_parties = args->interested_parties; ac->addr_str = addr_str; addr_str = NULL; gpr_mu_init(&ac->mu); @@ -304,7 +315,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, gpr_mu_lock(&ac->mu); grpc_timer_init(exec_ctx, &ac->alarm, - gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC), + gpr_convert_clock_type(args->deadline, GPR_CLOCK_MONOTONIC), tc_on_alarm, ac, gpr_now(GPR_CLOCK_MONOTONIC)); grpc_fd_notify_on_write(exec_ctx, ac->fd, &ac->write_closure); gpr_mu_unlock(&ac->mu); @@ -316,17 +327,13 @@ done: // overridden by api_fuzzer.c void (*grpc_tcp_client_connect_impl)( - grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, - grpc_pollset_set *interested_parties, const struct sockaddr *addr, - size_t addr_len, gpr_timespec deadline) = tcp_client_connect_impl; + grpc_exec_ctx *exec_ctx, grpc_closure *on_connect, grpc_endpoint **endpoint, + const grpc_tcp_client_connect_args *args) = tcp_client_connect_impl; void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, - grpc_pollset_set *interested_parties, - const struct sockaddr *addr, size_t addr_len, - gpr_timespec deadline) { - grpc_tcp_client_connect_impl(exec_ctx, closure, ep, interested_parties, addr, - addr_len, deadline); + const grpc_tcp_client_connect_args *args) { + grpc_tcp_client_connect_impl(exec_ctx, closure, ep, args); } #endif diff --git a/src/core/lib/iomgr/tcp_client_windows.c b/src/core/lib/iomgr/tcp_client_windows.c index 562cb9c6bff..b4517b3bdb0 100644 --- a/src/core/lib/iomgr/tcp_client_windows.c +++ b/src/core/lib/iomgr/tcp_client_windows.c @@ -128,9 +128,7 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { notification request for the connection, and one timeout alert. */ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, grpc_endpoint **endpoint, - grpc_pollset_set *interested_parties, - const struct sockaddr *addr, size_t addr_len, - gpr_timespec deadline) { + const grpc_tcp_client_connect_args *args) { SOCKET sock = INVALID_SOCKET; BOOL success; int status; @@ -143,6 +141,8 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, DWORD ioctl_num_bytes; grpc_winsocket_callback_info *info; grpc_error *error = GRPC_ERROR_NONE; + const struct sockaddr *addr = args->addr; + size_t addr_len = args->addr_len; *endpoint = NULL; @@ -208,7 +208,7 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, ac->endpoint = endpoint; grpc_closure_init(&ac->on_connect, on_connect, ac); - grpc_timer_init(exec_ctx, &ac->alarm, deadline, on_alarm, ac, + grpc_timer_init(exec_ctx, &ac->alarm, args->deadline, on_alarm, ac, gpr_now(GPR_CLOCK_MONOTONIC)); grpc_socket_notify_on_write(exec_ctx, socket, &ac->on_connect); return; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 2bebf48c70a..dec578edb5c 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -121,6 +121,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/socket_utils_common_posix.c', 'src/core/lib/iomgr/socket_utils_linux.c', 'src/core/lib/iomgr/socket_utils_posix.c', + 'src/core/lib/iomgr/socket_utils_tos_posix.c', 'src/core/lib/iomgr/socket_windows.c', 'src/core/lib/iomgr/tcp_client_posix.c', 'src/core/lib/iomgr/tcp_client_windows.c', diff --git a/test/core/end2end/fixtures/http_proxy.c b/test/core/end2end/fixtures/http_proxy.c index 22533b9694c..d0bd452a04b 100644 --- a/test/core/end2end/fixtures/http_proxy.c +++ b/test/core/end2end/fixtures/http_proxy.c @@ -356,10 +356,15 @@ static void on_read_request_done(grpc_exec_ctx* exec_ctx, void* arg, // The connection callback inherits our reference to conn. const gpr_timespec deadline = gpr_time_add( gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(10, GPR_TIMESPAN)); + grpc_tcp_client_connect_args tcp_client_connect_args; + tcp_client_connect_args.interested_parties = conn->pollset_set; + tcp_client_connect_args.addr = + (struct sockaddr*)&resolved_addresses->addrs[0].addr; + tcp_client_connect_args.addr_len = resolved_addresses->addrs[0].len; + tcp_client_connect_args.deadline = deadline; + tcp_client_connect_args.channel_args = NULL; grpc_tcp_client_connect(exec_ctx, &conn->on_server_connect_done, - &conn->server_endpoint, conn->pollset_set, - (struct sockaddr*)&resolved_addresses->addrs[0].addr, - resolved_addresses->addrs[0].len, deadline); + &conn->server_endpoint, &tcp_client_connect_args); grpc_resolved_addresses_destroy(resolved_addresses); } diff --git a/test/core/end2end/fuzzers/api_fuzzer.c b/test/core/end2end/fuzzers/api_fuzzer.c index 96ea82d95ea..5c0fb0f9385 100644 --- a/test/core/end2end/fuzzers/api_fuzzer.c +++ b/test/core/end2end/fuzzers/api_fuzzer.c @@ -229,10 +229,10 @@ void my_resolve_address(grpc_exec_ctx *exec_ctx, const char *addr, // client connection // defined in tcp_client_posix.c -extern void (*grpc_tcp_client_connect_impl)( - grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, - grpc_pollset_set *interested_parties, const struct sockaddr *addr, - size_t addr_len, gpr_timespec deadline); +extern void (*grpc_tcp_client_connect_impl)(grpc_exec_ctx *exec_ctx, + grpc_closure *closure, + grpc_endpoint **ep, + grpc_tcp_client_connect_args *args); static void sched_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, gpr_timespec deadline); @@ -288,10 +288,8 @@ static void sched_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure, static void my_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, - grpc_pollset_set *interested_parties, - const struct sockaddr *addr, size_t addr_len, - gpr_timespec deadline) { - sched_connect(exec_ctx, closure, ep, deadline); + grpc_tcp_client_connect_args *args) { + sched_connect(exec_ctx, closure, ep, args->deadline); } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/core/iomgr/socket_utils_tos_test.c b/test/core/iomgr/socket_utils_tos_test.c new file mode 100644 index 00000000000..ee3a43367b8 --- /dev/null +++ b/test/core/iomgr/socket_utils_tos_test.c @@ -0,0 +1,67 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include "src/core/lib/iomgr/socket_utils_posix.h" + +#include +#include +#include + +#include +#include "test/core/util/test_config.h" + +int main(int argc, char **argv) { + int sock; + grpc_arg tos_arg; + grpc_test_init(argc, argv); + + sock = socket(PF_INET, SOCK_STREAM, 0); + GPR_ASSERT(sock > 0); + + tos_arg.key = GRPC_ARG_TOS; + tos_arg.type = GRPC_ARG_INTEGER; + tos_arg.value.integer = IPTOS_LOWDELAY; + GPR_ASSERT( + GRPC_LOG_IF_ERROR("set_socket_tos", grpc_set_socket_tos(sock, &tos_arg))); + tos_arg.value.integer = IPTOS_THROUGHPUT; + GPR_ASSERT( + GRPC_LOG_IF_ERROR("set_socket_tos", grpc_set_socket_tos(sock, &tos_arg))); + tos_arg.value.integer = IPTOS_RELIABILITY; + GPR_ASSERT( + GRPC_LOG_IF_ERROR("set_socket_tos", grpc_set_socket_tos(sock, &tos_arg))); + + close(sock); + + return 0; +} diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index d0c10474239..aec4192ce50 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -92,6 +92,7 @@ void test_succeeds(void) { int connections_complete_before; grpc_closure done; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_tcp_client_connect_args tcp_client_connect_args; gpr_log(GPR_DEBUG, "test_succeeds"); @@ -111,9 +112,13 @@ void test_succeeds(void) { /* connect to it */ GPR_ASSERT(getsockname(svr_fd, (struct sockaddr *)&addr, &addr_len) == 0); grpc_closure_init(&done, must_succeed, NULL); - grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, - (struct sockaddr *)&addr, addr_len, - gpr_inf_future(GPR_CLOCK_REALTIME)); + tcp_client_connect_args.interested_parties = g_pollset_set; + tcp_client_connect_args.addr = (struct sockaddr *)&addr; + tcp_client_connect_args.addr_len = addr_len; + tcp_client_connect_args.deadline = gpr_inf_future(GPR_CLOCK_REALTIME); + tcp_client_connect_args.channel_args = NULL; + grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, + &tcp_client_connect_args); /* await the connection */ do { @@ -148,6 +153,7 @@ void test_fails(void) { int connections_complete_before; grpc_closure done; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_tcp_client_connect_args tcp_client_connect_args; gpr_log(GPR_DEBUG, "test_fails"); @@ -160,9 +166,13 @@ void test_fails(void) { /* connect to a broken address */ grpc_closure_init(&done, must_fail, NULL); - grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, - (struct sockaddr *)&addr, addr_len, - gpr_inf_future(GPR_CLOCK_REALTIME)); + tcp_client_connect_args.interested_parties = g_pollset_set; + tcp_client_connect_args.addr = (struct sockaddr *)&addr; + tcp_client_connect_args.addr_len = addr_len; + tcp_client_connect_args.deadline = gpr_inf_future(GPR_CLOCK_REALTIME); + tcp_client_connect_args.channel_args = NULL; + grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, + &tcp_client_connect_args); gpr_mu_lock(g_mu); diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 44f147aeb18..59998f03fba 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -994,6 +994,7 @@ src/core/lib/iomgr/sockaddr_utils.c \ src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ src/core/lib/iomgr/socket_utils_posix.c \ +src/core/lib/iomgr/socket_utils_tos_posix.c \ src/core/lib/iomgr/socket_windows.c \ src/core/lib/iomgr/tcp_client_posix.c \ src/core/lib/iomgr/tcp_client_windows.c \ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index c05d194e19e..5ecb3ee0f87 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -1858,6 +1858,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "socket_utils_tos_test", + "src": [ + "test/core/iomgr/socket_utils_tos_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", @@ -6533,6 +6550,7 @@ "src/core/lib/iomgr/socket_utils_linux.c", "src/core/lib/iomgr/socket_utils_posix.c", "src/core/lib/iomgr/socket_utils_posix.h", + "src/core/lib/iomgr/socket_utils_tos_posix.c", "src/core/lib/iomgr/socket_windows.c", "src/core/lib/iomgr/socket_windows.h", "src/core/lib/iomgr/tcp_client.h", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index c3395067c94..07b51d3c2f3 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -1819,6 +1819,25 @@ "posix" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "socket_utils_tos_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 965d06d1d5c..f115001507f 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -549,6 +549,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index dddd4ecce51..81dae475a1d 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -133,6 +133,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index b724c217ed4..da631cd0dc1 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -397,6 +397,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 92806fa04a2..b42b5ec68a7 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -187,6 +187,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index b46773632cb..921eaf3780d 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -517,6 +517,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 6ffc1eab708..c7d0106dc57 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -136,6 +136,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr