From ef8ef3b8f3c6226de34d311de8ec9918dfcc6ba9 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 9 Dec 2016 23:03:33 -0800 Subject: [PATCH 001/167] Tests for pollset_set. --- Makefile | 36 ++ build.yaml | 14 + test/core/iomgr/pollset_set_test.c | 472 +++++++++++++++++++++++ tools/run_tests/sources_and_headers.json | 17 + tools/run_tests/tests.json | 18 + 5 files changed, 557 insertions(+) create mode 100644 test/core/iomgr/pollset_set_test.c diff --git a/Makefile b/Makefile index 3a228b4414f..3f4430da9ad 100644 --- a/Makefile +++ b/Makefile @@ -1023,6 +1023,7 @@ no_server_test: $(BINDIR)/$(CONFIG)/no_server_test percent_decode_fuzzer: $(BINDIR)/$(CONFIG)/percent_decode_fuzzer percent_encode_fuzzer: $(BINDIR)/$(CONFIG)/percent_encode_fuzzer percent_encoding_test: $(BINDIR)/$(CONFIG)/percent_encoding_test +pollset_set_test: $(BINDIR)/$(CONFIG)/pollset_set_test resolve_address_test: $(BINDIR)/$(CONFIG)/resolve_address_test resource_quota_test: $(BINDIR)/$(CONFIG)/resource_quota_test secure_channel_create_test: $(BINDIR)/$(CONFIG)/secure_channel_create_test @@ -1354,6 +1355,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/murmur_hash_test \ $(BINDIR)/$(CONFIG)/no_server_test \ $(BINDIR)/$(CONFIG)/percent_encoding_test \ + $(BINDIR)/$(CONFIG)/pollset_set_test \ $(BINDIR)/$(CONFIG)/resolve_address_test \ $(BINDIR)/$(CONFIG)/resource_quota_test \ $(BINDIR)/$(CONFIG)/secure_channel_create_test \ @@ -1749,6 +1751,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/no_server_test || ( echo test no_server_test failed ; exit 1 ) $(E) "[RUN] Testing percent_encoding_test" $(Q) $(BINDIR)/$(CONFIG)/percent_encoding_test || ( echo test percent_encoding_test failed ; exit 1 ) + $(E) "[RUN] Testing pollset_set_test" + $(Q) $(BINDIR)/$(CONFIG)/pollset_set_test || ( echo test pollset_set_test failed ; exit 1 ) $(E) "[RUN] Testing resolve_address_test" $(Q) $(BINDIR)/$(CONFIG)/resolve_address_test || ( echo test resolve_address_test failed ; exit 1 ) $(E) "[RUN] Testing resource_quota_test" @@ -10603,6 +10607,38 @@ endif endif +POLLSET_SET_TEST_SRC = \ + test/core/iomgr/pollset_set_test.c \ + +POLLSET_SET_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(POLLSET_SET_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/pollset_set_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/pollset_set_test: $(POLLSET_SET_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(POLLSET_SET_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/pollset_set_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/iomgr/pollset_set_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_pollset_set_test: $(POLLSET_SET_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(POLLSET_SET_TEST_OBJS:.o=.dep) +endif +endif + + RESOLVE_ADDRESS_TEST_SRC = \ test/core/iomgr/resolve_address_test.c \ diff --git a/build.yaml b/build.yaml index 2555abce0c6..ef2dde591e7 100644 --- a/build.yaml +++ b/build.yaml @@ -2429,6 +2429,20 @@ targets: - grpc - gpr_test_util - gpr +- name: pollset_set_test + build: test + language: c + src: + - test/core/iomgr/pollset_set_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + exclude_iomgrs: + - uv + platforms: + - linux - name: resolve_address_test build: test language: c diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c new file mode 100644 index 00000000000..af971c0784e --- /dev/null +++ b/test/core/iomgr/pollset_set_test.c @@ -0,0 +1,472 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include "src/core/lib/iomgr/port.h" + +/* This test only relevant on linux systems */ +#ifdef GRPC_POSIX_SOCKET +#include "src/core/lib/iomgr/ev_posix.h" + +#include +#include +#include + +#include +#include + +#include "src/core/lib/iomgr/iomgr.h" +#include "test/core/util/test_config.h" + +/******************************************************************************* + * test_pollset_set + */ + +typedef struct test_pollset_set { grpc_pollset_set *pss; } test_pollset_set; + +void init_test_pollset_sets(test_pollset_set pollset_sets[], int num_pss) { + int i; + for (i = 0; i < num_pss; i++) { + pollset_sets[i].pss = grpc_pollset_set_create(); + } +} + +void cleanup_test_pollset_sets(test_pollset_set pollset_sets[], int num_pss) { + int i; + for (i = 0; i < num_pss; i++) { + grpc_pollset_set_destroy(pollset_sets[i].pss); + pollset_sets[i].pss = NULL; + } +} + +/******************************************************************************* + * test_pollset + */ + +typedef struct test_pollset { + grpc_pollset *ps; + gpr_mu *mu; +} test_pollset; + +static void init_test_pollsets(test_pollset pollsets[], int num_pollsets) { + int i; + for (i = 0; i < num_pollsets; i++) { + pollsets[i].ps = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(pollsets[i].ps, &pollsets[i].mu); + } +} + +static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, + grpc_error *error) { + grpc_pollset_destroy(p); +} + +static void cleanup_test_pollsets(grpc_exec_ctx *exec_ctx, + test_pollset pollsets[], int num_pollsets) { + grpc_closure destroyed; + int i; + + for (i = 0; i < num_pollsets; i++) { + grpc_closure_init(&destroyed, destroy_pollset, pollsets[i].ps); + grpc_pollset_shutdown(exec_ctx, pollsets[i].ps, &destroyed); + + grpc_exec_ctx_flush(exec_ctx); + gpr_free(pollsets[i].ps); + pollsets[i].ps = NULL; + } +} + +/******************************************************************************* + * test_fd + */ + +typedef struct test_fd { + grpc_fd *fd; + grpc_wakeup_fd wakeup_fd; + + bool is_on_readable_called; /* Is on_readable closure is called ? */ + grpc_closure on_readable; /* Closure to call when this fd is readable */ +} test_fd; + +void on_readable(grpc_exec_ctx *exec_ctx, void *tfd, grpc_error *error) { + ((test_fd *)tfd)->is_on_readable_called = true; +} + +static void reset_test_fd(grpc_exec_ctx *exec_ctx, test_fd *tfd) { + tfd->is_on_readable_called = false; + + grpc_closure_init(&tfd->on_readable, on_readable, tfd); + grpc_fd_notify_on_read(exec_ctx, tfd->fd, &tfd->on_readable); +} + +static void init_test_fds(grpc_exec_ctx *exec_ctx, test_fd tfds[], + int num_fds) { + int i; + + for (i = 0; i < num_fds; i++) { + GPR_ASSERT(GRPC_ERROR_NONE == grpc_wakeup_fd_init(&tfds[i].wakeup_fd)); + tfds[i].fd = grpc_fd_create(GRPC_WAKEUP_FD_GET_READ_FD(&tfds[i].wakeup_fd), + "test_fd"); + reset_test_fd(exec_ctx, &tfds[i]); + } +} + +static void cleanup_test_fds(grpc_exec_ctx *exec_ctx, test_fd *tfds, + int num_fds) { + int release_fd; + int i; + + for (i = 0; i < num_fds; i++) { + grpc_fd_shutdown(exec_ctx, tfds[i].fd); + grpc_exec_ctx_flush(exec_ctx); + + /* grpc_fd_orphan frees the memory allocated for grpc_fd. Normally it also + * calls close() on the underlying fd. In our case, we are using + * grpc_wakeup_fd and we would like to destroy it ourselves (by calling + * grpc_wakeup_fd_destroy). To prevent grpc_fd from calling close() on the + * underlying fd, call it with a non-NULL 'release_fd' parameter */ + grpc_fd_orphan(exec_ctx, tfds[i].fd, NULL, &release_fd, "test_fd_cleanup"); + grpc_exec_ctx_flush(exec_ctx); + + grpc_wakeup_fd_destroy(&tfds[i].wakeup_fd); + } +} + +static void make_test_fds_readable(test_fd tfds[], int num_fds) { + int i; + for (i = 0; i < num_fds; i++) { + GPR_ASSERT(GRPC_ERROR_NONE == grpc_wakeup_fd_wakeup(&tfds[i].wakeup_fd)); + } +} + +static void verify_readable_and_reset(grpc_exec_ctx *exec_ctx, test_fd tfds[], + int num_fds) { + int i; + for (i = 0; i < num_fds; i++) { + /* Verify that the on_readable callback was called */ + GPR_ASSERT(tfds[i].is_on_readable_called); + + /* Reset the tfd[i] structure */ + GPR_ASSERT(GRPC_ERROR_NONE == + grpc_wakeup_fd_consume_wakeup(&tfds[i].wakeup_fd)); + reset_test_fd(exec_ctx, &tfds[i]); + } +} + +/******************************************************************************* + * Main tests + */ + +/* Test some typical scenarios in pollset_set */ +static void pollset_set_test_basic() { + /* We construct the following structure for this test: + * + * +---> FD0 (Added before PSS1, PS1 and PS2 are added to PSS0) + * | + * +---> FD5 (Added after PSS1, PS1 and PS2 are added to PSS0) + * | + * | + * | +---> FD1 (Added before PSS1 is added to PSS0) + * | | + * | +---> FD6 (Added after PSS1 is added to PSS0) + * | | + * +---> PSS1--+ +--> FD2 (Added before PS0 is added to PSS1) + * | | | + * | +---> PS0---+ + * | | + * PSS0---+ +--> FD7 (Added after PS0 is added to PSS1) + * | + * | + * | +---> FD3 (Added before PS1 is added to PSS0) + * | | + * +---> PS1---+ + * | | + * | +---> FD8 (Added after PS1 added to PSS0) + * | + * | + * | +---> FD4 (Added before PS2 is added to PSS0) + * | | + * +---> PS2---+ + * | + * +---> FD9 (Added after PS2 is added to PSS0) + */ + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + int i; + grpc_pollset_worker *worker; + gpr_timespec deadline; + + test_fd tfds[10]; + test_pollset pollsets[3]; + test_pollset_set pollset_sets[2]; + int num_fds = sizeof(tfds) / sizeof(tfds[0]); + int num_ps = sizeof(pollsets) / sizeof(pollsets[0]); + int num_pss = sizeof(pollset_sets) / sizeof(pollset_sets[0]); + + init_test_fds(&exec_ctx, tfds, num_fds); + init_test_pollsets(pollsets, num_ps); + init_test_pollset_sets(pollset_sets, num_pss); + + /* Construct the pollset_set/pollset/fd tree (see diagram above) */ + + grpc_pollset_set_add_fd(&exec_ctx, pollset_sets[0].pss, tfds[0].fd); + grpc_pollset_set_add_fd(&exec_ctx, pollset_sets[1].pss, tfds[1].fd); + + grpc_pollset_add_fd(&exec_ctx, pollsets[0].ps, tfds[2].fd); + grpc_pollset_add_fd(&exec_ctx, pollsets[1].ps, tfds[3].fd); + grpc_pollset_add_fd(&exec_ctx, pollsets[2].ps, tfds[4].fd); + + grpc_pollset_set_add_pollset_set(&exec_ctx, pollset_sets[0].pss, + pollset_sets[1].pss); + + grpc_pollset_set_add_pollset(&exec_ctx, pollset_sets[1].pss, pollsets[0].ps); + grpc_pollset_set_add_pollset(&exec_ctx, pollset_sets[0].pss, pollsets[1].ps); + grpc_pollset_set_add_pollset(&exec_ctx, pollset_sets[0].pss, pollsets[2].ps); + + grpc_pollset_set_add_fd(&exec_ctx, pollset_sets[0].pss, tfds[5].fd); + grpc_pollset_set_add_fd(&exec_ctx, pollset_sets[1].pss, tfds[6].fd); + + grpc_pollset_add_fd(&exec_ctx, pollsets[0].ps, tfds[7].fd); + grpc_pollset_add_fd(&exec_ctx, pollsets[1].ps, tfds[8].fd); + grpc_pollset_add_fd(&exec_ctx, pollsets[2].ps, tfds[9].fd); + + grpc_exec_ctx_flush(&exec_ctx); + + /* Test that if any FD in the above structure is readable, it is observable by + * doing grpc_pollset_work on any pollset + * + * For every pollset, do the following: + * - (Ensure that all FDs are in reset state) + * - Make all FDs readable + * - Call grpc_pollset_work() on the pollset + * - Flush the exec_ctx + * - Verify that on_readable call back was called for all FDs (and + * reset the FDs) + * */ + for (i = 0; i < num_ps; i++) { + make_test_fds_readable(tfds, num_fds); + + gpr_mu_lock(pollsets[i].mu); + deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(2); + GPR_ASSERT(GRPC_ERROR_NONE == + grpc_pollset_work(&exec_ctx, pollsets[i].ps, &worker, + gpr_now(GPR_CLOCK_MONOTONIC), deadline)); + gpr_mu_unlock(pollsets[i].mu); + + grpc_exec_ctx_flush(&exec_ctx); + + verify_readable_and_reset(&exec_ctx, tfds, num_fds); + grpc_exec_ctx_flush(&exec_ctx); + } + + /* Test tear down */ + grpc_pollset_set_del_fd(&exec_ctx, pollset_sets[0].pss, tfds[0].fd); + grpc_pollset_set_del_fd(&exec_ctx, pollset_sets[0].pss, tfds[5].fd); + grpc_pollset_set_del_fd(&exec_ctx, pollset_sets[1].pss, tfds[1].fd); + grpc_pollset_set_del_fd(&exec_ctx, pollset_sets[1].pss, tfds[6].fd); + grpc_exec_ctx_flush(&exec_ctx); + + grpc_pollset_set_del_pollset(&exec_ctx, pollset_sets[1].pss, pollsets[0].ps); + grpc_pollset_set_del_pollset(&exec_ctx, pollset_sets[0].pss, pollsets[1].ps); + grpc_pollset_set_del_pollset(&exec_ctx, pollset_sets[0].pss, pollsets[2].ps); + + grpc_pollset_set_del_pollset_set(&exec_ctx, pollset_sets[0].pss, + pollset_sets[1].pss); + grpc_exec_ctx_flush(&exec_ctx); + + cleanup_test_fds(&exec_ctx, tfds, num_fds); + cleanup_test_pollsets(&exec_ctx, pollsets, num_ps); + cleanup_test_pollset_sets(pollset_sets, num_pss); + grpc_exec_ctx_finish(&exec_ctx); +} + +/* Same FD added multiple times to the pollset_set tree */ +void pollset_set_test_dup_fds() { + /* We construct the following structure for this test: + * + * +---> FD0 + * | + * | + * PSS0---+ + * | +---> FD0 (also under PSS0) + * | | + * +---> PSS1--+ +--> FD1 (also under PSS1) + * | | + * +---> PS ---+ + * | | + * | +--> FD2 + * +---> FD1 + */ + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_pollset_worker *worker; + gpr_timespec deadline; + + test_fd tfds[3]; + test_pollset pollset; + test_pollset_set pollset_sets[2]; + int num_fds = sizeof(tfds) / sizeof(tfds[0]); + int num_ps = 1; + int num_pss = sizeof(pollset_sets) / sizeof(pollset_sets[0]); + + init_test_fds(&exec_ctx, tfds, num_fds); + init_test_pollsets(&pollset, num_ps); + init_test_pollset_sets(pollset_sets, num_pss); + + /* Construct the structure */ + grpc_pollset_set_add_fd(&exec_ctx, pollset_sets[0].pss, tfds[0].fd); + grpc_pollset_set_add_fd(&exec_ctx, pollset_sets[1].pss, tfds[0].fd); + grpc_pollset_set_add_fd(&exec_ctx, pollset_sets[1].pss, tfds[1].fd); + + grpc_pollset_add_fd(&exec_ctx, pollset.ps, tfds[1].fd); + grpc_pollset_add_fd(&exec_ctx, pollset.ps, tfds[2].fd); + + grpc_pollset_set_add_pollset(&exec_ctx, pollset_sets[1].pss, pollset.ps); + grpc_pollset_set_add_pollset_set(&exec_ctx, pollset_sets[0].pss, + pollset_sets[1].pss); + + /* Test. Make all FDs readable and make sure that can be observed by doing a + * grpc_pollset_work on the pollset 'PS' */ + make_test_fds_readable(tfds, num_fds); + + gpr_mu_lock(pollset.mu); + deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(2); + GPR_ASSERT(GRPC_ERROR_NONE == + grpc_pollset_work(&exec_ctx, pollset.ps, &worker, + gpr_now(GPR_CLOCK_MONOTONIC), deadline)); + gpr_mu_unlock(pollset.mu); + grpc_exec_ctx_flush(&exec_ctx); + + verify_readable_and_reset(&exec_ctx, tfds, num_fds); + grpc_exec_ctx_flush(&exec_ctx); + + /* Tear down */ + grpc_pollset_set_del_fd(&exec_ctx, pollset_sets[0].pss, tfds[0].fd); + grpc_pollset_set_del_fd(&exec_ctx, pollset_sets[1].pss, tfds[0].fd); + grpc_pollset_set_del_fd(&exec_ctx, pollset_sets[1].pss, tfds[1].fd); + + grpc_pollset_set_del_pollset(&exec_ctx, pollset_sets[1].pss, pollset.ps); + grpc_pollset_set_del_pollset_set(&exec_ctx, pollset_sets[0].pss, + pollset_sets[1].pss); + grpc_exec_ctx_flush(&exec_ctx); + + cleanup_test_fds(&exec_ctx, tfds, num_fds); + cleanup_test_pollsets(&exec_ctx, &pollset, num_ps); + cleanup_test_pollset_sets(pollset_sets, num_pss); + grpc_exec_ctx_finish(&exec_ctx); +} + +/* Pollset_set with an empty pollset */ +void pollset_set_test_empty_pollset() { + /* We construct the following structure for this test: + * + * +---> PS0 (EMPTY) + * | + * +---> FD0 + * | + * PSS0---+ + * | +---> FD1 + * | | + * +---> PS1--+ + * | + * +---> FD2 + */ + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_pollset_worker *worker; + gpr_timespec deadline; + + test_fd tfds[3]; + test_pollset pollsets[2]; + test_pollset_set pollset_set; + int num_fds = sizeof(tfds) / sizeof(tfds[0]); + int num_ps = sizeof(pollsets) / sizeof(pollsets[0]); + int num_pss = 1; + + init_test_fds(&exec_ctx, tfds, num_fds); + init_test_pollsets(pollsets, num_ps); + init_test_pollset_sets(&pollset_set, num_pss); + + /* Construct the structure */ + grpc_pollset_set_add_fd(&exec_ctx, pollset_set.pss, tfds[0].fd); + grpc_pollset_add_fd(&exec_ctx, pollsets[1].ps, tfds[1].fd); + grpc_pollset_add_fd(&exec_ctx, pollsets[1].ps, tfds[2].fd); + + grpc_pollset_set_add_pollset(&exec_ctx, pollset_set.pss, pollsets[0].ps); + grpc_pollset_set_add_pollset(&exec_ctx, pollset_set.pss, pollsets[1].ps); + + /* Test. Make all FDs readable and make sure that can be observed by doing + * grpc_pollset_work on the empty pollset 'PS0' */ + make_test_fds_readable(tfds, num_fds); + + gpr_mu_lock(pollsets[0].mu); + deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(2); + GPR_ASSERT(GRPC_ERROR_NONE == + grpc_pollset_work(&exec_ctx, pollsets[0].ps, &worker, + gpr_now(GPR_CLOCK_MONOTONIC), deadline)); + gpr_mu_unlock(pollsets[0].mu); + grpc_exec_ctx_flush(&exec_ctx); + + verify_readable_and_reset(&exec_ctx, tfds, num_fds); + grpc_exec_ctx_flush(&exec_ctx); + + /* Tear down */ + grpc_pollset_set_del_fd(&exec_ctx, pollset_set.pss, tfds[0].fd); + grpc_pollset_set_del_pollset(&exec_ctx, pollset_set.pss, pollsets[0].ps); + grpc_pollset_set_del_pollset(&exec_ctx, pollset_set.pss, pollsets[1].ps); + grpc_exec_ctx_flush(&exec_ctx); + + cleanup_test_fds(&exec_ctx, tfds, num_fds); + cleanup_test_pollsets(&exec_ctx, pollsets, num_ps); + cleanup_test_pollset_sets(&pollset_set, num_pss); + grpc_exec_ctx_finish(&exec_ctx); +} + +int main(int argc, char **argv) { + const char *poll_strategy = NULL; + grpc_test_init(argc, argv); + grpc_iomgr_init(); + + poll_strategy = grpc_get_poll_strategy_name(); + if (poll_strategy != NULL && strcmp(poll_strategy, "epoll") == 0) { + pollset_set_test_basic(); + pollset_set_test_dup_fds(); + pollset_set_test_empty_pollset(); + } else { + gpr_log(GPR_INFO, + "Skipping the test. The test is only relevant for 'epoll' " + "strategy. and the current strategy is: '%s'", + poll_strategy); + } + + grpc_iomgr_shutdown(); + return 0; +} +#else /* defined(GRPC_LINUX_EPOLL) */ +int main(int argc, char **argv) { return 0; } +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 9dd5a1c85f0..3e61487ac65 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -1676,6 +1676,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "pollset_set_test", + "src": [ + "test/core/iomgr/pollset_set_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 8d1e9b45683..d96fa419224 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -1697,6 +1697,24 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "gtest": false, + "language": "c", + "name": "pollset_set_test", + "platforms": [ + "linux" + ] + }, { "args": [], "ci_platforms": [ From 9c0e9f1403c6d35b1ffdddb8cd1fb71f43648242 Mon Sep 17 00:00:00 2001 From: Jeremie Girault Date: Tue, 4 Oct 2016 13:26:30 +0200 Subject: [PATCH 002/167] fix build on sierra by adding OSATOMIC_USE_INLINED=1 flag to compiler --- Makefile | 2 +- build.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8f7328ae285..5128fac3f57 100644 --- a/Makefile +++ b/Makefile @@ -351,7 +351,7 @@ CXXFLAGS += -std=c++11 else CXXFLAGS += -std=c++0x endif -CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter +CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 LDFLAGS += -g CPPFLAGS += $(CPPFLAGS_$(CONFIG)) diff --git a/build.yaml b/build.yaml index de9d253ef1c..1217bb5912c 100644 --- a/build.yaml +++ b/build.yaml @@ -3794,7 +3794,7 @@ defaults: CPPFLAGS: -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX global: - CPPFLAGS: -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter + CPPFLAGS: -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 LDFLAGS: -g zlib: CFLAGS: -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-implicit-function-declaration From 740ae63a8a816369b18b8ac9602a9acc7cc0b30d Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 21 Oct 2016 13:59:14 -0700 Subject: [PATCH 003/167] Packet coalescing transport layer and end2end test changes --- .../cronet/transport/cronet_api_dummy.c | 3 +- .../cronet/transport/cronet_transport.c | 146 ++++++--- .../CoreCronetEnd2EndTests.m | 4 + src/objective-c/tests/Podfile | 1 + .../tests/Tests.xcodeproj/project.pbxproj | 1 + test/core/end2end/end2end_nosec_tests.c | 10 + test/core/end2end/end2end_tests.c | 8 + test/core/end2end/tests/packet_coalescing.c | 288 ++++++++++++++++++ .../objective_c/Cronet/cronet_c_for_grpc.h | 115 ++++--- 9 files changed, 496 insertions(+), 80 deletions(-) create mode 100644 test/core/end2end/tests/packet_coalescing.c diff --git a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c index 687026c9fde..38755604b97 100644 --- a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c +++ b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c @@ -77,9 +77,8 @@ int cronet_bidirectional_stream_write(cronet_bidirectional_stream* stream, return 0; } -int cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream) { +void cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream) { GPR_ASSERT(0); - return 0; } #endif /* GRPC_COMPILE_WITH_CRONET */ diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index afc59f4b12a..4063dcaefcb 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -86,7 +86,7 @@ enum e_op_id { /* Cronet callbacks. See cronet_c_for_grpc.h for documentation for each. */ -static void on_request_headers_sent(cronet_bidirectional_stream *); +static void on_stream_ready(cronet_bidirectional_stream *); static void on_response_headers_received( cronet_bidirectional_stream *, const cronet_bidirectional_stream_header_array *, const char *); @@ -99,7 +99,7 @@ static void on_succeeded(cronet_bidirectional_stream *); static void on_failed(cronet_bidirectional_stream *, int); static void on_canceled(cronet_bidirectional_stream *); static cronet_bidirectional_stream_callback cronet_callbacks = { - on_request_headers_sent, + on_stream_ready, on_response_headers_received, on_read_completed, on_write_completed, @@ -151,6 +151,11 @@ struct op_state { bool state_callback_received[OP_NUM_OPS]; bool fail_state; bool flush_read; +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + bool flush_cronet_when_ready; + bool pending_write_for_trailer; +#endif + bool unprocessed_send_message; grpc_error *cancel_error; /* data structure for storing data coming from server */ struct read_state rs; @@ -273,6 +278,9 @@ static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { new_op->next = storage->head; storage->head = new_op; storage->num_pending_ops++; + if (op->send_message) { + s->state.unprocessed_send_message = true; + } CRONET_LOG(GPR_DEBUG, "adding new op %p. %d in the queue.", new_op, storage->num_pending_ops); gpr_mu_unlock(&s->mu); @@ -405,8 +413,8 @@ static void on_succeeded(cronet_bidirectional_stream *stream) { /* Cronet callback */ -static void on_request_headers_sent(cronet_bidirectional_stream *stream) { - CRONET_LOG(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); +static void on_stream_ready(cronet_bidirectional_stream *stream) { + CRONET_LOG(GPR_DEBUG, "W: on_stream_ready(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; gpr_mu_lock(&s->mu); s->state.state_op_done[OP_SEND_INITIAL_METADATA] = true; @@ -416,6 +424,14 @@ static void on_request_headers_sent(cronet_bidirectional_stream *stream) { gpr_free(s->header_array.headers); s->header_array.headers = NULL; } +/* Send the initial metadata on wire if there is no SEND_MESSAGE or + * SEND_TRAILING_METADATA ops pending */ +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + if (s->state.flush_cronet_when_ready) { + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_flush (%p)", s->cbs); + cronet_bidirectional_stream_flush(stream); + } +#endif gpr_mu_unlock(&s->mu); execute_from_storage(s); } @@ -551,6 +567,10 @@ static void on_response_trailers_received( CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p, 0)", s->cbs); s->state.state_callback_received[OP_SEND_MESSAGE] = false; cronet_bidirectional_stream_write(s->cbs, "", 0, true); +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_flush (%p)", s->cbs); + cronet_bidirectional_stream_flush(s->cbs); +#endif s->state.state_op_done[OP_SEND_TRAILING_METADATA] = true; gpr_mu_unlock(&s->mu); @@ -598,7 +618,7 @@ static void convert_metadata_to_cronet_headers( curr = curr->next; num_headers_available++; } - /* Allocate enough memory. It is freed in the on_request_headers_sent callback + /* Allocate enough memory. It is freed in the on_stream_ready callback */ cronet_bidirectional_stream_header *headers = (cronet_bidirectional_stream_header *)gpr_malloc( @@ -740,12 +760,16 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; /* we haven't sent message yet */ - else if (curr_op->send_message && + else if (stream_state->unprocessed_send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; /* we haven't got on_write_completed for the send yet */ else if (stream_state->state_op_done[OP_SEND_MESSAGE] && - !stream_state->state_callback_received[OP_SEND_MESSAGE]) + !stream_state->state_callback_received[OP_SEND_MESSAGE] +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + && !stream_state->pending_write_for_trailer +#endif + ) result = false; } else if (op_id == OP_CANCEL_ERROR) { /* already executed */ @@ -831,6 +855,10 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); CRONET_LOG(GPR_DEBUG, "%p = cronet_bidirectional_stream_create()", s->cbs); +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + cronet_bidirectional_stream_disable_auto_flush(s->cbs, true); + cronet_bidirectional_stream_delay_request_headers_until_flush(s->cbs, true); +#endif char *url = NULL; const char *method = "POST"; s->header_array.headers = NULL; @@ -843,30 +871,17 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, cronet_bidirectional_stream_start(s->cbs, url, 0, method, &s->header_array, false); stream_state->state_op_done[OP_SEND_INITIAL_METADATA] = true; - result = ACTION_TAKEN_WITH_CALLBACK; - } else if (stream_op->recv_initial_metadata && - op_can_be_run(stream_op, stream_state, &oas->state, - OP_RECV_INITIAL_METADATA)) { - CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_INITIAL_METADATA", oas); - if (stream_state->state_op_done[OP_CANCEL_ERROR]) { - grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready, - GRPC_ERROR_CANCELLED, NULL); - } else if (stream_state->state_callback_received[OP_FAILED]) { - grpc_exec_ctx_sched( - exec_ctx, stream_op->recv_initial_metadata_ready, - make_error_with_desc(GRPC_STATUS_UNAVAILABLE, "Unavailable."), NULL); - } else { - grpc_chttp2_incoming_metadata_buffer_publish( - &oas->s->state.rs.initial_metadata, stream_op->recv_initial_metadata); - grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready, - GRPC_ERROR_NONE, NULL); +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + if (!stream_op->send_message && !stream_op->send_trailing_metadata) { + s->state.flush_cronet_when_ready = true; } - stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; - result = ACTION_TAKEN_NO_CALLBACK; +#endif + result = ACTION_TAKEN_WITH_CALLBACK; } else if (stream_op->send_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_MESSAGE", oas); + stream_state->unprocessed_send_message = false; if (stream_state->state_callback_received[OP_FAILED]) { result = NO_ACTION_POSSIBLE; CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed."); @@ -897,13 +912,63 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->state_callback_received[OP_SEND_MESSAGE] = false; cronet_bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, (int)write_buffer_size, false); +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + if (!stream_op->send_trailing_metadata) { + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_flush (%p)", + s->cbs); + cronet_bidirectional_stream_flush(s->cbs); + result = ACTION_TAKEN_WITH_CALLBACK; + } else { + stream_state->pending_write_for_trailer = true; + result = ACTION_TAKEN_NO_CALLBACK; + } +#else result = ACTION_TAKEN_WITH_CALLBACK; +#endif } else { result = NO_ACTION_POSSIBLE; } } stream_state->state_op_done[OP_SEND_MESSAGE] = true; oas->state.state_op_done[OP_SEND_MESSAGE] = true; + } else if (stream_op->send_trailing_metadata && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_SEND_TRAILING_METADATA)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_TRAILING_METADATA", oas); + if (stream_state->state_callback_received[OP_FAILED]) { + result = NO_ACTION_POSSIBLE; + CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed."); + } else { + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p, 0)", + s->cbs); + stream_state->state_callback_received[OP_SEND_MESSAGE] = false; + cronet_bidirectional_stream_write(s->cbs, "", 0, true); +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_flush (%p)", s->cbs); + cronet_bidirectional_stream_flush(s->cbs); +#endif + result = ACTION_TAKEN_WITH_CALLBACK; + } + stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true; + } else if (stream_op->recv_initial_metadata && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_RECV_INITIAL_METADATA)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_INITIAL_METADATA", oas); + if (stream_state->state_op_done[OP_CANCEL_ERROR]) { + grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready, + GRPC_ERROR_CANCELLED, NULL); + } else if (stream_state->state_callback_received[OP_FAILED]) { + grpc_exec_ctx_sched( + exec_ctx, stream_op->recv_initial_metadata_ready, + make_error_with_desc(GRPC_STATUS_UNAVAILABLE, "Unavailable."), NULL); + } else { + grpc_chttp2_incoming_metadata_buffer_publish( + &oas->s->state.rs.initial_metadata, stream_op->recv_initial_metadata); + grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready, + GRPC_ERROR_NONE, NULL); + } + stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; + result = ACTION_TAKEN_NO_CALLBACK; } else if (stream_op->recv_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_MESSAGE)) { @@ -962,6 +1027,16 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; + + /* Extra read to trigger on_succeed */ + stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes; + stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; + stream_state->rs.received_bytes = 0; + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read(%p)", s->cbs); + stream_state->state_op_done[OP_READ_REQ_MADE] = + true; /* Indicates that at least one read request has been made */ + cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, + stream_state->rs.remaining_bytes); result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_state->rs.remaining_bytes == 0) { @@ -1020,21 +1095,6 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, } stream_state->state_op_done[OP_RECV_TRAILING_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; - } else if (stream_op->send_trailing_metadata && - op_can_be_run(stream_op, stream_state, &oas->state, - OP_SEND_TRAILING_METADATA)) { - CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_TRAILING_METADATA", oas); - if (stream_state->state_callback_received[OP_FAILED]) { - result = NO_ACTION_POSSIBLE; - CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed."); - } else { - CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p, 0)", - s->cbs); - stream_state->state_callback_received[OP_SEND_MESSAGE] = false; - cronet_bidirectional_stream_write(s->cbs, "", 0, true); - result = ACTION_TAKEN_WITH_CALLBACK; - } - stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true; } else if (stream_op->cancel_error && op_can_be_run(stream_op, stream_state, &oas->state, OP_CANCEL_ERROR)) { @@ -1117,6 +1177,10 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, sizeof(s->state.state_callback_received)); s->state.fail_state = s->state.flush_read = false; s->state.cancel_error = NULL; +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + s->state.flush_cronet_when_ready = s->state.pending_write_for_trailer = false; +#endif + s->state.unprocessed_send_message = false; gpr_mu_init(&s->mu); return 0; } diff --git a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m index 4ba7badd866..fe9a7c2247b 100644 --- a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m +++ b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m @@ -346,6 +346,10 @@ static char *roots_filename; [self testIndividualCase:"no_op"]; } +- (void)testPacketCoalescing { + [self testIndividualCase:"packet_coalescing"]; +} + - (void)testPayload { [self testIndividualCase:"payload"]; } diff --git a/src/objective-c/tests/Podfile b/src/objective-c/tests/Podfile index 5785b976f2d..d1ef0886fe4 100644 --- a/src/objective-c/tests/Podfile +++ b/src/objective-c/tests/Podfile @@ -92,6 +92,7 @@ post_install do |installer| # GPR_UNREACHABLE_CODE causes "Control may reach end of non-void # function" warning config.build_settings['GCC_WARN_ABOUT_RETURN_TYPE'] = 'NO' + config.build_settings['GCC_PREPROCESSOR_DEFINITIONS'] = '$(inherited) COCOAPODS=1 GRPC_CRONET_WITH_PACKET_COALESCING=1' end end diff --git a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj index c4a6567ae0e..8455e71b026 100644 --- a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj +++ b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj @@ -1296,6 +1296,7 @@ "$(inherited)", "GPB_USE_PROTOBUF_FRAMEWORK_IMPORTS=1", "GRPC_COMPILE_WITH_CRONET=1", + "GRPC_CRONET_WITH_PACKET_COALESCING=1", ); INFOPLIST_FILE = InteropTestsRemoteWithCronet/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 9.3; diff --git a/test/core/end2end/end2end_nosec_tests.c b/test/core/end2end/end2end_nosec_tests.c index 663489082f9..86d6ab9c9a7 100644 --- a/test/core/end2end/end2end_nosec_tests.c +++ b/test/core/end2end/end2end_nosec_tests.c @@ -103,6 +103,8 @@ extern void no_logging(grpc_end2end_test_config config); extern void no_logging_pre_init(void); extern void no_op(grpc_end2end_test_config config); extern void no_op_pre_init(void); +extern void packet_coalescing(grpc_end2end_test_config config); +extern void packet_coalescing_pre_init(void); extern void payload(grpc_end2end_test_config config); extern void payload_pre_init(void); extern void ping(grpc_end2end_test_config config); @@ -135,6 +137,8 @@ extern void streaming_error_response(grpc_end2end_test_config config); extern void streaming_error_response_pre_init(void); extern void trailing_metadata(grpc_end2end_test_config config); extern void trailing_metadata_pre_init(void); +extern void packet_coalescing(grpc_end2end_test_config config); +extern void packet_coalescing_pre_init(void); void grpc_end2end_tests_pre_init(void) { GPR_ASSERT(!g_pre_init_called); @@ -169,6 +173,7 @@ void grpc_end2end_tests_pre_init(void) { network_status_change_pre_init(); no_logging_pre_init(); no_op_pre_init(); + packet_coalescing_pre_init(); payload_pre_init(); ping_pre_init(); ping_pong_streaming_pre_init(); @@ -224,6 +229,7 @@ void grpc_end2end_tests(int argc, char **argv, network_status_change(config); no_logging(config); no_op(config); + packet_coalescing(config); payload(config); ping(config); ping_pong_streaming(config); @@ -364,6 +370,10 @@ void grpc_end2end_tests(int argc, char **argv, no_op(config); continue; } + if (0 == strcmp("packet_coalescing", argv[i])) { + packet_coalescing(config); + continue; + } if (0 == strcmp("payload", argv[i])) { payload(config); continue; diff --git a/test/core/end2end/end2end_tests.c b/test/core/end2end/end2end_tests.c index 25c7c62fde5..66908bab606 100644 --- a/test/core/end2end/end2end_tests.c +++ b/test/core/end2end/end2end_tests.c @@ -105,6 +105,8 @@ extern void no_logging(grpc_end2end_test_config config); extern void no_logging_pre_init(void); extern void no_op(grpc_end2end_test_config config); extern void no_op_pre_init(void); +extern void packet_coalescing(grpc_end2end_test_config config); +extern void packet_coalescing_pre_init(void); extern void payload(grpc_end2end_test_config config); extern void payload_pre_init(void); extern void ping(grpc_end2end_test_config config); @@ -172,6 +174,7 @@ void grpc_end2end_tests_pre_init(void) { network_status_change_pre_init(); no_logging_pre_init(); no_op_pre_init(); + packet_coalescing_pre_init(); payload_pre_init(); ping_pre_init(); ping_pong_streaming_pre_init(); @@ -228,6 +231,7 @@ void grpc_end2end_tests(int argc, char **argv, network_status_change(config); no_logging(config); no_op(config); + packet_coalescing(config); payload(config); ping(config); ping_pong_streaming(config); @@ -372,6 +376,10 @@ void grpc_end2end_tests(int argc, char **argv, no_op(config); continue; } + if (0 == strcmp("packet_coalescing", argv[i])) { + packet_coalescing(config); + continue; + } if (0 == strcmp("payload", argv[i])) { payload(config); continue; diff --git a/test/core/end2end/tests/packet_coalescing.c b/test/core/end2end/tests/packet_coalescing.c new file mode 100644 index 00000000000..d560c57da80 --- /dev/null +++ b/test/core/end2end/tests/packet_coalescing.c @@ -0,0 +1,288 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include "test/core/end2end/end2end_tests.h" + +#include +#include + +#include +#include +#include +#include +#include +#include "test/core/end2end/cq_verifier.h" + +extern void gpr_default_log(gpr_log_func_args *args); + +static void *tag(intptr_t t) { return (void *)t; } + +static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, + const char *test_name, + grpc_channel_args *client_args, + grpc_channel_args *server_args) { + grpc_end2end_test_fixture f; + gpr_log(GPR_INFO, "%s/%s", test_name, config.name); + f = config.create_fixture(client_args, server_args); + config.init_server(&f, server_args); + config.init_client(&f, client_args); + return f; +} + +static gpr_timespec n_seconds_time(int n) { + return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(n); +} + +static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } + +static void drain_cq(grpc_completion_queue *cq) { + grpc_event ev; + do { + ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + } while (ev.type != GRPC_QUEUE_SHUTDOWN); +} + +static void shutdown_server(grpc_end2end_test_fixture *f) { + if (!f->server) return; + grpc_server_shutdown_and_notify(f->server, f->cq, tag(1000)); + GPR_ASSERT(grpc_completion_queue_pluck( + f->cq, tag(1000), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5), NULL) + .type == GRPC_OP_COMPLETE); + grpc_server_destroy(f->server); + f->server = NULL; +} + +static void shutdown_client(grpc_end2end_test_fixture *f) { + if (!f->client) return; + grpc_channel_destroy(f->client); + f->client = NULL; +} + +static void end_test(grpc_end2end_test_fixture *f) { + shutdown_server(f); + shutdown_client(f); + + grpc_completion_queue_shutdown(f->cq); + drain_cq(f->cq); + grpc_completion_queue_destroy(f->cq); +} + +static bool coalesced_message_and_eos; + +static void log_processor(gpr_log_func_args *args) { + const int file_len = (int)strlen(args->file); + const char suffix1[] = "secure_endpoint.c"; + const int suffix1_len = sizeof(suffix1) - 1; + const char suffix2[] = "tcp_posix.c"; + const int suffix2_len = sizeof(suffix2) - 1; + const char prefix[] = "READ"; + const int prefix_len = sizeof(prefix) - 1; + if (((file_len >= suffix1_len && + 0 == strcmp(suffix1, &args->file[file_len - suffix1_len])) || + (file_len >= suffix2_len && + 0 == strcmp(suffix2, &args->file[file_len - suffix2_len]))) && + 0 == strncmp(prefix, args->message, (size_t)prefix_len) && + strstr(args->message, + "00 00 10 00 01 00 00 00 01 00 00 00 00 0b 68 65 6c 6c 6f 20 77 " + "6f 72 6c 64")) { + fprintf(stderr, "%s, %s\n", args->file, args->message); + coalesced_message_and_eos = true; + } +} + +/* Request/response with metadata and payload.*/ +static void test_request_response_with_metadata_and_payload( + grpc_end2end_test_config config) { + grpc_call *c; + grpc_call *s; + grpc_slice request_payload_slice = + grpc_slice_from_copied_string("hello world"); + grpc_byte_buffer *request_payload = + grpc_raw_byte_buffer_create(&request_payload_slice, 1); + gpr_timespec deadline = five_seconds_time(); + grpc_metadata meta_c[2] = { + {"key1", "val1", 4, 0, {{NULL, NULL, NULL, NULL}}}, + {"key2", "val2", 4, 0, {{NULL, NULL, NULL, NULL}}}}; + grpc_metadata meta_s[2] = { + {"key3", "val3", 4, 0, {{NULL, NULL, NULL, NULL}}}, + {"key4", "val4", 4, 0, {{NULL, NULL, NULL, NULL}}}}; + grpc_end2end_test_fixture f = begin_test( + config, "test_request_response_with_metadata_and_payload", NULL, NULL); + cq_verifier *cqv = cq_verifier_create(f.cq); + grpc_op ops[6]; + grpc_op *op; + grpc_metadata_array initial_metadata_recv; + grpc_metadata_array trailing_metadata_recv; + grpc_metadata_array request_metadata_recv; + grpc_byte_buffer *request_payload_recv = NULL; + grpc_call_details call_details; + grpc_status_code status; + grpc_call_error error; + char *details = NULL; + size_t details_capacity = 0; + int was_cancelled = 2; + coalesced_message_and_eos = false; + + c = grpc_channel_create_call( + f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, "/foo", + get_host_override_string("foo.test.google.fr:1234", config), deadline, + NULL); + GPR_ASSERT(c); + + grpc_metadata_array_init(&initial_metadata_recv); + grpc_metadata_array_init(&trailing_metadata_recv); + grpc_metadata_array_init(&request_metadata_recv); + grpc_call_details_init(&call_details); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 2; + op->data.send_initial_metadata.metadata = meta_c; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_MESSAGE; + op->data.send_message = request_payload; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata = &initial_metadata_recv; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; + op->data.recv_status_on_client.status = &status; + op->data.recv_status_on_client.status_details = &details; + op->data.recv_status_on_client.status_details_capacity = &details_capacity; + op->flags = 0; + op->reserved = NULL; + op++; + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + error = + grpc_server_request_call(f.server, &s, &call_details, + &request_metadata_recv, f.cq, f.cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + CQ_EXPECT_COMPLETION(cqv, tag(101), 1); + cq_verify(cqv); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 2; + op->data.send_initial_metadata.metadata = meta_s; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_MESSAGE; + op->data.recv_message = &request_payload_recv; + op->flags = 0; + op->reserved = NULL; + op++; + error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(102), 1); + cq_verify(cqv); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; + op->data.recv_close_on_server.cancelled = &was_cancelled; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; + op->data.send_status_from_server.trailing_metadata_count = 0; + op->data.send_status_from_server.status = GRPC_STATUS_OK; + op->data.send_status_from_server.status_details = "xyz"; + op->flags = 0; + op->reserved = NULL; + op++; + error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(103), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(103), 1); + CQ_EXPECT_COMPLETION(cqv, tag(1), 1); + cq_verify(cqv); + + GPR_ASSERT(status == GRPC_STATUS_OK); + GPR_ASSERT(0 == strcmp(details, "xyz")); + GPR_ASSERT(0 == strcmp(call_details.method, "/foo")); + validate_host_override_string("foo.test.google.fr:1234", call_details.host, + config); + GPR_ASSERT(was_cancelled == 0); + GPR_ASSERT(byte_buffer_eq_string(request_payload_recv, "hello world")); + GPR_ASSERT(contains_metadata(&request_metadata_recv, "key1", "val1")); + GPR_ASSERT(contains_metadata(&request_metadata_recv, "key2", "val2")); + GPR_ASSERT(coalesced_message_and_eos); + + gpr_free(details); + grpc_metadata_array_destroy(&initial_metadata_recv); + grpc_metadata_array_destroy(&trailing_metadata_recv); + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + + grpc_call_destroy(c); + grpc_call_destroy(s); + + cq_verifier_destroy(cqv); + + grpc_byte_buffer_destroy(request_payload); + grpc_byte_buffer_destroy(request_payload_recv); + + end_test(&f); + config.tear_down_data(&f); +} + +void packet_coalescing(grpc_end2end_test_config config) { + /* The test case does not support the fixture socketpair_one_byte_at_a_time */ + if (0 == strcmp(config.name, "chttp2/socketpair_one_byte_at_a_time")) { + return; + } + gpr_set_log_verbosity(GPR_LOG_SEVERITY_DEBUG); + grpc_tracer_set_enabled("all", 1); + gpr_set_log_function(log_processor); + test_request_response_with_metadata_and_payload(config); + gpr_set_log_function(gpr_default_log); + grpc_tracer_set_enabled("all", 0); +} + +void packet_coalescing_pre_init(void) {} diff --git a/third_party/objective_c/Cronet/cronet_c_for_grpc.h b/third_party/objective_c/Cronet/cronet_c_for_grpc.h index 15a511aebd0..3d58a8370ef 100644 --- a/third_party/objective_c/Cronet/cronet_c_for_grpc.h +++ b/third_party/objective_c/Cronet/cronet_c_for_grpc.h @@ -5,6 +5,8 @@ #ifndef COMPONENTS_CRONET_IOS_CRONET_C_FOR_GRPC_H_ #define COMPONENTS_CRONET_IOS_CRONET_C_FOR_GRPC_H_ +#define CRONET_EXPORT __attribute__((visibility("default"))) + #ifdef __cplusplus extern "C" { #endif @@ -15,12 +17,10 @@ extern "C" { /* Opaque object representing Cronet Engine. Created and configured outside * of this API to facilitate sharing with other components */ -typedef struct cronet_engine { void* obj; } cronet_engine; - -void cronet_engine_add_quic_hint(cronet_engine* engine, - const char* host, - int port, - int alternate_port); +typedef struct cronet_engine { + void* obj; + void* annotation; +} cronet_engine; /* Cronet Bidirectional Stream API */ @@ -45,11 +45,12 @@ typedef struct cronet_bidirectional_stream_header_array { /* Set of callbacks used to receive callbacks from bidirectional stream. */ typedef struct cronet_bidirectional_stream_callback { - /* Invoked when request headers are sent. Indicates that stream has initiated - * the request. Consumer may call cronet_bidirectional_stream_write() to start - * writing data. + /* Invoked when the stream is ready for reading and writing. + * Consumer may call cronet_bidirectional_stream_read() to start reading data. + * Consumer may call cronet_bidirectional_stream_write() to start writing + * data. */ - void (*on_request_headers_sent)(cronet_bidirectional_stream* stream); + void (*on_stream_ready)(cronet_bidirectional_stream* stream); /* Invoked when initial response headers are received. * Consumer must call cronet_bidirectional_stream_read() to start reading. @@ -67,20 +68,19 @@ typedef struct cronet_bidirectional_stream_callback { * It may be invoked after on_response_trailers_received()}, if there was * pending read data before trailers were received. * - * If count is 0, it means the remote side has signaled that it will send no - * more data; future calls to cronet_bidirectional_stream_read() will result - * in the on_data_read() callback or on_succeded() callback if + * If |bytes_read| is 0, it means the remote side has signaled that it will + * send no more data; future calls to cronet_bidirectional_stream_read() + * will result in the on_data_read() callback or on_succeded() callback if * cronet_bidirectional_stream_write() was invoked with end_of_stream set to * true. */ void (*on_read_completed)(cronet_bidirectional_stream* stream, char* data, - int count); + int bytes_read); /** * Invoked when all data passed to cronet_bidirectional_stream_write() is - * sent. - * To continue writing, call cronet_bidirectional_stream_write(). + * sent. To continue writing, call cronet_bidirectional_stream_write(). */ void (*on_write_completed)(cronet_bidirectional_stream* stream, const char* data); @@ -117,7 +117,7 @@ typedef struct cronet_bidirectional_stream_callback { void (*on_canceled)(cronet_bidirectional_stream* stream); } cronet_bidirectional_stream_callback; -/* Create a new stream object that uses |engine| and |callback|. All stream +/* Creates a new stream object that uses |engine| and |callback|. All stream * tasks are performed asynchronously on the |engine| network thread. |callback| * methods are invoked synchronously on the |engine| network thread, but must * not run tasks on the current thread to prevent blocking networking operations @@ -129,6 +129,7 @@ typedef struct cronet_bidirectional_stream_callback { * * Both |calback| and |engine| must remain valid until stream is destroyed. */ +CRONET_EXPORT cronet_bidirectional_stream* cronet_bidirectional_stream_create( cronet_engine* engine, void* annotation, @@ -136,15 +137,40 @@ cronet_bidirectional_stream* cronet_bidirectional_stream_create( /* TBD: The following methods return int. Should it be a custom type? */ -/* Destroy stream object. Destroy could be called from any thread, including +/* Destroys stream object. Destroy could be called from any thread, including * network thread, but is posted, so |stream| is valid until calling task is * complete. */ +CRONET_EXPORT int cronet_bidirectional_stream_destroy(cronet_bidirectional_stream* stream); -/* Start the stream by sending request to |url| using |method| and |headers|. If - * |end_of_stream| is true, then no data is expected to be written. +/** + * Disables or enables auto flush. By default, data is flushed after + * every cronet_bidirectional_stream_write(). If the auto flush is disabled, + * the client should explicitly call cronet_bidirectional_stream_flush to flush + * the data. + */ +CRONET_EXPORT void cronet_bidirectional_stream_disable_auto_flush( + cronet_bidirectional_stream* stream, + bool disable_auto_flush); + +/** + * Delays sending request headers until cronet_bidirectional_stream_flush() + * is called. This flag is currently only respected when QUIC is negotiated. + * When true, QUIC will send request header frame along with data frame(s) + * as a single packet when possible. + */ +CRONET_EXPORT +void cronet_bidirectional_stream_delay_request_headers_until_flush( + cronet_bidirectional_stream* stream, + bool delay_headers_until_flush); + +/* Starts the stream by sending request to |url| using |method| and |headers|. + * If |end_of_stream| is true, then no data is expected to be written. The + * |method| is HTTP verb, with PUT having a special meaning to mark idempotent + * request, which could use QUIC 0-RTT. */ +CRONET_EXPORT int cronet_bidirectional_stream_start( cronet_bidirectional_stream* stream, const char* url, @@ -153,46 +179,61 @@ int cronet_bidirectional_stream_start( const cronet_bidirectional_stream_header_array* headers, bool end_of_stream); -/* Read response data into |buffer| of |capacity| length. Must only be called at - * most once in response to each invocation of the - * on_response_headers_received() and on_read_completed() methods of the - * cronet_bidirectional_stream_callback. - * Each call will result in an invocation of one of the callback's - * on_read_completed method if data is read, its on_succeeded() method if - * the stream is closed, or its on_failed() method if there's an error. +/* Reads response data into |buffer| of |capacity| length. Must only be called + * at most once in response to each invocation of the + * on_stream_ready()/on_response_headers_received() and on_read_completed() + * methods of the cronet_bidirectional_stream_callback. + * Each call will result in an invocation of the callback's + * on_read_completed() method if data is read, or its on_failed() method if + * there's an error. The callback's on_succeeded() method is also invoked if + * there is no more data to read and |end_of_stream| was previously sent. */ +CRONET_EXPORT int cronet_bidirectional_stream_read(cronet_bidirectional_stream* stream, char* buffer, int capacity); -/* Read response data into |buffer| of |capacity| length. Must only be called at - * most once in response to each invocation of the - * on_response_headers_received() and on_read_completed() methods of the - * cronet_bidirectional_stream_callback. - * Each call will result in an invocation of one of the callback's - * on_read_completed method if data is read, its on_succeeded() method if - * the stream is closed, or its on_failed() method if there's an error. +/* Writes request data from |buffer| of |buffer_length| length. If auto flush is + * disabled, data will be sent only after cronet_bidirectional_stream_flush() is + * called. + * Each call will result in an invocation the callback's on_write_completed() + * method if data is sent, or its on_failed() method if there's an error. + * The callback's on_succeeded() method is also invoked if |end_of_stream| is + * set and all response data has been read. */ +CRONET_EXPORT int cronet_bidirectional_stream_write(cronet_bidirectional_stream* stream, const char* buffer, - int count, + int buffer_length, bool end_of_stream); +/** + * Flushes pending writes. This method should not be called before invocation of + * on_stream_ready() method of the cronet_bidirectional_stream_callback. + * For each previously called cronet_bidirectional_stream_write() + * a corresponding on_write_completed() callback will be invoked when the buffer + * is sent. + */ +CRONET_EXPORT +void cronet_bidirectional_stream_flush(cronet_bidirectional_stream* stream); + /* Cancels the stream. Can be called at any time after * cronet_bidirectional_stream_start(). The on_canceled() method of * cronet_bidirectional_stream_callback will be invoked when cancelation * is complete and no further callback methods will be invoked. If the * stream has completed or has not started, calling * cronet_bidirectional_stream_cancel() has no effect and on_canceled() will not - * be invoked. At most one callback method may be invoked after + * be invoked. At most one callback method may be invoked after * cronet_bidirectional_stream_cancel() has completed. */ -int cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream); +CRONET_EXPORT +void cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream); /* Returns true if the |stream| was successfully started and is now done * (succeeded, canceled, or failed). * Returns false if the |stream| stream is not yet started or is in progress. */ +CRONET_EXPORT bool cronet_bidirectional_stream_is_done(cronet_bidirectional_stream* stream); #ifdef __cplusplus From 8803ae8c3a643f35d15c9269872d3c1fd2cc8409 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 3 Jan 2017 21:16:05 -0800 Subject: [PATCH 004/167] build changes --- Makefile | 2 + test/core/end2end/end2end_nosec_tests.c | 2 - test/core/end2end/gen_build_yaml.py | 3 +- .../generated/sources_and_headers.json | 2 + tools/run_tests/generated/tests.json | 738 +++++++++++++++++- .../end2end_nosec_tests.vcxproj | 2 + .../end2end_nosec_tests.vcxproj.filters | 3 + .../tests/end2end_tests/end2end_tests.vcxproj | 2 + .../end2end_tests.vcxproj.filters | 3 + 9 files changed, 745 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 8f7328ae285..66e992ef09a 100644 --- a/Makefile +++ b/Makefile @@ -7199,6 +7199,7 @@ LIBEND2END_TESTS_SRC = \ test/core/end2end/tests/network_status_change.c \ test/core/end2end/tests/no_logging.c \ test/core/end2end/tests/no_op.c \ + test/core/end2end/tests/packet_coalescing.c \ test/core/end2end/tests/payload.c \ test/core/end2end/tests/ping.c \ test/core/end2end/tests/ping_pong_streaming.c \ @@ -7285,6 +7286,7 @@ LIBEND2END_NOSEC_TESTS_SRC = \ test/core/end2end/tests/network_status_change.c \ test/core/end2end/tests/no_logging.c \ test/core/end2end/tests/no_op.c \ + test/core/end2end/tests/packet_coalescing.c \ test/core/end2end/tests/payload.c \ test/core/end2end/tests/ping.c \ test/core/end2end/tests/ping_pong_streaming.c \ diff --git a/test/core/end2end/end2end_nosec_tests.c b/test/core/end2end/end2end_nosec_tests.c index 86d6ab9c9a7..0581607e808 100644 --- a/test/core/end2end/end2end_nosec_tests.c +++ b/test/core/end2end/end2end_nosec_tests.c @@ -137,8 +137,6 @@ extern void streaming_error_response(grpc_end2end_test_config config); extern void streaming_error_response_pre_init(void); extern void trailing_metadata(grpc_end2end_test_config config); extern void trailing_metadata_pre_init(void); -extern void packet_coalescing(grpc_end2end_test_config config); -extern void packet_coalescing_pre_init(void); void grpc_end2end_tests_pre_init(void) { GPR_ASSERT(!g_pre_init_called); diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py index 201a92a1fdc..3ae2b2cb9c9 100755 --- a/test/core/end2end/gen_build_yaml.py +++ b/test/core/end2end/gen_build_yaml.py @@ -91,6 +91,7 @@ LOWCPU = 0.1 # maps test names to options END2END_TESTS = { + 'authority_not_supported': default_test_options, 'bad_hostname': default_test_options, 'binary_metadata': default_test_options, 'resource_quota_server': default_test_options._replace(large_writes=True, @@ -125,6 +126,7 @@ END2END_TESTS = { 'network_status_change': default_test_options, 'no_logging': default_test_options._replace(traceable=False), 'no_op': default_test_options, + 'packet_coalescing': default_test_options, 'payload': default_test_options, 'load_reporting_hook': default_test_options, 'ping_pong_streaming': default_test_options, @@ -142,7 +144,6 @@ END2END_TESTS = { 'simple_request': default_test_options, 'streaming_error_response': default_test_options, 'trailing_metadata': default_test_options, - 'authority_not_supported': default_test_options, } diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 6ae269cc20d..a96856c1ab9 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -6347,6 +6347,7 @@ "test/core/end2end/tests/network_status_change.c", "test/core/end2end/tests/no_logging.c", "test/core/end2end/tests/no_op.c", + "test/core/end2end/tests/packet_coalescing.c", "test/core/end2end/tests/payload.c", "test/core/end2end/tests/ping.c", "test/core/end2end/tests/ping_pong_streaming.c", @@ -6416,6 +6417,7 @@ "test/core/end2end/tests/network_status_change.c", "test/core/end2end/tests/no_logging.c", "test/core/end2end/tests/no_op.c", + "test/core/end2end/tests/packet_coalescing.c", "test/core/end2end/tests/payload.c", "test/core/end2end/tests/ping.c", "test/core/end2end/tests/ping_pong_streaming.c", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index b76263b8b98..0ba2f2a6f48 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -5712,6 +5712,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_census_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -6795,6 +6818,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -7847,6 +7893,28 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_fakesec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -8843,6 +8911,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_fd_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -9880,6 +9971,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -10837,6 +10951,25 @@ "linux" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_test", + "platforms": [ + "linux" + ] + }, { "args": [ "payload" @@ -11810,6 +11943,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -12922,6 +13078,30 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_http_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -14021,6 +14201,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_load_reporting_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -15133,6 +15336,30 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_oauth2_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -16165,6 +16392,30 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -17151,7 +17402,7 @@ }, { "args": [ - "payload" + "packet_coalescing" ], "ci_platforms": [ "windows", @@ -17175,7 +17426,7 @@ }, { "args": [ - "ping_pong_streaming" + "payload" ], "ci_platforms": [ "windows", @@ -17199,7 +17450,7 @@ }, { "args": [ - "registered_call" + "ping_pong_streaming" ], "ci_platforms": [ "windows", @@ -17223,14 +17474,14 @@ }, { "args": [ - "request_with_flags" + "registered_call" ], "ci_platforms": [ "windows", "linux", "posix" ], - "cpu_cost": 0.1, + "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [ "uv" @@ -17247,14 +17498,14 @@ }, { "args": [ - "request_with_payload" + "request_with_flags" ], "ci_platforms": [ "windows", "linux", "posix" ], - "cpu_cost": 1.0, + "cpu_cost": 0.1, "exclude_configs": [], "exclude_iomgrs": [ "uv" @@ -17271,7 +17522,7 @@ }, { "args": [ - "resource_quota_server" + "request_with_payload" ], "ci_platforms": [ "windows", @@ -17295,7 +17546,31 @@ }, { "args": [ - "server_finishes_request" + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ + "server_finishes_request" ], "ci_platforms": [ "windows", @@ -18109,6 +18384,30 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair+trace_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -19149,6 +19448,32 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [ + "msan" + ], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_1byte_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -20202,6 +20527,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -21285,6 +21633,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_cert_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -22301,6 +22672,30 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_ssl_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -23303,6 +23698,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_uds_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -24363,6 +24781,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_census_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -25423,6 +25864,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -26412,6 +26876,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_fd_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -27426,6 +27913,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -28364,6 +28874,25 @@ "linux" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_nosec_test", + "platforms": [ + "linux" + ] + }, { "args": [ "payload" @@ -29314,6 +29843,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -30402,6 +30954,30 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_http_proxy_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -31478,6 +32054,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_load_reporting_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -32470,6 +33069,30 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_proxy_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -33430,6 +34053,30 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -34366,6 +35013,30 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair+trace_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -35380,6 +36051,32 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [ + "msan" + ], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_1byte_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" @@ -36385,6 +37082,29 @@ "posix" ] }, + { + "args": [ + "packet_coalescing" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_uds_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "payload" diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj index 4fb8f8f4a15..196d924287f 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj @@ -215,6 +215,8 @@ + + diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters index ff82a4dd43c..73801e36354 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters @@ -97,6 +97,9 @@ test\core\end2end\tests + + test\core\end2end\tests + test\core\end2end\tests diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj index 0b7d7c2e752..7b1701005db 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj @@ -217,6 +217,8 @@ + + diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters index e641930e64a..2b707651768 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters @@ -100,6 +100,9 @@ test\core\end2end\tests + + test\core\end2end\tests + test\core\end2end\tests From dc739bd659384252ae31234a439f1abcd7a512ff Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 5 Jan 2017 21:04:40 -0800 Subject: [PATCH 005/167] Remove packet coalescing testing on chttp2 --- Makefile | 2 - .../CoreCronetEnd2EndTests.m | 9 +- test/core/end2end/end2end_nosec_tests.c | 8 - test/core/end2end/end2end_tests.c | 8 - test/core/end2end/gen_build_yaml.py | 1 - test/core/end2end/tests/packet_coalescing.c | 4 - .../generated/sources_and_headers.json | 2 - tools/run_tests/generated/tests.json | 736 +----------------- .../end2end_nosec_tests.vcxproj | 2 - .../end2end_nosec_tests.vcxproj.filters | 3 - .../tests/end2end_tests/end2end_tests.vcxproj | 2 - .../end2end_tests.vcxproj.filters | 3 - 12 files changed, 16 insertions(+), 764 deletions(-) diff --git a/Makefile b/Makefile index 66e992ef09a..8f7328ae285 100644 --- a/Makefile +++ b/Makefile @@ -7199,7 +7199,6 @@ LIBEND2END_TESTS_SRC = \ test/core/end2end/tests/network_status_change.c \ test/core/end2end/tests/no_logging.c \ test/core/end2end/tests/no_op.c \ - test/core/end2end/tests/packet_coalescing.c \ test/core/end2end/tests/payload.c \ test/core/end2end/tests/ping.c \ test/core/end2end/tests/ping_pong_streaming.c \ @@ -7286,7 +7285,6 @@ LIBEND2END_NOSEC_TESTS_SRC = \ test/core/end2end/tests/network_status_change.c \ test/core/end2end/tests/no_logging.c \ test/core/end2end/tests/no_op.c \ - test/core/end2end/tests/packet_coalescing.c \ test/core/end2end/tests/payload.c \ test/core/end2end/tests/ping.c \ test/core/end2end/tests/ping_pong_streaming.c \ diff --git a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m index fe9a7c2247b..315d9424845 100644 --- a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m +++ b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m @@ -168,6 +168,9 @@ static grpc_end2end_test_config configs[] = { chttp2_tear_down_secure_fullstack}, }; +extern void packet_coalescing(grpc_end2end_test_config config); +extern void packet_coalescing_pre_init(void); + static char *roots_filename; @interface CoreCronetEnd2EndTests : XCTestCase @@ -347,7 +350,11 @@ static char *roots_filename; } - (void)testPacketCoalescing { - [self testIndividualCase:"packet_coalescing"]; + // Directly invoke the test function since the test is for Cronet only and thus not included in + // end2end_tests.c + // TODO (mxyan): Do the same to all test cases so that this file will no longer depend on + // end2end_tests.c + packet_coalescing(configs[0]); } - (void)testPayload { diff --git a/test/core/end2end/end2end_nosec_tests.c b/test/core/end2end/end2end_nosec_tests.c index 0581607e808..663489082f9 100644 --- a/test/core/end2end/end2end_nosec_tests.c +++ b/test/core/end2end/end2end_nosec_tests.c @@ -103,8 +103,6 @@ extern void no_logging(grpc_end2end_test_config config); extern void no_logging_pre_init(void); extern void no_op(grpc_end2end_test_config config); extern void no_op_pre_init(void); -extern void packet_coalescing(grpc_end2end_test_config config); -extern void packet_coalescing_pre_init(void); extern void payload(grpc_end2end_test_config config); extern void payload_pre_init(void); extern void ping(grpc_end2end_test_config config); @@ -171,7 +169,6 @@ void grpc_end2end_tests_pre_init(void) { network_status_change_pre_init(); no_logging_pre_init(); no_op_pre_init(); - packet_coalescing_pre_init(); payload_pre_init(); ping_pre_init(); ping_pong_streaming_pre_init(); @@ -227,7 +224,6 @@ void grpc_end2end_tests(int argc, char **argv, network_status_change(config); no_logging(config); no_op(config); - packet_coalescing(config); payload(config); ping(config); ping_pong_streaming(config); @@ -368,10 +364,6 @@ void grpc_end2end_tests(int argc, char **argv, no_op(config); continue; } - if (0 == strcmp("packet_coalescing", argv[i])) { - packet_coalescing(config); - continue; - } if (0 == strcmp("payload", argv[i])) { payload(config); continue; diff --git a/test/core/end2end/end2end_tests.c b/test/core/end2end/end2end_tests.c index 66908bab606..25c7c62fde5 100644 --- a/test/core/end2end/end2end_tests.c +++ b/test/core/end2end/end2end_tests.c @@ -105,8 +105,6 @@ extern void no_logging(grpc_end2end_test_config config); extern void no_logging_pre_init(void); extern void no_op(grpc_end2end_test_config config); extern void no_op_pre_init(void); -extern void packet_coalescing(grpc_end2end_test_config config); -extern void packet_coalescing_pre_init(void); extern void payload(grpc_end2end_test_config config); extern void payload_pre_init(void); extern void ping(grpc_end2end_test_config config); @@ -174,7 +172,6 @@ void grpc_end2end_tests_pre_init(void) { network_status_change_pre_init(); no_logging_pre_init(); no_op_pre_init(); - packet_coalescing_pre_init(); payload_pre_init(); ping_pre_init(); ping_pong_streaming_pre_init(); @@ -231,7 +228,6 @@ void grpc_end2end_tests(int argc, char **argv, network_status_change(config); no_logging(config); no_op(config); - packet_coalescing(config); payload(config); ping(config); ping_pong_streaming(config); @@ -376,10 +372,6 @@ void grpc_end2end_tests(int argc, char **argv, no_op(config); continue; } - if (0 == strcmp("packet_coalescing", argv[i])) { - packet_coalescing(config); - continue; - } if (0 == strcmp("payload", argv[i])) { payload(config); continue; diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py index 3ae2b2cb9c9..78779908a94 100755 --- a/test/core/end2end/gen_build_yaml.py +++ b/test/core/end2end/gen_build_yaml.py @@ -126,7 +126,6 @@ END2END_TESTS = { 'network_status_change': default_test_options, 'no_logging': default_test_options._replace(traceable=False), 'no_op': default_test_options, - 'packet_coalescing': default_test_options, 'payload': default_test_options, 'load_reporting_hook': default_test_options, 'ping_pong_streaming': default_test_options, diff --git a/test/core/end2end/tests/packet_coalescing.c b/test/core/end2end/tests/packet_coalescing.c index d560c57da80..336f765a13f 100644 --- a/test/core/end2end/tests/packet_coalescing.c +++ b/test/core/end2end/tests/packet_coalescing.c @@ -273,10 +273,6 @@ static void test_request_response_with_metadata_and_payload( } void packet_coalescing(grpc_end2end_test_config config) { - /* The test case does not support the fixture socketpair_one_byte_at_a_time */ - if (0 == strcmp(config.name, "chttp2/socketpair_one_byte_at_a_time")) { - return; - } gpr_set_log_verbosity(GPR_LOG_SEVERITY_DEBUG); grpc_tracer_set_enabled("all", 1); gpr_set_log_function(log_processor); diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index a96856c1ab9..6ae269cc20d 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -6347,7 +6347,6 @@ "test/core/end2end/tests/network_status_change.c", "test/core/end2end/tests/no_logging.c", "test/core/end2end/tests/no_op.c", - "test/core/end2end/tests/packet_coalescing.c", "test/core/end2end/tests/payload.c", "test/core/end2end/tests/ping.c", "test/core/end2end/tests/ping_pong_streaming.c", @@ -6417,7 +6416,6 @@ "test/core/end2end/tests/network_status_change.c", "test/core/end2end/tests/no_logging.c", "test/core/end2end/tests/no_op.c", - "test/core/end2end/tests/packet_coalescing.c", "test/core/end2end/tests/payload.c", "test/core/end2end/tests/ping.c", "test/core/end2end/tests/ping_pong_streaming.c", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 0ba2f2a6f48..b76263b8b98 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -5712,29 +5712,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_census_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -6818,29 +6795,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_compress_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -7893,28 +7847,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_fakesec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -8911,29 +8843,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_fd_test", - "platforms": [ - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -9971,29 +9880,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_full_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -10951,25 +10837,6 @@ "linux" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "linux" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_full+pipe_test", - "platforms": [ - "linux" - ] - }, { "args": [ "payload" @@ -11943,29 +11810,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_full+trace_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -13078,30 +12922,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_http_proxy_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -14201,29 +14021,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_load_reporting_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -15336,30 +15133,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_oauth2_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -16392,30 +16165,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_proxy_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -17402,7 +17151,7 @@ }, { "args": [ - "packet_coalescing" + "payload" ], "ci_platforms": [ "windows", @@ -17426,7 +17175,7 @@ }, { "args": [ - "payload" + "ping_pong_streaming" ], "ci_platforms": [ "windows", @@ -17450,7 +17199,7 @@ }, { "args": [ - "ping_pong_streaming" + "registered_call" ], "ci_platforms": [ "windows", @@ -17474,14 +17223,14 @@ }, { "args": [ - "registered_call" + "request_with_flags" ], "ci_platforms": [ "windows", "linux", "posix" ], - "cpu_cost": 1.0, + "cpu_cost": 0.1, "exclude_configs": [], "exclude_iomgrs": [ "uv" @@ -17498,14 +17247,14 @@ }, { "args": [ - "request_with_flags" + "request_with_payload" ], "ci_platforms": [ "windows", "linux", "posix" ], - "cpu_cost": 0.1, + "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [ "uv" @@ -17522,31 +17271,7 @@ }, { "args": [ - "request_with_payload" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_sockpair_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "resource_quota_server" + "resource_quota_server" ], "ci_platforms": [ "windows", @@ -18384,30 +18109,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_sockpair+trace_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -19448,32 +19149,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [ - "msan" - ], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_sockpair_1byte_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -20527,29 +20202,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_ssl_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -21633,29 +21285,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_ssl_cert_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -22672,30 +22301,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_ssl_proxy_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -23698,29 +23303,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_uds_test", - "platforms": [ - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -24781,29 +24363,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_census_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -25864,29 +25423,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_compress_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -26876,29 +26412,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_fd_nosec_test", - "platforms": [ - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -27913,29 +27426,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_full_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -28874,25 +28364,6 @@ "linux" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "linux" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_full+pipe_nosec_test", - "platforms": [ - "linux" - ] - }, { "args": [ "payload" @@ -29843,29 +29314,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_full+trace_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -30954,30 +30402,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_http_proxy_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -32054,29 +31478,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_load_reporting_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -33069,30 +32470,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_proxy_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -34053,30 +33430,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_sockpair_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -35013,30 +34366,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_sockpair+trace_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -36051,32 +35380,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "windows", - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [ - "msan" - ], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_sockpair_1byte_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" @@ -37082,29 +36385,6 @@ "posix" ] }, - { - "args": [ - "packet_coalescing" - ], - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_uds_nosec_test", - "platforms": [ - "linux", - "mac", - "posix" - ] - }, { "args": [ "payload" diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj index 196d924287f..4fb8f8f4a15 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj @@ -215,8 +215,6 @@ - - diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters index 73801e36354..ff82a4dd43c 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters @@ -97,9 +97,6 @@ test\core\end2end\tests - - test\core\end2end\tests - test\core\end2end\tests diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj index 7b1701005db..0b7d7c2e752 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj @@ -217,8 +217,6 @@ - - diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters index 2b707651768..e641930e64a 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters @@ -100,9 +100,6 @@ test\core\end2end\tests - - test\core\end2end\tests - test\core\end2end\tests From 0542d11ea8ab3ffde3458fbf7daf7a2af93b19fa Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 9 Jan 2017 15:07:11 -0800 Subject: [PATCH 006/167] Re-ran generate_projects.sh --- tools/doxygen/Doxyfile.c++ | 42 +++++++++++----------- tools/doxygen/Doxyfile.c++.internal | 40 ++++++++++----------- tools/doxygen/Doxyfile.core | 38 ++++++++++---------- tools/doxygen/Doxyfile.core.internal | 54 ++++++++++++++-------------- 4 files changed, 87 insertions(+), 87 deletions(-) diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 2c16f6f24a0..bb620f6b4a0 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -848,34 +848,34 @@ include/grpc/impl/codegen/sync.h \ include/grpc/impl/codegen/sync_generic.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ +doc/command_line_tool.md \ +doc/statuscodes.md \ +doc/interop-test-descriptions.md \ doc/fail_fast.md \ +doc/server-reflection.md \ +doc/c-style-guide.md \ +doc/load-balancing.md \ +doc/compression_cookbook.md \ +doc/connectivity-semantics-and-api.md \ +doc/PROTOCOL-HTTP2.md \ +doc/health-checking.md \ +doc/binary-logging.md \ +doc/stress_test_framework.md \ +doc/negative-http2-interop-test-descriptions.md \ doc/compression.md \ +doc/server_reflection_tutorial.md \ +doc/cpp-style-guide.md \ doc/environment_variables.md \ -doc/stress_test_framework.md \ doc/PROTOCOL-WEB.md \ -doc/cpp-style-guide.md \ -doc/http-grpc-status-mapping.md \ -doc/wait-for-ready.md \ -doc/command_line_tool.md \ -doc/c-style-guide.md \ -doc/server_reflection_tutorial.md \ -doc/health-checking.md \ -doc/connection-backoff-interop-test-description.md \ -doc/epoll-polling-engine.md \ doc/naming.md \ -doc/binary-logging.md \ -doc/connectivity-semantics-and-api.md \ +doc/epoll-polling-engine.md \ doc/connection-backoff.md \ -doc/compression_cookbook.md \ -doc/PROTOCOL-HTTP2.md \ -doc/load-balancing.md \ -doc/negative-http2-interop-test-descriptions.md \ -doc/server-reflection.md \ -doc/interop-test-descriptions.md \ -doc/statuscodes.md \ +doc/connection-backoff-interop-test-description.md \ doc/g_stands_for.md \ -doc/cpp/perf_notes.md \ -doc/cpp/pending_api_cleanups.md +doc/http-grpc-status-mapping.md \ +doc/wait-for-ready.md \ +doc/cpp/pending_api_cleanups.md \ +doc/cpp/perf_notes.md # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 89915d7b08e..a8dd34820da 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -894,34 +894,34 @@ src/cpp/util/status.cc \ src/cpp/util/string_ref.cc \ src/cpp/util/time_cc.cc \ src/cpp/codegen/codegen_init.cc \ +doc/command_line_tool.md \ +doc/statuscodes.md \ +doc/interop-test-descriptions.md \ doc/fail_fast.md \ +doc/server-reflection.md \ +doc/c-style-guide.md \ +doc/load-balancing.md \ +doc/compression_cookbook.md \ +doc/connectivity-semantics-and-api.md \ +doc/PROTOCOL-HTTP2.md \ +doc/health-checking.md \ +doc/binary-logging.md \ +doc/stress_test_framework.md \ +doc/negative-http2-interop-test-descriptions.md \ doc/compression.md \ +doc/server_reflection_tutorial.md \ +doc/cpp-style-guide.md \ doc/environment_variables.md \ -doc/stress_test_framework.md \ doc/PROTOCOL-WEB.md \ -doc/cpp-style-guide.md \ -doc/http-grpc-status-mapping.md \ -doc/wait-for-ready.md \ -doc/command_line_tool.md \ -doc/c-style-guide.md \ -doc/server_reflection_tutorial.md \ -doc/health-checking.md \ -doc/connection-backoff-interop-test-description.md \ -doc/epoll-polling-engine.md \ doc/naming.md \ -doc/binary-logging.md \ -doc/connectivity-semantics-and-api.md \ +doc/epoll-polling-engine.md \ doc/connection-backoff.md \ -doc/compression_cookbook.md \ -doc/PROTOCOL-HTTP2.md \ -doc/load-balancing.md \ -doc/negative-http2-interop-test-descriptions.md \ -doc/server-reflection.md \ -doc/interop-test-descriptions.md \ -doc/statuscodes.md \ +doc/connection-backoff-interop-test-description.md \ doc/g_stands_for.md \ -doc/cpp/perf_notes.md \ +doc/http-grpc-status-mapping.md \ +doc/wait-for-ready.md \ doc/cpp/pending_api_cleanups.md \ +doc/cpp/perf_notes.md \ src/cpp/README.md # This tag can be used to specify the character encoding of the source files diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index e7fc1dbc573..562f3fdc2ee 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -826,32 +826,32 @@ include/grpc/impl/codegen/sync.h \ include/grpc/impl/codegen/sync_generic.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ +doc/command_line_tool.md \ +doc/statuscodes.md \ +doc/interop-test-descriptions.md \ doc/fail_fast.md \ +doc/server-reflection.md \ +doc/c-style-guide.md \ +doc/load-balancing.md \ +doc/compression_cookbook.md \ +doc/connectivity-semantics-and-api.md \ +doc/PROTOCOL-HTTP2.md \ +doc/health-checking.md \ +doc/binary-logging.md \ +doc/stress_test_framework.md \ +doc/negative-http2-interop-test-descriptions.md \ doc/compression.md \ +doc/server_reflection_tutorial.md \ +doc/cpp-style-guide.md \ doc/environment_variables.md \ -doc/stress_test_framework.md \ doc/PROTOCOL-WEB.md \ -doc/cpp-style-guide.md \ -doc/http-grpc-status-mapping.md \ -doc/wait-for-ready.md \ -doc/command_line_tool.md \ -doc/c-style-guide.md \ -doc/server_reflection_tutorial.md \ -doc/health-checking.md \ -doc/connection-backoff-interop-test-description.md \ -doc/epoll-polling-engine.md \ doc/naming.md \ -doc/binary-logging.md \ -doc/connectivity-semantics-and-api.md \ +doc/epoll-polling-engine.md \ doc/connection-backoff.md \ -doc/compression_cookbook.md \ -doc/PROTOCOL-HTTP2.md \ -doc/load-balancing.md \ -doc/negative-http2-interop-test-descriptions.md \ -doc/server-reflection.md \ -doc/interop-test-descriptions.md \ -doc/statuscodes.md \ +doc/connection-backoff-interop-test-description.md \ doc/g_stands_for.md \ +doc/http-grpc-status-mapping.md \ +doc/wait-for-ready.md \ doc/core/pending_api_cleanups.md # This tag can be used to specify the character encoding of the source files diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 8b60152009b..7b647d2ed8d 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1281,52 +1281,52 @@ src/core/lib/support/tmpfile_msys.c \ src/core/lib/support/tmpfile_posix.c \ src/core/lib/support/tmpfile_windows.c \ src/core/lib/support/wrap_memcpy.c \ +doc/command_line_tool.md \ +doc/statuscodes.md \ +doc/interop-test-descriptions.md \ doc/fail_fast.md \ +doc/server-reflection.md \ +doc/c-style-guide.md \ +doc/load-balancing.md \ +doc/compression_cookbook.md \ +doc/connectivity-semantics-and-api.md \ +doc/PROTOCOL-HTTP2.md \ +doc/health-checking.md \ +doc/binary-logging.md \ +doc/stress_test_framework.md \ +doc/negative-http2-interop-test-descriptions.md \ doc/compression.md \ +doc/server_reflection_tutorial.md \ +doc/cpp-style-guide.md \ doc/environment_variables.md \ -doc/stress_test_framework.md \ doc/PROTOCOL-WEB.md \ -doc/cpp-style-guide.md \ -doc/http-grpc-status-mapping.md \ -doc/wait-for-ready.md \ -doc/command_line_tool.md \ -doc/c-style-guide.md \ -doc/server_reflection_tutorial.md \ -doc/health-checking.md \ -doc/connection-backoff-interop-test-description.md \ -doc/epoll-polling-engine.md \ doc/naming.md \ -doc/binary-logging.md \ -doc/connectivity-semantics-and-api.md \ +doc/epoll-polling-engine.md \ doc/connection-backoff.md \ -doc/compression_cookbook.md \ -doc/PROTOCOL-HTTP2.md \ -doc/load-balancing.md \ -doc/negative-http2-interop-test-descriptions.md \ -doc/server-reflection.md \ -doc/interop-test-descriptions.md \ -doc/statuscodes.md \ +doc/connection-backoff-interop-test-description.md \ doc/g_stands_for.md \ +doc/http-grpc-status-mapping.md \ +doc/wait-for-ready.md \ doc/core/pending_api_cleanups.md \ src/core/README.md \ src/core/ext/README.md \ src/core/ext/transport/README.md \ src/core/ext/transport/chttp2/README.md \ -src/core/ext/transport/chttp2/client/secure/README.md \ -src/core/ext/transport/chttp2/client/insecure/README.md \ src/core/ext/transport/chttp2/transport/README.md \ -src/core/ext/transport/chttp2/server/secure/README.md \ src/core/ext/transport/chttp2/server/insecure/README.md \ +src/core/ext/transport/chttp2/server/secure/README.md \ +src/core/ext/transport/chttp2/client/insecure/README.md \ +src/core/ext/transport/chttp2/client/secure/README.md \ +src/core/ext/census/README.md \ +src/core/ext/census/gen/README.md \ src/core/ext/client_channel/README.md \ src/core/ext/resolver/README.md \ -src/core/ext/resolver/sockaddr/README.md \ src/core/ext/resolver/dns/native/README.md \ -src/core/ext/census/README.md \ -src/core/ext/census/gen/README.md \ +src/core/ext/resolver/sockaddr/README.md \ src/core/lib/README.md \ -src/core/lib/tsi/README.md \ -src/core/lib/channel/README.md \ src/core/lib/transport/README.md \ +src/core/lib/channel/README.md \ +src/core/lib/tsi/README.md \ src/core/lib/iomgr/README.md \ src/core/lib/surface/README.md From 2b939832d50fe6823df2658c53fd3fb9003fdfd0 Mon Sep 17 00:00:00 2001 From: Vizerai Date: Thu, 19 Jan 2017 13:12:11 -0800 Subject: [PATCH 007/167] update --- BUILD | 5 ++ build.yaml | 5 ++ src/core/ext/census/trace_label.h | 59 +++++++++++++ src/core/ext/census/trace_propagation.h | 62 +++++++++++++ src/core/ext/census/trace_status.h | 45 ++++++++++ src/core/ext/census/trace_string.h | 47 ++++++++++ src/core/ext/census/tracing.c | 43 ++++++--- src/core/ext/census/tracing.h | 113 ++++++++++++++++++++++++ 8 files changed, 367 insertions(+), 12 deletions(-) create mode 100644 src/core/ext/census/trace_label.h create mode 100644 src/core/ext/census/trace_propagation.h create mode 100644 src/core/ext/census/trace_status.h create mode 100644 src/core/ext/census/trace_string.h create mode 100644 src/core/ext/census/tracing.h diff --git a/BUILD b/BUILD index 54192514cc7..10d93d1f9cd 100644 --- a/BUILD +++ b/BUILD @@ -284,6 +284,11 @@ grpc_cc_library( "src/core/ext/census/resource.h", "src/core/ext/census/rpc_metric_id.h", "src/core/ext/census/trace_context.h", + "src/core/ext/census/trace_label.h", + "src/core/ext/census/trace_propagation.h", + "src/core/ext/census/trace_status.h", + "src/core/ext/census/trace_string.h", + "src/core/ext/census/tracing.h", ], external_deps = [ "nanopb", diff --git a/build.yaml b/build.yaml index eaf64e860f1..0b490268c69 100644 --- a/build.yaml +++ b/build.yaml @@ -31,6 +31,11 @@ filegroups: - src/core/ext/census/resource.h - src/core/ext/census/rpc_metric_id.h - src/core/ext/census/trace_context.h + - src/core/ext/census/trace_label.h + - src/core/ext/census/trace_propagation.h + - src/core/ext/census/trace_status.h + - src/core/ext/census/trace_string.h + - src/core/ext/census/tracing.h src: - src/core/ext/census/base_resources.c - src/core/ext/census/context.c diff --git a/src/core/ext/census/trace_label.h b/src/core/ext/census/trace_label.h new file mode 100644 index 00000000000..b58543b7d8c --- /dev/null +++ b/src/core/ext/census/trace_label.h @@ -0,0 +1,59 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_CENSUS_TRACE_LABEL_H +#define GRPC_CORE_EXT_CENSUS_TRACE_LABEL_H + +#include "src/core/ext/census/trace_string.h" + +/* Trace label (key/value pair) stores a label name and the label value. */ +typedef struct trace_label { + trace_string key; + enum label_type { + /* A string value */ + LABEL_STRING = 1, + /* An integer value. */ + LABEL_INT = 2, + /* A boolean value. */ + LABEL_BOOL = 3, + } value_type; + + union value { + trace_string label_str; + int64_t label_int; + bool label_bool; + } value; + size_t val_len; +} trace_label; + +#endif diff --git a/src/core/ext/census/trace_propagation.h b/src/core/ext/census/trace_propagation.h new file mode 100644 index 00000000000..093492c3e8e --- /dev/null +++ b/src/core/ext/census/trace_propagation.h @@ -0,0 +1,62 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_CENSUS_TRACE_PROPAGATION_H +#define GRPC_CORE_EXT_CENSUS_TRACE_PROPAGATION_H + +#include "src/core/ext/census/tracing.h" + +/* Encoding and decoding functions for receiving and sending propagating data + over the wire. Only RPC libraries should be calling these + functions. These functions return the number of bytes encoded/decoded + (0 if a failure has occurred). buf_size indicates the size of the + input/output buffer. trace_span_context is a struct that includes the + trace ID, span ID, and a set of option flags (is_sampled, etc.). */ + +/* Converts a span context to a binary byte buffer. */ +size_t trace_span_context_to_binary(const trace_span_context *ctxt, char *buf, + size_t buf_size); + +/* Reads a binary byte buffer and populates a span context structure. */ +size_t binary_to_trace_span_context(const char *buf, size_t buf_size, + trace_span_context *ctxt); + +/* Converts a span context to a http format buffer. */ +size_t trace_span_context_to_http_format(const trace_span_context *ctxt, + char *buf, size_t buf_size); + +/* Reads a http format buffer and populates a span context structure. */ +size_t http_format_to_trace_span_context(const char *buf, size_t buf_size, + trace_span_context *ctxt); + +#endif diff --git a/src/core/ext/census/trace_status.h b/src/core/ext/census/trace_status.h new file mode 100644 index 00000000000..a4bd4116c30 --- /dev/null +++ b/src/core/ext/census/trace_status.h @@ -0,0 +1,45 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_CENSUS_TRACE_STATUS_H +#define GRPC_CORE_EXT_CENSUS_TRACE_STATUS_H + +#include "src/core/ext/census/trace_string.h" + +/* Stores a status code and status message for a trace. */ +typedef struct trace_status { + int errorCode; + trace_string errorMessage; +} trace_status; + +#endif diff --git a/src/core/ext/census/trace_string.h b/src/core/ext/census/trace_string.h new file mode 100644 index 00000000000..0f71056c3bc --- /dev/null +++ b/src/core/ext/census/trace_string.h @@ -0,0 +1,47 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_CENSUS_TRACE_STRING_H +#define GRPC_CORE_EXT_CENSUS_TRACE_STRING_H + +#include + +/* String struct for tracing messages. Since this is a C API, we do not have + access to a string class. This is intended for use by higher level + languages which wrap around the C API, as most of them have a string class. + This will also be more efficient when copying, as we have an explicitly + specified length. Also, grpc_slice has reference counting which allows for + interning. */ +typedef struct trace_string { grpc_slice string_slice; } trace_string; + +#endif diff --git a/src/core/ext/census/tracing.c b/src/core/ext/census/tracing.c index 9371fffc8d7..8d22f43eed5 100644 --- a/src/core/ext/census/tracing.c +++ b/src/core/ext/census/tracing.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,21 +31,40 @@ * */ -//#include "src/core/ext/census/tracing.h" - +#include "src/core/ext/census/tracing.h" #include -#include +#include +#include +#include +#include "src/core/ext/census/mlog.h" -/* TODO(aveitch): These are all placeholder implementations. */ +void trace_start_span(const trace_span_context *span_ctxt, + const trace_string name, const start_span_options *opts, + trace_span_context *new_span_ctxt, + bool has_remote_parent) { + // Noop implementation. +} -int census_trace_mask(const census_context *context) { - abort(); - return CENSUS_TRACE_MASK_NONE; +void trace_add_span_annotation(const trace_string description, + const trace_label *labels, const size_t n_labels, + trace_span_context *span_ctxt) { + // Noop implementation. } -void census_set_trace_mask(int trace_mask) { abort(); } +void trace_add_span_network_event_annotation(const trace_string description, + const trace_label *labels, + const size_t n_labels, + const gpr_timespec timestamp, + bool sent, uint64_t id, + trace_span_context *span_ctxt) { + // Noop implementation. +} + +void trace_add_span_labels(const trace_label *labels, const size_t n_labels, + trace_span_context *span_ctxt) { + // Noop implementation. +} -void census_trace_print(census_context *context, uint32_t type, - const char *buffer, size_t n) { - abort(); +void trace_end_span(const trace_status *status, trace_span_context *span_ctxt) { + // Noop implementation. } diff --git a/src/core/ext/census/tracing.h b/src/core/ext/census/tracing.h new file mode 100644 index 00000000000..5be39e377ac --- /dev/null +++ b/src/core/ext/census/tracing.h @@ -0,0 +1,113 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_CENSUS_TRACING_H +#define GRPC_CORE_EXT_CENSUS_TRACING_H + +#include +#include +#include "src/core/ext/census/trace_context.h" +#include "src/core/ext/census/trace_label.h" +#include "src/core/ext/census/trace_status.h" + +/* The tracing level. */ +public +enum TraceLevel { + /* Annotations on this context will be silently discarded. */ + NO_TRACING = 0, + /* Annotations will not be saved to a persistent store. They will be + available via local APIs only. It is not propagated to the child. */ + TRANSIENT_TRACING = 1, + /* Annotations are recorded for the entire distributed trace and they are + saved to a persistent store. It is propagated to the child. */ + PERSISTENT_TRACING = 2, +} + +typedef struct trace_span_context { + /* Trace span context stores Span ID, Trace ID, and option flags. */ + uint64_t trace_id_hi; + uint64_t trace_id_lo; + uint64_t span_id; + uint32_t span_options; +} trace_span_context; + +typedef struct start_span_options { + /* If set, this will override the Span.local_start_time for the Span. */ + gpr_timespec local_start_timestamp; + + /* If set, the Spans are linked to the created Span. */ + trace_span_context *linked_spans; + /* The number of linked spans. */ + size_t n_linked_spans; +} start_span_options; + +/* Create a new child Span (or root if parent is NULL), with parent being the + designated Span. The child span will have the provided name and starting + span options (optional). The bool has_remote_parent marks whether the + context refers to a remote parent span or not. */ +void trace_start_span(const trace_span_context *span_ctxt, + const trace_string name, const start_span_options *opts, + trace_span_context *new_span_ctxt, + bool has_remote_parent); + +/* Add a new Annotation to the Span. The description corresponds to + Span->annotations[].description. */ +void trace_add_span_annotation(const trace_string description, + const trace_label *labels, const size_t n_labels, + trace_span_context *span_ctxt); + +/* Add a new NetworkEvent annotation to a Span. This function is only intended + to be used by RPC systems (either client or server), not by higher level + applications. The timestamp type will be system-defined, the sent argument + designates whether this is a network send event (client request, server + reply)or receive (server request, client reply). The id argument corresponds + to Span.Annotation.NetworkEvent.id from the data model, and serves to uniquely + identify each network message. */ +void trace_add_span_network_event(const trace_string description, + const trace_label *labels, + const size_t n_labels, + const gpr_timespec timestamp, bool sent, + uint64_t id, trace_span_context *span_ctxt); + +/* Add a set of labels to the Span. These will correspond to the field +Span.labels in the data model. */ +void trace_add_span_labels(const trace_label *labels, const size_t n_labels, + trace_span_context *span_ctxt); + +/* Mark the end of Span Execution with the given status. Only the timing of the +first EndSpan call for a given Span will be recorded, and implementations are +free to ignore all further calls using the Span. EndSpanOptions can +optionally be NULL. */ +void trace_end_span(const trace_status *status, trace_span_context *span_ctxt); + +#endif From 548d59f305b43d22ca1ef03c5fa82576f32d469a Mon Sep 17 00:00:00 2001 From: Vizerai Date: Thu, 19 Jan 2017 16:41:37 -0800 Subject: [PATCH 008/167] update --- src/core/ext/census/tracing.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/ext/census/tracing.h b/src/core/ext/census/tracing.h index 5be39e377ac..0632b841bd1 100644 --- a/src/core/ext/census/tracing.h +++ b/src/core/ext/census/tracing.h @@ -41,7 +41,6 @@ #include "src/core/ext/census/trace_status.h" /* The tracing level. */ -public enum TraceLevel { /* Annotations on this context will be silently discarded. */ NO_TRACING = 0, @@ -51,7 +50,7 @@ enum TraceLevel { /* Annotations are recorded for the entire distributed trace and they are saved to a persistent store. It is propagated to the child. */ PERSISTENT_TRACING = 2, -} +}; typedef struct trace_span_context { /* Trace span context stores Span ID, Trace ID, and option flags. */ From 645e30a9311af1721bd8b02b533953c2c38d8263 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 20 Jan 2017 10:59:15 -0800 Subject: [PATCH 009/167] Fix compilation errors after latest merge with master --- test/core/iomgr/pollset_set_test.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index af971c0784e..335e3662e6d 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -95,7 +95,8 @@ static void cleanup_test_pollsets(grpc_exec_ctx *exec_ctx, int i; for (i = 0; i < num_pollsets; i++) { - grpc_closure_init(&destroyed, destroy_pollset, pollsets[i].ps); + grpc_closure_init(&destroyed, destroy_pollset, pollsets[i].ps, + grpc_schedule_on_exec_ctx); grpc_pollset_shutdown(exec_ctx, pollsets[i].ps, &destroyed); grpc_exec_ctx_flush(exec_ctx); @@ -123,7 +124,8 @@ void on_readable(grpc_exec_ctx *exec_ctx, void *tfd, grpc_error *error) { static void reset_test_fd(grpc_exec_ctx *exec_ctx, test_fd *tfd) { tfd->is_on_readable_called = false; - grpc_closure_init(&tfd->on_readable, on_readable, tfd); + grpc_closure_init(&tfd->on_readable, on_readable, tfd, + grpc_schedule_on_exec_ctx); grpc_fd_notify_on_read(exec_ctx, tfd->fd, &tfd->on_readable); } @@ -449,6 +451,7 @@ void pollset_set_test_empty_pollset() { int main(int argc, char **argv) { const char *poll_strategy = NULL; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_iomgr_init(); @@ -464,7 +467,8 @@ int main(int argc, char **argv) { poll_strategy); } - grpc_iomgr_shutdown(); + grpc_iomgr_shutdown(&exec_ctx); + grpc_exec_ctx_finish(&exec_ctx); return 0; } #else /* defined(GRPC_LINUX_EPOLL) */ From 0e5b67f4a2405654a99cac5d5b54816f9339c4fa Mon Sep 17 00:00:00 2001 From: Vizerai Date: Fri, 20 Jan 2017 14:02:18 -0800 Subject: [PATCH 010/167] update --- gRPC-Core.podspec | 12 +++++++++++- grpc.gemspec | 5 +++++ package.xml | 5 +++++ third_party/gflags | 2 +- tools/doxygen/Doxyfile.core.internal | 5 +++++ .../run_tests/generated/sources_and_headers.json | 14 ++++++++++++-- vsprojects/vcxproj/grpc/grpc.vcxproj | 5 +++++ vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 15 +++++++++++++++ .../vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 5 +++++ .../grpc_unsecure/grpc_unsecure.vcxproj.filters | 15 +++++++++++++++ 10 files changed, 79 insertions(+), 4 deletions(-) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 1eb178931dc..1dd2ad46b22 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -432,6 +432,11 @@ Pod::Spec.new do |s| 'src/core/ext/census/resource.h', 'src/core/ext/census/rpc_metric_id.h', 'src/core/ext/census/trace_context.h', + 'src/core/ext/census/trace_label.h', + 'src/core/ext/census/trace_propagation.h', + 'src/core/ext/census/trace_status.h', + 'src/core/ext/census/trace_string.h', + 'src/core/ext/census/tracing.h', 'src/core/lib/surface/init.c', 'src/core/lib/channel/channel_args.c', 'src/core/lib/channel/channel_stack.c', @@ -843,7 +848,12 @@ Pod::Spec.new do |s| 'src/core/ext/census/mlog.h', 'src/core/ext/census/resource.h', 'src/core/ext/census/rpc_metric_id.h', - 'src/core/ext/census/trace_context.h' + 'src/core/ext/census/trace_context.h', + 'src/core/ext/census/trace_label.h', + 'src/core/ext/census/trace_propagation.h', + 'src/core/ext/census/trace_status.h', + 'src/core/ext/census/trace_string.h', + 'src/core/ext/census/tracing.h' end s.subspec 'Cronet-Interface' do |ss| diff --git a/grpc.gemspec b/grpc.gemspec index cfc0c35aa3e..ecc96bc2fea 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -349,6 +349,11 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/census/resource.h ) s.files += %w( src/core/ext/census/rpc_metric_id.h ) s.files += %w( src/core/ext/census/trace_context.h ) + s.files += %w( src/core/ext/census/trace_label.h ) + s.files += %w( src/core/ext/census/trace_propagation.h ) + s.files += %w( src/core/ext/census/trace_status.h ) + s.files += %w( src/core/ext/census/trace_string.h ) + s.files += %w( src/core/ext/census/tracing.h ) s.files += %w( src/core/lib/surface/init.c ) s.files += %w( src/core/lib/channel/channel_args.c ) s.files += %w( src/core/lib/channel/channel_stack.c ) diff --git a/package.xml b/package.xml index aa1894f4086..cac087ad6c6 100644 --- a/package.xml +++ b/package.xml @@ -358,6 +358,11 @@ + + + + + diff --git a/third_party/gflags b/third_party/gflags index f8a0efe03aa..05b155ff591 160000 --- a/third_party/gflags +++ b/third_party/gflags @@ -1 +1 @@ -Subproject commit f8a0efe03aa69b3336d8e228b37d4ccb17324b88 +Subproject commit 05b155ff59114735ec8cd089f669c4c3d8f59029 diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index ffe83a2ddb3..019a0784b5b 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -884,7 +884,12 @@ src/core/ext/census/resource.h \ src/core/ext/census/rpc_metric_id.h \ src/core/ext/census/trace_context.c \ src/core/ext/census/trace_context.h \ +src/core/ext/census/trace_label.h \ +src/core/ext/census/trace_propagation.h \ +src/core/ext/census/trace_status.h \ +src/core/ext/census/trace_string.h \ src/core/ext/census/tracing.c \ +src/core/ext/census/tracing.h \ src/core/ext/client_channel/README.md \ src/core/ext/client_channel/channel_connectivity.c \ src/core/ext/client_channel/client_channel.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index c57b17611be..51946bf02f7 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -6551,7 +6551,12 @@ "src/core/ext/census/mlog.h", "src/core/ext/census/resource.h", "src/core/ext/census/rpc_metric_id.h", - "src/core/ext/census/trace_context.h" + "src/core/ext/census/trace_context.h", + "src/core/ext/census/trace_label.h", + "src/core/ext/census/trace_propagation.h", + "src/core/ext/census/trace_status.h", + "src/core/ext/census/trace_string.h", + "src/core/ext/census/tracing.h" ], "is_filegroup": true, "language": "c", @@ -6582,7 +6587,12 @@ "src/core/ext/census/rpc_metric_id.h", "src/core/ext/census/trace_context.c", "src/core/ext/census/trace_context.h", - "src/core/ext/census/tracing.c" + "src/core/ext/census/trace_label.h", + "src/core/ext/census/trace_propagation.h", + "src/core/ext/census/trace_status.h", + "src/core/ext/census/trace_string.h", + "src/core/ext/census/tracing.c", + "src/core/ext/census/tracing.h" ], "third_party": false, "type": "filegroup" diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index c159db63f55..15198621ca2 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -481,6 +481,11 @@ + + + + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 1e099f3285e..c7bdd43a4d7 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -1292,6 +1292,21 @@ src\core\ext\census + + src\core\ext\census + + + src\core\ext\census + + + src\core\ext\census + + + src\core\ext\census + + + src\core\ext\census + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index dcb943a3a11..5db67a1a4c2 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -447,6 +447,11 @@ + + + + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index ac84e0188c8..d60461a2bb3 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -1133,6 +1133,21 @@ src\core\ext\census + + src\core\ext\census + + + src\core\ext\census + + + src\core\ext\census + + + src\core\ext\census + + + src\core\ext\census + From 7e87c3292927e2c4fa0a0adeaf614cdd0cc4710b Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 23 Jan 2017 14:11:23 -0800 Subject: [PATCH 011/167] generate_projects.sh --- CMakeLists.txt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ca0b805429..c10773fc02a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6919,6 +6919,30 @@ target_link_libraries(percent_encoding_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +add_executable(pollset_set_test + test/core/iomgr/pollset_set_test.c +) + +target_include_directories(pollset_set_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include +) + +target_link_libraries(pollset_set_test + grpc_test_util + grpc + gpr_test_util + gpr +) + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_executable(resolve_address_test test/core/iomgr/resolve_address_test.c ) From b0bd22dfc7116975245c6d15fcc9485a25885845 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 20 Jan 2017 17:06:23 -0800 Subject: [PATCH 012/167] Test packet coalescing using TLS endpoint --- .../tests/CronetUnitTests/CronetUnitTests.m | 228 +++++++++++++++++- 1 file changed, 225 insertions(+), 3 deletions(-) diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m index dbd28076ddb..dcd7f2fa8db 100644 --- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m +++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m @@ -50,6 +50,9 @@ #import "src/core/lib/support/string.h" #import "src/core/lib/support/tmpfile.h" #import "test/core/util/test_config.h" +#import "test/core/end2end/data/ssl_test_data.h" + +#import static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; @@ -77,20 +80,70 @@ static void drain_cq(grpc_completion_queue *cq) { grpc_init(); [Cronet setHttp2Enabled:YES]; + [Cronet setSslKeyLogFileName:@"Documents/key"]; + [Cronet enableTestCertVerifierForTesting]; NSURL *url = [[[NSFileManager defaultManager] URLsForDirectory:NSDocumentDirectory inDomains:NSUserDomainMask] lastObject]; NSLog(@"Documents directory: %@", url); [Cronet start]; [Cronet startNetLogToFile:@"Documents/cronet_netlog.json" logBytes:YES]; + + init_ssl(); } + (void)tearDown { grpc_shutdown(); + cleanup_ssl(); [super tearDown]; } +void init_ssl(void) { + SSL_load_error_strings(); + OpenSSL_add_ssl_algorithms(); +} + +void cleanup_ssl(void) { + EVP_cleanup(); +} + +int alpn_cb(SSL *ssl, const unsigned char **out, unsigned char *outlen, + const unsigned char *in, unsigned int inlen, void *arg) { + // Always select "h2" as the ALPN protocol to be used + *out = (const unsigned char*)"h2"; + *outlen = 2; + return SSL_TLSEXT_ERR_OK; +} + +void init_ctx(SSL_CTX *ctx) { + // Install server certificate + BIO *pem = BIO_new_mem_buf((void*)test_server1_cert, (int)strlen(test_server1_cert)); + X509 *cert = PEM_read_bio_X509_AUX(pem, NULL, NULL, ""); + SSL_CTX_use_certificate(ctx, cert); + X509_free(cert); + BIO_free(pem); + + // Install server private key + pem = BIO_new_mem_buf((void *)test_server1_key, (int)strlen(test_server1_key)); + EVP_PKEY *key = PEM_read_bio_PrivateKey(pem, NULL, NULL, ""); + SSL_CTX_use_PrivateKey(ctx, key); + EVP_PKEY_free(key); + BIO_free(pem); + + // Select cipher suite + SSL_CTX_set_cipher_list(ctx, "ECDHE-RSA-AES128-GCM-SHA256"); + + // Select ALPN protocol + SSL_CTX_set_alpn_select_cb(ctx, alpn_cb, NULL); +} + +unsigned int parse_h2_length(const char *field) { + return ((unsigned int)(unsigned char)(field[0])) * 65536 + + ((unsigned int)(unsigned char)(field[1])) * 256 + + ((unsigned int)(unsigned char)(field[2])); +} + - (void)testInternalError { grpc_call *c; grpc_slice request_payload_slice = @@ -106,7 +159,7 @@ static void drain_cq(grpc_completion_queue *cq) { char *addr; gpr_join_host_port(&addr, "127.0.0.1", port); grpc_completion_queue *cq = grpc_completion_queue_create(NULL); - cronet_engine *cronetEngine = [Cronet getGlobalEngine]; + stream_engine *cronetEngine = [Cronet getGlobalEngine]; grpc_channel *client = grpc_cronet_secure_channel_create(cronetEngine, addr, NULL, NULL); @@ -174,14 +227,19 @@ static void drain_cq(grpc_completion_queue *cq) { dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ int sl = socket(AF_INET, SOCK_STREAM, 0); GPR_ASSERT(sl >= 0); + + // Make and TCP endpoint to accept the connection struct sockaddr_in s_addr; memset(&s_addr, 0, sizeof(s_addr)); s_addr.sin_family = AF_INET; s_addr.sin_addr.s_addr = htonl(INADDR_ANY); s_addr.sin_port = htons(port); - bind(sl, (struct sockaddr*)&s_addr, sizeof(s_addr)); - listen(sl, 5); + GPR_ASSERT(0 == bind(sl, (struct sockaddr*)&s_addr, sizeof(s_addr))); + GPR_ASSERT(0 == listen(sl, 5)); int s = accept(sl, NULL, NULL); + GPR_ASSERT(s >= 0); + + // Close the connection after 1 second to trigger Cronet's on_failed() sleep(1); close(s); close(sl); @@ -211,4 +269,168 @@ static void drain_cq(grpc_completion_queue *cq) { grpc_completion_queue_destroy(cq); } +- (void)testPacketCoalescing { + grpc_call *c; + grpc_slice request_payload_slice = + grpc_slice_from_copied_string("hello world"); + grpc_byte_buffer *request_payload = + grpc_raw_byte_buffer_create(&request_payload_slice, 1); + gpr_timespec deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5); + grpc_metadata meta_c[2] = { + {"key1", "val1", 4, 0, {{NULL, NULL, NULL, NULL}}}, + {"key2", "val2", 4, 0, {{NULL, NULL, NULL, NULL}}}}; + + int port = grpc_pick_unused_port_or_die(); + char *addr; + gpr_join_host_port(&addr, "127.0.0.1", port); + grpc_completion_queue *cq = grpc_completion_queue_create(NULL); + stream_engine *cronetEngine = [Cronet getGlobalEngine]; + grpc_channel *client = grpc_cronet_secure_channel_create(cronetEngine, addr, + NULL, NULL); + + cq_verifier *cqv = cq_verifier_create(cq); + grpc_op ops[6]; + grpc_op *op; + grpc_metadata_array initial_metadata_recv; + grpc_metadata_array trailing_metadata_recv; + grpc_metadata_array request_metadata_recv; + grpc_byte_buffer *response_payload_recv = NULL; + grpc_call_details call_details; + grpc_status_code status; + grpc_call_error error; + char *details = NULL; + size_t details_capacity = 0; + + c = grpc_channel_create_call( + client, NULL, GRPC_PROPAGATE_DEFAULTS, cq, "/foo", + NULL, deadline, NULL); + GPR_ASSERT(c); + + grpc_metadata_array_init(&initial_metadata_recv); + grpc_metadata_array_init(&trailing_metadata_recv); + grpc_metadata_array_init(&request_metadata_recv); + grpc_call_details_init(&call_details); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 2; + op->data.send_initial_metadata.metadata = meta_c; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_MESSAGE; + op->data.send_message = request_payload; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata = &initial_metadata_recv; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_MESSAGE; + op->data.recv_message = &response_payload_recv; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; + op->data.recv_status_on_client.status = &status; + op->data.recv_status_on_client.status_details = &details; + op->data.recv_status_on_client.status_details_capacity = &details_capacity; + op->flags = 0; + op->reserved = NULL; + op++; + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), (void*)1, NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + int sl = socket(AF_INET, SOCK_STREAM, 0); + GPR_ASSERT(sl >= 0); + struct sockaddr_in s_addr; + memset(&s_addr, 0, sizeof(s_addr)); + s_addr.sin_family = AF_INET; + s_addr.sin_addr.s_addr = htonl(INADDR_ANY); + s_addr.sin_port = htons(port); + GPR_ASSERT(0 == bind(sl, (struct sockaddr*)&s_addr, sizeof(s_addr))); + GPR_ASSERT(0 == listen(sl, 5)); + int s = accept(sl, NULL, NULL); + GPR_ASSERT(s >= 0); + struct timeval tv; + tv.tv_sec = 2; + tv.tv_usec = 0; + setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + + // Make an TLS endpoint to receive Cronet's transmission + SSL_CTX *ctx = SSL_CTX_new(TLSv1_2_server_method()); + init_ctx(ctx); + SSL *ssl = SSL_new(ctx); + SSL_set_fd(ssl, s); + SSL_accept(ssl); + + const char magic[] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; + + char buf[4096]; + long len; + bool coalesced = false; + while ((len = SSL_read(ssl, buf, sizeof(buf))) > 0) { + NSLog(@"Read len: %ld", len); + + // Analyze the HTTP/2 frames in the same TLS PDU to identify if + // coalescing is successful + unsigned int p = 0; + while (p < len) { + if (len - p >= 24 && 0 == memcmp(&buf[p], magic, 24)) { + p += 24; + continue; + } + + if (buf[p+3] == 0 && // Type is DATA + parse_h2_length(&buf[p]) == 0x10 && // Length is correct + (buf[p+4] & 1) != 0 && // EOS bit is set + 0 == memcmp("hello world", &buf[p+14], 11)) { // Message is correct + coalesced = true; + break; + } + p += (parse_h2_length(&buf[p]) + 9); + } + if (coalesced) { + break; + } + } + + XCTAssert(coalesced); + SSL_free(ssl); + SSL_CTX_free(ctx); + close(s); + close(sl); + }); + + CQ_EXPECT_COMPLETION(cqv, (void*)1, 1); + cq_verify(cqv); + + gpr_free(details); + grpc_metadata_array_destroy(&initial_metadata_recv); + grpc_metadata_array_destroy(&trailing_metadata_recv); + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + + grpc_call_destroy(c); + + cq_verifier_destroy(cqv); + + grpc_byte_buffer_destroy(request_payload); + grpc_byte_buffer_destroy(response_payload_recv); + + grpc_channel_destroy(client); + grpc_completion_queue_shutdown(cq); + drain_cq(cq); + grpc_completion_queue_destroy(cq); +} + @end From 60ab7ef00ac0a988ee2672c636d946c964e6fa41 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 23 Jan 2017 23:00:35 -0800 Subject: [PATCH 013/167] Dynamically enable/disable packet coalecsing and test it --- include/grpc/grpc_cronet.h | 2 + .../client/secure/cronet_channel_create.c | 6 ++ .../cronet/transport/cronet_transport.c | 72 +++++++++---------- .../tests/CronetUnitTests/CronetUnitTests.m | 13 +++- src/objective-c/tests/Podfile | 1 - 5 files changed, 52 insertions(+), 42 deletions(-) diff --git a/include/grpc/grpc_cronet.h b/include/grpc/grpc_cronet.h index 295e0f55e80..566c34a388d 100644 --- a/include/grpc/grpc_cronet.h +++ b/include/grpc/grpc_cronet.h @@ -44,6 +44,8 @@ GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( void *engine, const char *target, const grpc_channel_args *args, void *reserved); +GRPCAPI void grpc_cronet_use_packet_coalescing(bool use_coalescing); + #ifdef __cplusplus } #endif diff --git a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c index 477cf07f45d..2e40020ae05 100644 --- a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c +++ b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c @@ -51,6 +51,8 @@ typedef struct cronet_transport { extern grpc_transport_vtable grpc_cronet_vtable; +bool grpc_cronet_packet_coalescing_enabled = true; + GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( void *engine, const char *target, const grpc_channel_args *args, void *reserved) { @@ -67,3 +69,7 @@ GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( return grpc_channel_create(&exec_ctx, target, args, GRPC_CLIENT_DIRECT_CHANNEL, (grpc_transport *)ct); } + +GRPCAPI void grpc_cronet_use_packet_coalescing(bool use_coalescing) { + grpc_cronet_packet_coalescing_enabled = use_coalescing; +} diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 447f3f31ec1..5429eb32e3c 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -61,6 +61,8 @@ /* TODO (makdharma): Hook up into the wider tracing mechanism */ int grpc_cronet_trace = 0; +extern bool grpc_cronet_packet_coalescing_enabled; + enum e_op_result { ACTION_TAKEN_WITH_CALLBACK, ACTION_TAKEN_NO_CALLBACK, @@ -150,12 +152,13 @@ struct op_state { bool state_callback_received[OP_NUM_OPS]; bool fail_state; bool flush_read; -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING bool flush_cronet_when_ready; bool pending_write_for_trailer; -#endif bool unprocessed_send_message; grpc_error *cancel_error; + + /* Whether packet coalescing is enabled */ + bool packet_coalescing_enabled; /* data structure for storing data coming from server */ struct read_state rs; /* data structure for storing data going to the server */ @@ -425,12 +428,10 @@ static void on_stream_ready(bidirectional_stream *stream) { } /* Send the initial metadata on wire if there is no SEND_MESSAGE or * SEND_TRAILING_METADATA ops pending */ -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - if (s->state.flush_cronet_when_ready) { + if (s->state.packet_coalescing_enabled && s->state.flush_cronet_when_ready) { CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_flush (%p)", s->cbs); bidirectional_stream_flush(stream); } -#endif gpr_mu_unlock(&s->mu); execute_from_storage(s); } @@ -568,10 +569,10 @@ static void on_response_trailers_received( CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, 0)", s->cbs); s->state.state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, "", 0, true); -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); - bidirectional_stream_flush(s->cbs); -#endif + if (s->state.packet_coalescing_enabled) { + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); + bidirectional_stream_flush(s->cbs); + } s->state.state_op_done[OP_SEND_TRAILING_METADATA] = true; gpr_mu_unlock(&s->mu); @@ -768,11 +769,9 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, result = false; /* we haven't got on_write_completed for the send yet */ else if (stream_state->state_op_done[OP_SEND_MESSAGE] && - !stream_state->state_callback_received[OP_SEND_MESSAGE] -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - && !stream_state->pending_write_for_trailer -#endif - ) + !stream_state->state_callback_received[OP_SEND_MESSAGE] && + !(stream_state->packet_coalescing_enabled && + stream_state->pending_write_for_trailer)) result = false; } else if (op_id == OP_CANCEL_ERROR) { /* already executed */ @@ -858,10 +857,10 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, s->cbs = bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); CRONET_LOG(GPR_DEBUG, "%p = bidirectional_stream_create()", s->cbs); -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - bidirectional_stream_disable_auto_flush(s->cbs, true); - bidirectional_stream_delay_request_headers_until_flush(s->cbs, true); -#endif + if (stream_state->packet_coalescing_enabled) { + bidirectional_stream_disable_auto_flush(s->cbs, true); + bidirectional_stream_delay_request_headers_until_flush(s->cbs, true); + } char *url = NULL; const char *method = "POST"; s->header_array.headers = NULL; @@ -872,11 +871,10 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, CRONET_LOG(GPR_DEBUG, "bidirectional_stream_start(%p, %s)", s->cbs, url); bidirectional_stream_start(s->cbs, url, 0, method, &s->header_array, false); stream_state->state_op_done[OP_SEND_INITIAL_METADATA] = true; -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - if (!stream_op->send_message && !stream_op->send_trailing_metadata) { + if (stream_state->packet_coalescing_enabled && !stream_op->send_message && + !stream_op->send_trailing_metadata) { s->state.flush_cronet_when_ready = true; } -#endif result = ACTION_TAKEN_WITH_CALLBACK; } else if (stream_op->send_message && op_can_be_run(stream_op, stream_state, &oas->state, @@ -913,19 +911,18 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, (int)write_buffer_size, false); -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - if (!stream_op->send_trailing_metadata) { - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", - s->cbs); - bidirectional_stream_flush(s->cbs); - result = ACTION_TAKEN_WITH_CALLBACK; + if (stream_state->packet_coalescing_enabled) { + if (!stream_op->send_trailing_metadata) { + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); + bidirectional_stream_flush(s->cbs); + result = ACTION_TAKEN_WITH_CALLBACK; + } else { + stream_state->pending_write_for_trailer = true; + result = ACTION_TAKEN_NO_CALLBACK; + } } else { - stream_state->pending_write_for_trailer = true; - result = ACTION_TAKEN_NO_CALLBACK; + result = ACTION_TAKEN_WITH_CALLBACK; } -#else - result = ACTION_TAKEN_WITH_CALLBACK; -#endif } else { result = NO_ACTION_POSSIBLE; } @@ -944,10 +941,10 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, s->cbs); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, "", 0, true); -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); - bidirectional_stream_flush(s->cbs); -#endif + if (stream_state->packet_coalescing_enabled) { + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); + bidirectional_stream_flush(s->cbs); + } result = ACTION_TAKEN_WITH_CALLBACK; } stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true; @@ -1176,10 +1173,9 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, sizeof(s->state.state_callback_received)); s->state.fail_state = s->state.flush_read = false; s->state.cancel_error = NULL; -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING s->state.flush_cronet_when_ready = s->state.pending_write_for_trailer = false; -#endif s->state.unprocessed_send_message = false; + s->state.packet_coalescing_enabled = grpc_cronet_packet_coalescing_enabled; gpr_mu_init(&s->mu); return 0; } diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m index dcd7f2fa8db..9bbf3cdb113 100644 --- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m +++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m @@ -269,7 +269,9 @@ unsigned int parse_h2_length(const char *field) { grpc_completion_queue_destroy(cq); } -- (void)testPacketCoalescing { +- (void)PacketCoalescing:(bool)use_coalescing { + grpc_cronet_use_packet_coalescing(use_coalescing); + grpc_call *c; grpc_slice request_payload_slice = grpc_slice_from_copied_string("hello world"); @@ -379,7 +381,7 @@ unsigned int parse_h2_length(const char *field) { long len; bool coalesced = false; while ((len = SSL_read(ssl, buf, sizeof(buf))) > 0) { - NSLog(@"Read len: %ld", len); + gpr_log(GPR_DEBUG, "Read len: %ld", len); // Analyze the HTTP/2 frames in the same TLS PDU to identify if // coalescing is successful @@ -404,7 +406,7 @@ unsigned int parse_h2_length(const char *field) { } } - XCTAssert(coalesced); + XCTAssert(coalesced == use_coalescing); SSL_free(ssl); SSL_CTX_free(ctx); close(s); @@ -433,4 +435,9 @@ unsigned int parse_h2_length(const char *field) { grpc_completion_queue_destroy(cq); } +- (void)testPacketCoalescing { + [self PacketCoalescing:false]; + [self PacketCoalescing:true]; +} + @end diff --git a/src/objective-c/tests/Podfile b/src/objective-c/tests/Podfile index 462c6a8e0eb..3760330be9a 100644 --- a/src/objective-c/tests/Podfile +++ b/src/objective-c/tests/Podfile @@ -97,7 +97,6 @@ post_install do |installer| # GPR_UNREACHABLE_CODE causes "Control may reach end of non-void # function" warning config.build_settings['GCC_WARN_ABOUT_RETURN_TYPE'] = 'NO' - config.build_settings['GCC_PREPROCESSOR_DEFINITIONS'] = '$(inherited) COCOAPODS=1 GRPC_CRONET_WITH_PACKET_COALESCING=1' end end From 8d1d95d07cdb7e8fa78ab93399fcf3742fcee63f Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 23 Jan 2017 23:01:10 -0800 Subject: [PATCH 014/167] Clean up old packet coalescing tests --- .../CoreCronetEnd2EndTests.m | 11 - test/core/end2end/tests/packet_coalescing.c | 284 ------------------ 2 files changed, 295 deletions(-) delete mode 100644 test/core/end2end/tests/packet_coalescing.c diff --git a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m index 9af38fd9b22..01612a84b28 100644 --- a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m +++ b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m @@ -170,9 +170,6 @@ static grpc_end2end_test_config configs[] = { chttp2_tear_down_secure_fullstack}, }; -extern void packet_coalescing(grpc_end2end_test_config config); -extern void packet_coalescing_pre_init(void); - static char *roots_filename; @interface CoreCronetEnd2EndTests : XCTestCase @@ -351,14 +348,6 @@ static char *roots_filename; [self testIndividualCase:"no_op"]; } -- (void)testPacketCoalescing { - // Directly invoke the test function since the test is for Cronet only and thus not included in - // end2end_tests.c - // TODO (mxyan): Do the same to all test cases so that this file will no longer depend on - // end2end_tests.c - packet_coalescing(configs[0]); -} - - (void)testPayload { [self testIndividualCase:"payload"]; } diff --git a/test/core/end2end/tests/packet_coalescing.c b/test/core/end2end/tests/packet_coalescing.c deleted file mode 100644 index 336f765a13f..00000000000 --- a/test/core/end2end/tests/packet_coalescing.c +++ /dev/null @@ -1,284 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ -#include "test/core/end2end/end2end_tests.h" - -#include -#include - -#include -#include -#include -#include -#include -#include "test/core/end2end/cq_verifier.h" - -extern void gpr_default_log(gpr_log_func_args *args); - -static void *tag(intptr_t t) { return (void *)t; } - -static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, - const char *test_name, - grpc_channel_args *client_args, - grpc_channel_args *server_args) { - grpc_end2end_test_fixture f; - gpr_log(GPR_INFO, "%s/%s", test_name, config.name); - f = config.create_fixture(client_args, server_args); - config.init_server(&f, server_args); - config.init_client(&f, client_args); - return f; -} - -static gpr_timespec n_seconds_time(int n) { - return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(n); -} - -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } - -static void drain_cq(grpc_completion_queue *cq) { - grpc_event ev; - do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); - } while (ev.type != GRPC_QUEUE_SHUTDOWN); -} - -static void shutdown_server(grpc_end2end_test_fixture *f) { - if (!f->server) return; - grpc_server_shutdown_and_notify(f->server, f->cq, tag(1000)); - GPR_ASSERT(grpc_completion_queue_pluck( - f->cq, tag(1000), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5), NULL) - .type == GRPC_OP_COMPLETE); - grpc_server_destroy(f->server); - f->server = NULL; -} - -static void shutdown_client(grpc_end2end_test_fixture *f) { - if (!f->client) return; - grpc_channel_destroy(f->client); - f->client = NULL; -} - -static void end_test(grpc_end2end_test_fixture *f) { - shutdown_server(f); - shutdown_client(f); - - grpc_completion_queue_shutdown(f->cq); - drain_cq(f->cq); - grpc_completion_queue_destroy(f->cq); -} - -static bool coalesced_message_and_eos; - -static void log_processor(gpr_log_func_args *args) { - const int file_len = (int)strlen(args->file); - const char suffix1[] = "secure_endpoint.c"; - const int suffix1_len = sizeof(suffix1) - 1; - const char suffix2[] = "tcp_posix.c"; - const int suffix2_len = sizeof(suffix2) - 1; - const char prefix[] = "READ"; - const int prefix_len = sizeof(prefix) - 1; - if (((file_len >= suffix1_len && - 0 == strcmp(suffix1, &args->file[file_len - suffix1_len])) || - (file_len >= suffix2_len && - 0 == strcmp(suffix2, &args->file[file_len - suffix2_len]))) && - 0 == strncmp(prefix, args->message, (size_t)prefix_len) && - strstr(args->message, - "00 00 10 00 01 00 00 00 01 00 00 00 00 0b 68 65 6c 6c 6f 20 77 " - "6f 72 6c 64")) { - fprintf(stderr, "%s, %s\n", args->file, args->message); - coalesced_message_and_eos = true; - } -} - -/* Request/response with metadata and payload.*/ -static void test_request_response_with_metadata_and_payload( - grpc_end2end_test_config config) { - grpc_call *c; - grpc_call *s; - grpc_slice request_payload_slice = - grpc_slice_from_copied_string("hello world"); - grpc_byte_buffer *request_payload = - grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); - grpc_metadata meta_c[2] = { - {"key1", "val1", 4, 0, {{NULL, NULL, NULL, NULL}}}, - {"key2", "val2", 4, 0, {{NULL, NULL, NULL, NULL}}}}; - grpc_metadata meta_s[2] = { - {"key3", "val3", 4, 0, {{NULL, NULL, NULL, NULL}}}, - {"key4", "val4", 4, 0, {{NULL, NULL, NULL, NULL}}}}; - grpc_end2end_test_fixture f = begin_test( - config, "test_request_response_with_metadata_and_payload", NULL, NULL); - cq_verifier *cqv = cq_verifier_create(f.cq); - grpc_op ops[6]; - grpc_op *op; - grpc_metadata_array initial_metadata_recv; - grpc_metadata_array trailing_metadata_recv; - grpc_metadata_array request_metadata_recv; - grpc_byte_buffer *request_payload_recv = NULL; - grpc_call_details call_details; - grpc_status_code status; - grpc_call_error error; - char *details = NULL; - size_t details_capacity = 0; - int was_cancelled = 2; - coalesced_message_and_eos = false; - - c = grpc_channel_create_call( - f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, "/foo", - get_host_override_string("foo.test.google.fr:1234", config), deadline, - NULL); - GPR_ASSERT(c); - - grpc_metadata_array_init(&initial_metadata_recv); - grpc_metadata_array_init(&trailing_metadata_recv); - grpc_metadata_array_init(&request_metadata_recv); - grpc_call_details_init(&call_details); - - memset(ops, 0, sizeof(ops)); - op = ops; - op->op = GRPC_OP_SEND_INITIAL_METADATA; - op->data.send_initial_metadata.count = 2; - op->data.send_initial_metadata.metadata = meta_c; - op->flags = 0; - op->reserved = NULL; - op++; - op->op = GRPC_OP_SEND_MESSAGE; - op->data.send_message = request_payload; - op->flags = 0; - op->reserved = NULL; - op++; - op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; - op->flags = 0; - op->reserved = NULL; - op++; - op->op = GRPC_OP_RECV_INITIAL_METADATA; - op->data.recv_initial_metadata = &initial_metadata_recv; - op->flags = 0; - op->reserved = NULL; - op++; - op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; - op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; - op->data.recv_status_on_client.status = &status; - op->data.recv_status_on_client.status_details = &details; - op->data.recv_status_on_client.status_details_capacity = &details_capacity; - op->flags = 0; - op->reserved = NULL; - op++; - error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL); - GPR_ASSERT(GRPC_CALL_OK == error); - - error = - grpc_server_request_call(f.server, &s, &call_details, - &request_metadata_recv, f.cq, f.cq, tag(101)); - GPR_ASSERT(GRPC_CALL_OK == error); - CQ_EXPECT_COMPLETION(cqv, tag(101), 1); - cq_verify(cqv); - - memset(ops, 0, sizeof(ops)); - op = ops; - op->op = GRPC_OP_SEND_INITIAL_METADATA; - op->data.send_initial_metadata.count = 2; - op->data.send_initial_metadata.metadata = meta_s; - op->flags = 0; - op->reserved = NULL; - op++; - op->op = GRPC_OP_RECV_MESSAGE; - op->data.recv_message = &request_payload_recv; - op->flags = 0; - op->reserved = NULL; - op++; - error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), NULL); - GPR_ASSERT(GRPC_CALL_OK == error); - - CQ_EXPECT_COMPLETION(cqv, tag(102), 1); - cq_verify(cqv); - - memset(ops, 0, sizeof(ops)); - op = ops; - op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; - op->data.recv_close_on_server.cancelled = &was_cancelled; - op->flags = 0; - op->reserved = NULL; - op++; - op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; - op->data.send_status_from_server.trailing_metadata_count = 0; - op->data.send_status_from_server.status = GRPC_STATUS_OK; - op->data.send_status_from_server.status_details = "xyz"; - op->flags = 0; - op->reserved = NULL; - op++; - error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(103), NULL); - GPR_ASSERT(GRPC_CALL_OK == error); - - CQ_EXPECT_COMPLETION(cqv, tag(103), 1); - CQ_EXPECT_COMPLETION(cqv, tag(1), 1); - cq_verify(cqv); - - GPR_ASSERT(status == GRPC_STATUS_OK); - GPR_ASSERT(0 == strcmp(details, "xyz")); - GPR_ASSERT(0 == strcmp(call_details.method, "/foo")); - validate_host_override_string("foo.test.google.fr:1234", call_details.host, - config); - GPR_ASSERT(was_cancelled == 0); - GPR_ASSERT(byte_buffer_eq_string(request_payload_recv, "hello world")); - GPR_ASSERT(contains_metadata(&request_metadata_recv, "key1", "val1")); - GPR_ASSERT(contains_metadata(&request_metadata_recv, "key2", "val2")); - GPR_ASSERT(coalesced_message_and_eos); - - gpr_free(details); - grpc_metadata_array_destroy(&initial_metadata_recv); - grpc_metadata_array_destroy(&trailing_metadata_recv); - grpc_metadata_array_destroy(&request_metadata_recv); - grpc_call_details_destroy(&call_details); - - grpc_call_destroy(c); - grpc_call_destroy(s); - - cq_verifier_destroy(cqv); - - grpc_byte_buffer_destroy(request_payload); - grpc_byte_buffer_destroy(request_payload_recv); - - end_test(&f); - config.tear_down_data(&f); -} - -void packet_coalescing(grpc_end2end_test_config config) { - gpr_set_log_verbosity(GPR_LOG_SEVERITY_DEBUG); - grpc_tracer_set_enabled("all", 1); - gpr_set_log_function(log_processor); - test_request_response_with_metadata_and_payload(config); - gpr_set_log_function(gpr_default_log); - grpc_tracer_set_enabled("all", 0); -} - -void packet_coalescing_pre_init(void) {} From 9078535c0a8f2192a3c48362aa17fdba6c8951ab Mon Sep 17 00:00:00 2001 From: Vizerai Date: Tue, 24 Jan 2017 13:02:45 -0800 Subject: [PATCH 015/167] updated headers. --- src/core/ext/census/tracing.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/ext/census/tracing.c b/src/core/ext/census/tracing.c index 8d22f43eed5..566d539e6ef 100644 --- a/src/core/ext/census/tracing.c +++ b/src/core/ext/census/tracing.c @@ -32,10 +32,11 @@ */ #include "src/core/ext/census/tracing.h" + #include #include #include -#include +#include "third_party/boringssl/include/openssl/rand.h" #include "src/core/ext/census/mlog.h" void trace_start_span(const trace_span_context *span_ctxt, From c09e61bbda22d81e49e961a3af89bc9e4ad3c589 Mon Sep 17 00:00:00 2001 From: Vizerai Date: Tue, 24 Jan 2017 16:46:31 -0800 Subject: [PATCH 016/167] update --- BUILD | 1 + src/core/ext/census/tracing.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/BUILD b/BUILD index 10d93d1f9cd..507f7abfbb8 100644 --- a/BUILD +++ b/BUILD @@ -292,6 +292,7 @@ grpc_cc_library( ], external_deps = [ "nanopb", + "submodule_boringssl", ], language = "c", public_hdrs = [ diff --git a/src/core/ext/census/tracing.c b/src/core/ext/census/tracing.c index 566d539e6ef..f02aefc3d7d 100644 --- a/src/core/ext/census/tracing.c +++ b/src/core/ext/census/tracing.c @@ -36,7 +36,7 @@ #include #include #include -#include "third_party/boringssl/include/openssl/rand.h" +#include "third_party/boringssl-with-bazel/src/include/openssl/rand.h" #include "src/core/ext/census/mlog.h" void trace_start_span(const trace_span_context *span_ctxt, From b0b86d7504dc7cf25d2616927196203c8ee41373 Mon Sep 17 00:00:00 2001 From: Vizerai Date: Wed, 25 Jan 2017 13:45:54 -0800 Subject: [PATCH 017/167] updated to correctly build with Bazel. --- BUILD | 2 +- src/core/ext/census/tracing.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILD b/BUILD index 507f7abfbb8..b4835696324 100644 --- a/BUILD +++ b/BUILD @@ -292,7 +292,7 @@ grpc_cc_library( ], external_deps = [ "nanopb", - "submodule_boringssl", + "libssl", ], language = "c", public_hdrs = [ diff --git a/src/core/ext/census/tracing.c b/src/core/ext/census/tracing.c index f02aefc3d7d..8b74628f4fb 100644 --- a/src/core/ext/census/tracing.c +++ b/src/core/ext/census/tracing.c @@ -36,7 +36,7 @@ #include #include #include -#include "third_party/boringssl-with-bazel/src/include/openssl/rand.h" +#include #include "src/core/ext/census/mlog.h" void trace_start_span(const trace_span_context *span_ctxt, From 113267be684063fd4e8db5e83954d6ba1d520b53 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 25 Jan 2017 14:37:19 -0800 Subject: [PATCH 018/167] Address code review comments --- test/core/iomgr/pollset_set_test.c | 53 +++++++++++------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index 335e3662e6d..e0675bd9d2c 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -32,9 +32,8 @@ */ #include "src/core/lib/iomgr/port.h" -/* This test only relevant on linux systems */ -#ifdef GRPC_POSIX_SOCKET -#include "src/core/lib/iomgr/ev_posix.h" +/* This test only relevant on linux systems where epoll is available */ +#ifdef GRPC_LINUX_EPOLL #include #include @@ -43,6 +42,7 @@ #include #include +#include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr.h" #include "test/core/util/test_config.h" @@ -52,16 +52,14 @@ typedef struct test_pollset_set { grpc_pollset_set *pss; } test_pollset_set; -void init_test_pollset_sets(test_pollset_set pollset_sets[], int num_pss) { - int i; - for (i = 0; i < num_pss; i++) { +void init_test_pollset_sets(test_pollset_set *pollset_sets, int num_pss) { + for (int i = 0; i < num_pss; i++) { pollset_sets[i].pss = grpc_pollset_set_create(); } } -void cleanup_test_pollset_sets(test_pollset_set pollset_sets[], int num_pss) { - int i; - for (i = 0; i < num_pss; i++) { +void cleanup_test_pollset_sets(test_pollset_set *pollset_sets, int num_pss) { + for (int i = 0; i < num_pss; i++) { grpc_pollset_set_destroy(pollset_sets[i].pss); pollset_sets[i].pss = NULL; } @@ -76,9 +74,8 @@ typedef struct test_pollset { gpr_mu *mu; } test_pollset; -static void init_test_pollsets(test_pollset pollsets[], int num_pollsets) { - int i; - for (i = 0; i < num_pollsets; i++) { +static void init_test_pollsets(test_pollset *pollsets, int num_pollsets) { + for (int i = 0; i < num_pollsets; i++) { pollsets[i].ps = gpr_malloc(grpc_pollset_size()); grpc_pollset_init(pollsets[i].ps, &pollsets[i].mu); } @@ -90,11 +87,9 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, } static void cleanup_test_pollsets(grpc_exec_ctx *exec_ctx, - test_pollset pollsets[], int num_pollsets) { + test_pollset *pollsets, int num_pollsets) { grpc_closure destroyed; - int i; - - for (i = 0; i < num_pollsets; i++) { + for (int i = 0; i < num_pollsets; i++) { grpc_closure_init(&destroyed, destroy_pollset, pollsets[i].ps, grpc_schedule_on_exec_ctx); grpc_pollset_shutdown(exec_ctx, pollsets[i].ps, &destroyed); @@ -129,11 +124,8 @@ static void reset_test_fd(grpc_exec_ctx *exec_ctx, test_fd *tfd) { grpc_fd_notify_on_read(exec_ctx, tfd->fd, &tfd->on_readable); } -static void init_test_fds(grpc_exec_ctx *exec_ctx, test_fd tfds[], - int num_fds) { - int i; - - for (i = 0; i < num_fds; i++) { +static void init_test_fds(grpc_exec_ctx *exec_ctx, test_fd *tfds, int num_fds) { + for (int i = 0; i < num_fds; i++) { GPR_ASSERT(GRPC_ERROR_NONE == grpc_wakeup_fd_init(&tfds[i].wakeup_fd)); tfds[i].fd = grpc_fd_create(GRPC_WAKEUP_FD_GET_READ_FD(&tfds[i].wakeup_fd), "test_fd"); @@ -144,9 +136,8 @@ static void init_test_fds(grpc_exec_ctx *exec_ctx, test_fd tfds[], static void cleanup_test_fds(grpc_exec_ctx *exec_ctx, test_fd *tfds, int num_fds) { int release_fd; - int i; - for (i = 0; i < num_fds; i++) { + for (int i = 0; i < num_fds; i++) { grpc_fd_shutdown(exec_ctx, tfds[i].fd); grpc_exec_ctx_flush(exec_ctx); @@ -162,17 +153,15 @@ static void cleanup_test_fds(grpc_exec_ctx *exec_ctx, test_fd *tfds, } } -static void make_test_fds_readable(test_fd tfds[], int num_fds) { - int i; - for (i = 0; i < num_fds; i++) { +static void make_test_fds_readable(test_fd *tfds, int num_fds) { + for (int i = 0; i < num_fds; i++) { GPR_ASSERT(GRPC_ERROR_NONE == grpc_wakeup_fd_wakeup(&tfds[i].wakeup_fd)); } } -static void verify_readable_and_reset(grpc_exec_ctx *exec_ctx, test_fd tfds[], +static void verify_readable_and_reset(grpc_exec_ctx *exec_ctx, test_fd *tfds, int num_fds) { - int i; - for (i = 0; i < num_fds; i++) { + for (int i = 0; i < num_fds; i++) { /* Verify that the on_readable callback was called */ GPR_ASSERT(tfds[i].is_on_readable_called); @@ -221,7 +210,6 @@ static void pollset_set_test_basic() { * +---> FD9 (Added after PS2 is added to PSS0) */ grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - int i; grpc_pollset_worker *worker; gpr_timespec deadline; @@ -272,7 +260,7 @@ static void pollset_set_test_basic() { * - Verify that on_readable call back was called for all FDs (and * reset the FDs) * */ - for (i = 0; i < num_ps; i++) { + for (int i = 0; i < num_ps; i++) { make_test_fds_readable(tfds, num_fds); gpr_mu_lock(pollsets[i].mu); @@ -450,12 +438,11 @@ void pollset_set_test_empty_pollset() { } int main(int argc, char **argv) { - const char *poll_strategy = NULL; + const char *poll_strategy = grpc_get_poll_strategy_name(); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_iomgr_init(); - poll_strategy = grpc_get_poll_strategy_name(); if (poll_strategy != NULL && strcmp(poll_strategy, "epoll") == 0) { pollset_set_test_basic(); pollset_set_test_dup_fds(); From 9930e4b39486b3eb62609c67b87d7bc80f63e50d Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 25 Jan 2017 14:48:20 -0800 Subject: [PATCH 019/167] Code review comments --- test/core/iomgr/pollset_set_test.c | 38 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index e0675bd9d2c..3e8dcb4f9c9 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -41,6 +41,7 @@ #include #include +#include #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr.h" @@ -52,13 +53,14 @@ typedef struct test_pollset_set { grpc_pollset_set *pss; } test_pollset_set; -void init_test_pollset_sets(test_pollset_set *pollset_sets, int num_pss) { +void init_test_pollset_sets(test_pollset_set *pollset_sets, const int num_pss) { for (int i = 0; i < num_pss; i++) { pollset_sets[i].pss = grpc_pollset_set_create(); } } -void cleanup_test_pollset_sets(test_pollset_set *pollset_sets, int num_pss) { +void cleanup_test_pollset_sets(test_pollset_set *pollset_sets, + const int num_pss) { for (int i = 0; i < num_pss; i++) { grpc_pollset_set_destroy(pollset_sets[i].pss); pollset_sets[i].pss = NULL; @@ -74,7 +76,7 @@ typedef struct test_pollset { gpr_mu *mu; } test_pollset; -static void init_test_pollsets(test_pollset *pollsets, int num_pollsets) { +static void init_test_pollsets(test_pollset *pollsets, const int num_pollsets) { for (int i = 0; i < num_pollsets; i++) { pollsets[i].ps = gpr_malloc(grpc_pollset_size()); grpc_pollset_init(pollsets[i].ps, &pollsets[i].mu); @@ -87,7 +89,8 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, } static void cleanup_test_pollsets(grpc_exec_ctx *exec_ctx, - test_pollset *pollsets, int num_pollsets) { + test_pollset *pollsets, + const int num_pollsets) { grpc_closure destroyed; for (int i = 0; i < num_pollsets; i++) { grpc_closure_init(&destroyed, destroy_pollset, pollsets[i].ps, @@ -124,7 +127,8 @@ static void reset_test_fd(grpc_exec_ctx *exec_ctx, test_fd *tfd) { grpc_fd_notify_on_read(exec_ctx, tfd->fd, &tfd->on_readable); } -static void init_test_fds(grpc_exec_ctx *exec_ctx, test_fd *tfds, int num_fds) { +static void init_test_fds(grpc_exec_ctx *exec_ctx, test_fd *tfds, + const int num_fds) { for (int i = 0; i < num_fds; i++) { GPR_ASSERT(GRPC_ERROR_NONE == grpc_wakeup_fd_init(&tfds[i].wakeup_fd)); tfds[i].fd = grpc_fd_create(GRPC_WAKEUP_FD_GET_READ_FD(&tfds[i].wakeup_fd), @@ -134,7 +138,7 @@ static void init_test_fds(grpc_exec_ctx *exec_ctx, test_fd *tfds, int num_fds) { } static void cleanup_test_fds(grpc_exec_ctx *exec_ctx, test_fd *tfds, - int num_fds) { + const int num_fds) { int release_fd; for (int i = 0; i < num_fds; i++) { @@ -153,14 +157,14 @@ static void cleanup_test_fds(grpc_exec_ctx *exec_ctx, test_fd *tfds, } } -static void make_test_fds_readable(test_fd *tfds, int num_fds) { +static void make_test_fds_readable(test_fd *tfds, const int num_fds) { for (int i = 0; i < num_fds; i++) { GPR_ASSERT(GRPC_ERROR_NONE == grpc_wakeup_fd_wakeup(&tfds[i].wakeup_fd)); } } static void verify_readable_and_reset(grpc_exec_ctx *exec_ctx, test_fd *tfds, - int num_fds) { + const int num_fds) { for (int i = 0; i < num_fds; i++) { /* Verify that the on_readable callback was called */ GPR_ASSERT(tfds[i].is_on_readable_called); @@ -216,9 +220,9 @@ static void pollset_set_test_basic() { test_fd tfds[10]; test_pollset pollsets[3]; test_pollset_set pollset_sets[2]; - int num_fds = sizeof(tfds) / sizeof(tfds[0]); - int num_ps = sizeof(pollsets) / sizeof(pollsets[0]); - int num_pss = sizeof(pollset_sets) / sizeof(pollset_sets[0]); + const int num_fds = GPR_ARRAY_SIZE(tfds); + const int num_ps = GPR_ARRAY_SIZE(pollsets); + const int num_pss = GPR_ARRAY_SIZE(pollset_sets); init_test_fds(&exec_ctx, tfds, num_fds); init_test_pollsets(pollsets, num_ps); @@ -321,9 +325,9 @@ void pollset_set_test_dup_fds() { test_fd tfds[3]; test_pollset pollset; test_pollset_set pollset_sets[2]; - int num_fds = sizeof(tfds) / sizeof(tfds[0]); - int num_ps = 1; - int num_pss = sizeof(pollset_sets) / sizeof(pollset_sets[0]); + const int num_fds = GPR_ARRAY_SIZE(tfds); + const int num_ps = 1; + const int num_pss = GPR_ARRAY_SIZE(pollset_sets); init_test_fds(&exec_ctx, tfds, num_fds); init_test_pollsets(&pollset, num_ps); @@ -394,9 +398,9 @@ void pollset_set_test_empty_pollset() { test_fd tfds[3]; test_pollset pollsets[2]; test_pollset_set pollset_set; - int num_fds = sizeof(tfds) / sizeof(tfds[0]); - int num_ps = sizeof(pollsets) / sizeof(pollsets[0]); - int num_pss = 1; + const int num_fds = GPR_ARRAY_SIZE(tfds); + const int num_ps = GPR_ARRAY_SIZE(pollsets); + const int num_pss = 1; init_test_fds(&exec_ctx, tfds, num_fds); init_test_pollsets(pollsets, num_ps); From 2685ffa738113a00c103525b6bc4bc91e8b4b7a2 Mon Sep 17 00:00:00 2001 From: Vizerai Date: Thu, 26 Jan 2017 18:41:14 -0800 Subject: [PATCH 020/167] update. --- src/core/ext/census/trace_label.h | 6 ++++-- src/core/ext/census/trace_propagation.h | 13 +++++++------ src/core/ext/census/trace_status.h | 2 +- src/core/ext/census/trace_string.h | 5 ++++- src/core/ext/census/tracing.h | 22 +++++++++++++++++----- 5 files changed, 33 insertions(+), 15 deletions(-) diff --git a/src/core/ext/census/trace_label.h b/src/core/ext/census/trace_label.h index b58543b7d8c..0e4a8d885f1 100644 --- a/src/core/ext/census/trace_label.h +++ b/src/core/ext/census/trace_label.h @@ -36,10 +36,13 @@ #include "src/core/ext/census/trace_string.h" -/* Trace label (key/value pair) stores a label name and the label value. */ +/* Trace label (key/value pair) stores a label name and the label value. The + value can be one of trace_string/int64_t/bool. */ typedef struct trace_label { trace_string key; enum label_type { + /* Unknown value for debugging/error purposes */ + LABEL_UNKNOWN = 0, /* A string value */ LABEL_STRING = 1, /* An integer value. */ @@ -53,7 +56,6 @@ typedef struct trace_label { int64_t label_int; bool label_bool; } value; - size_t val_len; } trace_label; #endif diff --git a/src/core/ext/census/trace_propagation.h b/src/core/ext/census/trace_propagation.h index 093492c3e8e..75c4ebaa398 100644 --- a/src/core/ext/census/trace_propagation.h +++ b/src/core/ext/census/trace_propagation.h @@ -36,7 +36,7 @@ #include "src/core/ext/census/tracing.h" -/* Encoding and decoding functions for receiving and sending propagating data +/* Encoding and decoding functions for receiving and sending trace contexts over the wire. Only RPC libraries should be calling these functions. These functions return the number of bytes encoded/decoded (0 if a failure has occurred). buf_size indicates the size of the @@ -44,18 +44,19 @@ trace ID, span ID, and a set of option flags (is_sampled, etc.). */ /* Converts a span context to a binary byte buffer. */ -size_t trace_span_context_to_binary(const trace_span_context *ctxt, char *buf, - size_t buf_size); +size_t trace_span_context_to_binary(const trace_span_context *ctxt, + uint8_t *buf, size_t buf_size); /* Reads a binary byte buffer and populates a span context structure. */ -size_t binary_to_trace_span_context(const char *buf, size_t buf_size, +size_t binary_to_trace_span_context(const uint8_t *buf, size_t buf_size, trace_span_context *ctxt); -/* Converts a span context to a http format buffer. */ +/* Converts a span context to an http metadata compatible string. */ size_t trace_span_context_to_http_format(const trace_span_context *ctxt, char *buf, size_t buf_size); -/* Reads a http format buffer and populates a span context structure. */ +/* Reads an http metadata compatible string and populates a span context + structure. */ size_t http_format_to_trace_span_context(const char *buf, size_t buf_size, trace_span_context *ctxt); diff --git a/src/core/ext/census/trace_status.h b/src/core/ext/census/trace_status.h index a4bd4116c30..adc0ebd0c02 100644 --- a/src/core/ext/census/trace_status.h +++ b/src/core/ext/census/trace_status.h @@ -38,7 +38,7 @@ /* Stores a status code and status message for a trace. */ typedef struct trace_status { - int errorCode; + int64_t errorCode; trace_string errorMessage; } trace_status; diff --git a/src/core/ext/census/trace_string.h b/src/core/ext/census/trace_string.h index 0f71056c3bc..8e77ee9f7ed 100644 --- a/src/core/ext/census/trace_string.h +++ b/src/core/ext/census/trace_string.h @@ -42,6 +42,9 @@ This will also be more efficient when copying, as we have an explicitly specified length. Also, grpc_slice has reference counting which allows for interning. */ -typedef struct trace_string { grpc_slice string_slice; } trace_string; +typedef struct trace_string { + char *string; + size_t length; +} trace_string; #endif diff --git a/src/core/ext/census/tracing.h b/src/core/ext/census/tracing.h index 0632b841bd1..c2b947ae407 100644 --- a/src/core/ext/census/tracing.h +++ b/src/core/ext/census/tracing.h @@ -40,23 +40,31 @@ #include "src/core/ext/census/trace_label.h" #include "src/core/ext/census/trace_status.h" +/* This is the low level tracing API that other languages will interface with. + This is not intended to be accessed by the end-user, therefore it has been + designed with performance in mind rather than ease of use. */ + /* The tracing level. */ enum TraceLevel { /* Annotations on this context will be silently discarded. */ NO_TRACING = 0, /* Annotations will not be saved to a persistent store. They will be - available via local APIs only. It is not propagated to the child. */ + available via local APIs only. This setting is not propagated to child + spans. */ TRANSIENT_TRACING = 1, /* Annotations are recorded for the entire distributed trace and they are - saved to a persistent store. It is propagated to the child. */ + saved to a persistent store. This setting is propagated to child spans. */ PERSISTENT_TRACING = 2, }; typedef struct trace_span_context { /* Trace span context stores Span ID, Trace ID, and option flags. */ + /* Trace ID is 128 bits split into 2 64-bit chunks (hi and lo). */ uint64_t trace_id_hi; uint64_t trace_id_lo; + /* Span ID is 64 bits. */ uint64_t span_id; + /* Span-options is 32-bit value which contains flag options. */ uint32_t span_options; } trace_span_context; @@ -64,7 +72,10 @@ typedef struct start_span_options { /* If set, this will override the Span.local_start_time for the Span. */ gpr_timespec local_start_timestamp; - /* If set, the Spans are linked to the created Span. */ + /* Linked spans can be used to identify spans that are linked to this span in + a different trace. This can be used (for example) in batching operations, + where a single batch handler processes multiple requests from different + traces. If set, points to a list of Spans are linked to the created Span.*/ trace_span_context *linked_spans; /* The number of linked spans. */ size_t n_linked_spans; @@ -79,8 +90,9 @@ void trace_start_span(const trace_span_context *span_ctxt, trace_span_context *new_span_ctxt, bool has_remote_parent); -/* Add a new Annotation to the Span. The description corresponds to - Span->annotations[].description. */ +/* Add a new Annotation to the Span. Annotations consist of a description + (trace_string) and a set of n labels (trace_label). This can be populated + with arbitrary user data. */ void trace_add_span_annotation(const trace_string description, const trace_label *labels, const size_t n_labels, trace_span_context *span_ctxt); From 7d5a560192da155e28181e1e67c4905a88ad8d65 Mon Sep 17 00:00:00 2001 From: Vizerai Date: Fri, 27 Jan 2017 14:12:57 -0800 Subject: [PATCH 021/167] gflags 2.2.0 --- third_party/gflags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/gflags b/third_party/gflags index 05b155ff591..f8a0efe03aa 160000 --- a/third_party/gflags +++ b/third_party/gflags @@ -1 +1 @@ -Subproject commit 05b155ff59114735ec8cd089f669c4c3d8f59029 +Subproject commit f8a0efe03aa69b3336d8e228b37d4ccb17324b88 From 608982d1fc1cf84e2907e948ac4b9cba75d3e782 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 30 Jan 2017 10:45:39 -0800 Subject: [PATCH 022/167] fix grpc_fd_shutdown invocation after latest merge --- test/core/iomgr/pollset_set_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index 3e8dcb4f9c9..f1a56ecafb9 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -142,7 +142,7 @@ static void cleanup_test_fds(grpc_exec_ctx *exec_ctx, test_fd *tfds, int release_fd; for (int i = 0; i < num_fds; i++) { - grpc_fd_shutdown(exec_ctx, tfds[i].fd); + grpc_fd_shutdown(exec_ctx, tfds[i].fd, GRPC_ERROR_CREATE("fd cleanup")); grpc_exec_ctx_flush(exec_ctx); /* grpc_fd_orphan frees the memory allocated for grpc_fd. Normally it also From f7af2a9a0517ada19e0ce1a092ca064985f604e5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 31 Jan 2017 15:08:31 -0800 Subject: [PATCH 023/167] Update latency profiler to use (more appropriate) microbenchmarks --- src/core/lib/slice/slice_intern.c | 6 +- test/cpp/microbenchmarks/bm_fullstack.cc | 4 + .../latency_profile/run_latency_profile.sh | 61 +--------- tools/profiling/microbenchmark/bm.py | 109 ++++++++++++++++++ 4 files changed, 119 insertions(+), 61 deletions(-) create mode 100755 tools/profiling/microbenchmark/bm.py diff --git a/src/core/lib/slice/slice_intern.c b/src/core/lib/slice/slice_intern.c index 7cbd17bffd8..32adc4df97d 100644 --- a/src/core/lib/slice/slice_intern.c +++ b/src/core/lib/slice/slice_intern.c @@ -215,7 +215,9 @@ bool grpc_slice_is_interned(grpc_slice slice) { } grpc_slice grpc_slice_intern(grpc_slice slice) { + GPR_TIMER_BEGIN("grpc_slice_intern", 0); if (GRPC_IS_STATIC_METADATA_STRING(slice)) { + GPR_TIMER_END("grpc_slice_intern", 0); return slice; } @@ -225,6 +227,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) { static_metadata_hash[(hash + i) % GPR_ARRAY_SIZE(static_metadata_hash)]; if (ent.hash == hash && ent.idx < GRPC_STATIC_MDSTR_COUNT && grpc_slice_eq(grpc_static_slice_table[ent.idx], slice)) { + GPR_TIMER_END("grpc_slice_intern", 0); return grpc_static_slice_table[ent.idx]; } } @@ -247,7 +250,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) { /* and treat this as if we were never here... sshhh */ } else { gpr_mu_unlock(&shard->mu); - GPR_TIMER_END("grpc_mdstr_from_buffer", 0); + GPR_TIMER_END("grpc_slice_intern", 0); return materialize(s); } } @@ -275,6 +278,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) { gpr_mu_unlock(&shard->mu); + GPR_TIMER_END("grpc_slice_intern", 0); return materialize(s); } diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index c3e96c572c8..589b319eb70 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -57,6 +57,7 @@ extern "C" { #include "test/core/util/passthru_endpoint.h" #include "test/core/util/port.h" } +#include "src/core/lib/profiling/timers.h" #include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "third_party/benchmark/include/benchmark/benchmark.h" @@ -402,6 +403,7 @@ static void BM_UnaryPingPong(benchmark::State& state) { std::unique_ptr stub( EchoTestService::NewStub(fixture->channel())); while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); recv_response.Clear(); ClientContext cli_ctx; ClientContextMutator cli_ctx_mut(&cli_ctx); @@ -470,6 +472,7 @@ static void BM_PumpStreamClientToServer(benchmark::State& state) { } response_rw.Read(&recv_request, tag(0)); while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); request_rw->Write(send_request, tag(1)); while (true) { GPR_ASSERT(fixture->cq()->Next(&t, &ok)); @@ -527,6 +530,7 @@ static void BM_PumpStreamServerToClient(benchmark::State& state) { } request_rw->Read(&recv_response, tag(0)); while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); response_rw.Write(send_response, tag(1)); while (true) { GPR_ASSERT(fixture->cq()->Next(&t, &ok)); diff --git a/tools/profiling/latency_profile/run_latency_profile.sh b/tools/profiling/latency_profile/run_latency_profile.sh index 618db202dc4..7ebe308e0a0 100755 --- a/tools/profiling/latency_profile/run_latency_profile.sh +++ b/tools/profiling/latency_profile/run_latency_profile.sh @@ -28,55 +28,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# format argument via -# $ echo '{...}' | python -mjson.tool -read -r -d '' SCENARIOS_JSON_ARG <<'EOF' -{ - "scenarios": [ - { - "benchmark_seconds": 5, - "client_config": { - "client_channels": 1, - "client_type": "SYNC_CLIENT", - "histogram_params": { - "max_possible": 60000000000.0, - "resolution": 0.01 - }, - "load_params": { - "closed_loop": {} - }, - "outstanding_rpcs_per_channel": 1, - "payload_config": { - "simple_params": { - "req_size": 0, - "resp_size": 0 - } - }, - "rpc_type": "UNARY", - "security_params": { - "server_host_override": "foo.test.google.fr", - "use_test_ca": true - } - }, - "name": "cpp_protobuf_sync_unary_ping_pong_secure", - "num_clients": 1, - "num_servers": 1, - "server_config": { - "core_limit": 1, - "security_params": { - "server_host_override": "foo.test.google.fr", - "use_test_ca": true - }, - "server_type": "SYNC_SERVER" - }, - "spawn_local_worker_count": 2, - "warmup_seconds": 5 - } - ] -} - -EOF - set -ex cd $(dirname $0)/../../.. @@ -93,14 +44,4 @@ else PYTHON=python2.7 fi -make CONFIG=basicprof -j$CPUS qps_json_driver - -mkdir -p reports -bins/basicprof/qps_json_driver --scenarios_json="$SCENARIOS_JSON_ARG" - -echo 'Latency profile for:
' > reports/index.html -echo "

${SCENARIOS_JSON_ARG}

" >> reports/index.html -echo '

' >> reports/index.html
-$PYTHON tools/profiling/latency_profile/profile_analyzer.py \
-    --source=latency_trace.txt --fmt=simple >> reports/index.html
-echo '

' >> reports/index.html +$PYTHON tools/profiling/microbenchmark/bm.py bm_fullstack diff --git a/tools/profiling/microbenchmark/bm.py b/tools/profiling/microbenchmark/bm.py new file mode 100755 index 00000000000..064bb7af1b4 --- /dev/null +++ b/tools/profiling/microbenchmark/bm.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python2.7 +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import multiprocessing +import os +import subprocess +import sys + +flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph') + +def fnize(s): + out = '' + for c in s: + if c in '<>, /': + if len(out) and out[-1] == '_': continue + out += '_' + else: + out += c + return out + +os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) +if not os.path.exists('reports'): + os.makedirs('reports') + +# index html +index_html = """ + + +Microbenchmark Results + + +""" + +def heading(name): + global index_html + index_html += "

%s

\n" % name + +def link(txt, tgt): + global index_html + index_html += "

%s

\n" % (tgt, txt) + +for bm_name in sys.argv[1:]: + # generate latency profiles + heading('Latency Profiles: %s' % bm_name) + subprocess.check_call( + ['make', bm_name, + 'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()]) + for line in subprocess.check_output(['bins/basicprof/%s' % bm_name, + '--benchmark_list_tests']).splitlines(): + link(line, 'reports/%s.txt' % fnize(line)) + with open('reports/%s.txt' % fnize(line), 'w') as f: + f.write(subprocess.check_output(['bins/basicprof/%s' % bm_name, + '--benchmark_filter=^%s$' % line])) + f.write('\n***********************************************************\n') + f.write(subprocess.check_output([ + sys.executable, 'tools/profiling/latency_profile/profile_analyzer.py', + '--source', 'latency_trace.txt', '--fmt', 'simple'])) + + # generate flamegraphs + heading('Flamegraphs: %s' % bm_name) + subprocess.check_call( + ['make', bm_name, + 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()]) + for line in subprocess.check_output(['bins/mutrace/%s' % bm_name, + '--benchmark_list_tests']).splitlines(): + subprocess.check_call(['sudo', 'perf', 'record', '-g', '-F', '99', + 'bins/mutrace/%s' % bm_name, + '--benchmark_filter=^%s$' % line, + '--benchmark_min_time=20']) + with open('/tmp/bm.perf', 'w') as f: + f.write(subprocess.check_output(['sudo', 'perf', 'script'])) + with open('/tmp/bm.folded', 'w') as f: + f.write(subprocess.check_output([ + '%s/stackcollapse-perf.pl' % flamegraph_dir, '/tmp/bm.perf'])) + link(line, 'reports/%s.svg' % fnize(line)) + with open('reports/%s.svg' % fnize(line), 'w') as f: + f.write(subprocess.check_output([ + '%s/flamegraph.pl' % flamegraph_dir, '/tmp/bm.folded'])) + +index_html += "\n\n" +with open('reports/index.html', 'w') as f: + w.write(index_html) From 7240b54ddb1a10fc4400c81bede0ac0390aa6e0f Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 31 Jan 2017 16:00:49 -0800 Subject: [PATCH 024/167] make grpc-encoding:identity implicit --- src/core/lib/channel/compress_filter.c | 10 ++++++---- test/core/end2end/bad_server_response_test.c | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/core/lib/channel/compress_filter.c b/src/core/lib/channel/compress_filter.c index c860d60d881..22781c78399 100644 --- a/src/core/lib/channel/compress_filter.c +++ b/src/core/lib/channel/compress_filter.c @@ -105,7 +105,6 @@ static grpc_error *process_send_initial_metadata( static grpc_error *process_send_initial_metadata( grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_metadata_batch *initial_metadata) { - grpc_error *error; call_data *calld = elem->call_data; channel_data *channeld = elem->channel_data; /* Parse incoming request for compression. If any, it'll be available @@ -144,10 +143,13 @@ static grpc_error *process_send_initial_metadata( calld->has_compression_algorithm = 1; /* GPR_TRUE */ } + grpc_error *error = GRPC_ERROR_NONE; /* hint compression algorithm */ - error = grpc_metadata_batch_add_tail( - exec_ctx, initial_metadata, &calld->compression_algorithm_storage, - grpc_compression_encoding_mdelem(calld->compression_algorithm)); + if (calld->compression_algorithm != GRPC_COMPRESS_NONE) { + error = grpc_metadata_batch_add_tail( + exec_ctx, initial_metadata, &calld->compression_algorithm_storage, + grpc_compression_encoding_mdelem(calld->compression_algorithm)); + } if (error != GRPC_ERROR_NONE) return error; diff --git a/test/core/end2end/bad_server_response_test.c b/test/core/end2end/bad_server_response_test.c index d5f428eb829..9cfe0cebe01 100644 --- a/test/core/end2end/bad_server_response_test.c +++ b/test/core/end2end/bad_server_response_test.c @@ -82,7 +82,7 @@ #define HTTP1_DETAIL_MSG "Trying to connect an http1.x server" /* TODO(zyc) Check the content of incomming data instead of using this length */ -#define EXPECTED_INCOMING_DATA_LENGTH (size_t)310 +#define EXPECTED_INCOMING_DATA_LENGTH (size_t)300 struct rpc_state { char *target; @@ -135,7 +135,7 @@ static void handle_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { gpr_log(GPR_DEBUG, "got %" PRIuPTR " bytes, expected %" PRIuPTR " bytes", state.incoming_data_length, EXPECTED_INCOMING_DATA_LENGTH); - if (state.incoming_data_length > EXPECTED_INCOMING_DATA_LENGTH) { + if (state.incoming_data_length >= EXPECTED_INCOMING_DATA_LENGTH) { handle_write(exec_ctx); } else { grpc_endpoint_read(exec_ctx, state.tcp, &state.temp_incoming_buffer, From eb5ee45eec0c6f88bcf649f383060f3dc34f2084 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 1 Feb 2017 11:57:33 -0800 Subject: [PATCH 025/167] Revert "Dynamically enable/disable packet coalecsing and test it" This reverts commit 60ab7ef00ac0a988ee2672c636d946c964e6fa41. --- include/grpc/grpc_cronet.h | 2 - .../client/secure/cronet_channel_create.c | 6 -- .../cronet/transport/cronet_transport.c | 72 ++++++++++--------- .../tests/CronetUnitTests/CronetUnitTests.m | 13 +--- src/objective-c/tests/Podfile | 1 + 5 files changed, 42 insertions(+), 52 deletions(-) diff --git a/include/grpc/grpc_cronet.h b/include/grpc/grpc_cronet.h index 566c34a388d..295e0f55e80 100644 --- a/include/grpc/grpc_cronet.h +++ b/include/grpc/grpc_cronet.h @@ -44,8 +44,6 @@ GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( void *engine, const char *target, const grpc_channel_args *args, void *reserved); -GRPCAPI void grpc_cronet_use_packet_coalescing(bool use_coalescing); - #ifdef __cplusplus } #endif diff --git a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c index 2e40020ae05..477cf07f45d 100644 --- a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c +++ b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c @@ -51,8 +51,6 @@ typedef struct cronet_transport { extern grpc_transport_vtable grpc_cronet_vtable; -bool grpc_cronet_packet_coalescing_enabled = true; - GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( void *engine, const char *target, const grpc_channel_args *args, void *reserved) { @@ -69,7 +67,3 @@ GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( return grpc_channel_create(&exec_ctx, target, args, GRPC_CLIENT_DIRECT_CHANNEL, (grpc_transport *)ct); } - -GRPCAPI void grpc_cronet_use_packet_coalescing(bool use_coalescing) { - grpc_cronet_packet_coalescing_enabled = use_coalescing; -} diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 5429eb32e3c..447f3f31ec1 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -61,8 +61,6 @@ /* TODO (makdharma): Hook up into the wider tracing mechanism */ int grpc_cronet_trace = 0; -extern bool grpc_cronet_packet_coalescing_enabled; - enum e_op_result { ACTION_TAKEN_WITH_CALLBACK, ACTION_TAKEN_NO_CALLBACK, @@ -152,13 +150,12 @@ struct op_state { bool state_callback_received[OP_NUM_OPS]; bool fail_state; bool flush_read; +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING bool flush_cronet_when_ready; bool pending_write_for_trailer; +#endif bool unprocessed_send_message; grpc_error *cancel_error; - - /* Whether packet coalescing is enabled */ - bool packet_coalescing_enabled; /* data structure for storing data coming from server */ struct read_state rs; /* data structure for storing data going to the server */ @@ -428,10 +425,12 @@ static void on_stream_ready(bidirectional_stream *stream) { } /* Send the initial metadata on wire if there is no SEND_MESSAGE or * SEND_TRAILING_METADATA ops pending */ - if (s->state.packet_coalescing_enabled && s->state.flush_cronet_when_ready) { +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + if (s->state.flush_cronet_when_ready) { CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_flush (%p)", s->cbs); bidirectional_stream_flush(stream); } +#endif gpr_mu_unlock(&s->mu); execute_from_storage(s); } @@ -569,10 +568,10 @@ static void on_response_trailers_received( CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, 0)", s->cbs); s->state.state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, "", 0, true); - if (s->state.packet_coalescing_enabled) { - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); - bidirectional_stream_flush(s->cbs); - } +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); + bidirectional_stream_flush(s->cbs); +#endif s->state.state_op_done[OP_SEND_TRAILING_METADATA] = true; gpr_mu_unlock(&s->mu); @@ -769,9 +768,11 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, result = false; /* we haven't got on_write_completed for the send yet */ else if (stream_state->state_op_done[OP_SEND_MESSAGE] && - !stream_state->state_callback_received[OP_SEND_MESSAGE] && - !(stream_state->packet_coalescing_enabled && - stream_state->pending_write_for_trailer)) + !stream_state->state_callback_received[OP_SEND_MESSAGE] +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + && !stream_state->pending_write_for_trailer +#endif + ) result = false; } else if (op_id == OP_CANCEL_ERROR) { /* already executed */ @@ -857,10 +858,10 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, s->cbs = bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); CRONET_LOG(GPR_DEBUG, "%p = bidirectional_stream_create()", s->cbs); - if (stream_state->packet_coalescing_enabled) { - bidirectional_stream_disable_auto_flush(s->cbs, true); - bidirectional_stream_delay_request_headers_until_flush(s->cbs, true); - } +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + bidirectional_stream_disable_auto_flush(s->cbs, true); + bidirectional_stream_delay_request_headers_until_flush(s->cbs, true); +#endif char *url = NULL; const char *method = "POST"; s->header_array.headers = NULL; @@ -871,10 +872,11 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, CRONET_LOG(GPR_DEBUG, "bidirectional_stream_start(%p, %s)", s->cbs, url); bidirectional_stream_start(s->cbs, url, 0, method, &s->header_array, false); stream_state->state_op_done[OP_SEND_INITIAL_METADATA] = true; - if (stream_state->packet_coalescing_enabled && !stream_op->send_message && - !stream_op->send_trailing_metadata) { +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + if (!stream_op->send_message && !stream_op->send_trailing_metadata) { s->state.flush_cronet_when_ready = true; } +#endif result = ACTION_TAKEN_WITH_CALLBACK; } else if (stream_op->send_message && op_can_be_run(stream_op, stream_state, &oas->state, @@ -911,18 +913,19 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, (int)write_buffer_size, false); - if (stream_state->packet_coalescing_enabled) { - if (!stream_op->send_trailing_metadata) { - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); - bidirectional_stream_flush(s->cbs); - result = ACTION_TAKEN_WITH_CALLBACK; - } else { - stream_state->pending_write_for_trailer = true; - result = ACTION_TAKEN_NO_CALLBACK; - } - } else { +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + if (!stream_op->send_trailing_metadata) { + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", + s->cbs); + bidirectional_stream_flush(s->cbs); result = ACTION_TAKEN_WITH_CALLBACK; + } else { + stream_state->pending_write_for_trailer = true; + result = ACTION_TAKEN_NO_CALLBACK; } +#else + result = ACTION_TAKEN_WITH_CALLBACK; +#endif } else { result = NO_ACTION_POSSIBLE; } @@ -941,10 +944,10 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, s->cbs); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, "", 0, true); - if (stream_state->packet_coalescing_enabled) { - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); - bidirectional_stream_flush(s->cbs); - } +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); + bidirectional_stream_flush(s->cbs); +#endif result = ACTION_TAKEN_WITH_CALLBACK; } stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true; @@ -1173,9 +1176,10 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, sizeof(s->state.state_callback_received)); s->state.fail_state = s->state.flush_read = false; s->state.cancel_error = NULL; +#ifdef GRPC_CRONET_WITH_PACKET_COALESCING s->state.flush_cronet_when_ready = s->state.pending_write_for_trailer = false; +#endif s->state.unprocessed_send_message = false; - s->state.packet_coalescing_enabled = grpc_cronet_packet_coalescing_enabled; gpr_mu_init(&s->mu); return 0; } diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m index 9bbf3cdb113..dcd7f2fa8db 100644 --- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m +++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m @@ -269,9 +269,7 @@ unsigned int parse_h2_length(const char *field) { grpc_completion_queue_destroy(cq); } -- (void)PacketCoalescing:(bool)use_coalescing { - grpc_cronet_use_packet_coalescing(use_coalescing); - +- (void)testPacketCoalescing { grpc_call *c; grpc_slice request_payload_slice = grpc_slice_from_copied_string("hello world"); @@ -381,7 +379,7 @@ unsigned int parse_h2_length(const char *field) { long len; bool coalesced = false; while ((len = SSL_read(ssl, buf, sizeof(buf))) > 0) { - gpr_log(GPR_DEBUG, "Read len: %ld", len); + NSLog(@"Read len: %ld", len); // Analyze the HTTP/2 frames in the same TLS PDU to identify if // coalescing is successful @@ -406,7 +404,7 @@ unsigned int parse_h2_length(const char *field) { } } - XCTAssert(coalesced == use_coalescing); + XCTAssert(coalesced); SSL_free(ssl); SSL_CTX_free(ctx); close(s); @@ -435,9 +433,4 @@ unsigned int parse_h2_length(const char *field) { grpc_completion_queue_destroy(cq); } -- (void)testPacketCoalescing { - [self PacketCoalescing:false]; - [self PacketCoalescing:true]; -} - @end diff --git a/src/objective-c/tests/Podfile b/src/objective-c/tests/Podfile index 3760330be9a..462c6a8e0eb 100644 --- a/src/objective-c/tests/Podfile +++ b/src/objective-c/tests/Podfile @@ -97,6 +97,7 @@ post_install do |installer| # GPR_UNREACHABLE_CODE causes "Control may reach end of non-void # function" warning config.build_settings['GCC_WARN_ABOUT_RETURN_TYPE'] = 'NO' + config.build_settings['GCC_PREPROCESSOR_DEFINITIONS'] = '$(inherited) COCOAPODS=1 GRPC_CRONET_WITH_PACKET_COALESCING=1' end end From 191d52c4c1246b7fb80a901186fa39f101738815 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 1 Feb 2017 12:46:05 -0800 Subject: [PATCH 026/167] Renamed define for better readability --- test/core/end2end/bad_server_response_test.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/core/end2end/bad_server_response_test.c b/test/core/end2end/bad_server_response_test.c index 9cfe0cebe01..39a98e84ca9 100644 --- a/test/core/end2end/bad_server_response_test.c +++ b/test/core/end2end/bad_server_response_test.c @@ -82,7 +82,9 @@ #define HTTP1_DETAIL_MSG "Trying to connect an http1.x server" /* TODO(zyc) Check the content of incomming data instead of using this length */ -#define EXPECTED_INCOMING_DATA_LENGTH (size_t)300 +/* The 'bad' server will start sending responses after reading this amount of + * data from the client. */ +#define SERVER_INCOMING_DATA_LENGTH_LOWER_THRESHOLD (size_t)200 struct rpc_state { char *target; @@ -134,8 +136,10 @@ static void handle_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { } gpr_log(GPR_DEBUG, "got %" PRIuPTR " bytes, expected %" PRIuPTR " bytes", - state.incoming_data_length, EXPECTED_INCOMING_DATA_LENGTH); - if (state.incoming_data_length >= EXPECTED_INCOMING_DATA_LENGTH) { + state.incoming_data_length, + SERVER_INCOMING_DATA_LENGTH_LOWER_THRESHOLD); + if (state.incoming_data_length >= + SERVER_INCOMING_DATA_LENGTH_LOWER_THRESHOLD) { handle_write(exec_ctx); } else { grpc_endpoint_read(exec_ctx, state.tcp, &state.temp_incoming_buffer, From 0a2fae9aedeef98b39c5f825cb28b5ca32ecc6a2 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 1 Feb 2017 14:49:03 -0800 Subject: [PATCH 027/167] Dynamically enable packet coalescing by channel args --- BUILD | 1 + build.yaml | 1 + gRPC-Core.podspec | 3 +- include/grpc/impl/codegen/grpc_types.h | 4 + .../client/secure/cronet_channel_create.c | 13 +- .../cronet/transport/cronet_transport.c | 181 +++++++++++------- .../cronet/transport/cronet_transport.h | 43 +++++ .../tests/CronetUnitTests/CronetUnitTests.m | 20 +- src/objective-c/tests/Podfile | 1 - templates/gRPC-Core.podspec.template | 3 +- .../generated/sources_and_headers.json | 4 +- 11 files changed, 186 insertions(+), 88 deletions(-) create mode 100644 src/core/ext/transport/cronet/transport/cronet_transport.h diff --git a/BUILD b/BUILD index 54192514cc7..e5c2d20d25f 100644 --- a/BUILD +++ b/BUILD @@ -1028,6 +1028,7 @@ grpc_cc_library( ], hdrs = [ "third_party/Cronet/bidirectional_stream_c.h", + "src/core/ext/transport/cronet/transport/cronet_transport.h", ], language = "c", public_hdrs = [ diff --git a/build.yaml b/build.yaml index 23e2659ea1e..63c05b76c6c 100644 --- a/build.yaml +++ b/build.yaml @@ -684,6 +684,7 @@ filegroups: - include/grpc/grpc_security.h - include/grpc/grpc_security_constants.h headers: + - src/core/ext/transport/cronet/transport/cronet_transport.h - third_party/Cronet/bidirectional_stream_c.h src: - src/core/ext/transport/cronet/client/secure/cronet_channel_create.c diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 1eb178931dc..57816d066d9 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -848,7 +848,8 @@ Pod::Spec.new do |s| s.subspec 'Cronet-Interface' do |ss| ss.header_mappings_dir = 'include/grpc' - ss.source_files = 'include/grpc/grpc_cronet.h' + ss.source_files = 'include/grpc/grpc_cronet.h', + 'src/core/ext/transport/cronet/transport/cronet_transport.h' end s.subspec 'Cronet-Implementation' do |ss| diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index ee8101aab8d..74be7d0ccfc 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -217,6 +217,10 @@ typedef struct { #define GRPC_ARG_LB_POLICY_NAME "grpc.lb_policy_name" /** The grpc_socket_mutator instance that set the socket options. A pointer. */ #define GRPC_ARG_SOCKET_MUTATOR "grpc.socket_mutator" +/** If non-zero, Cronet transport will coalesce packets to fewer frames when + * possible. */ +#define GRPC_ARG_USE_CRONET_PACKET_COALESCING \ + "grpc.use_cronet_packet_coalescing" /** \} */ /** Result of a grpc call. If the caller satisfies the prerequisites of a diff --git a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c index 477cf07f45d..b6e9e845df3 100644 --- a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c +++ b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c @@ -39,6 +39,7 @@ #include #include +#include "src/core/ext/transport/cronet/transport/cronet_transport.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/transport/transport_impl.h" @@ -54,16 +55,14 @@ extern grpc_transport_vtable grpc_cronet_vtable; GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( void *engine, const char *target, const grpc_channel_args *args, void *reserved) { - cronet_transport *ct = gpr_malloc(sizeof(cronet_transport)); - ct->base.vtable = &grpc_cronet_vtable; - ct->engine = engine; - ct->host = gpr_malloc(strlen(target) + 1); - strcpy(ct->host, target); gpr_log(GPR_DEBUG, "grpc_create_cronet_transport: stream_engine = %p, target=%s", engine, - ct->host); + target); + + grpc_transport *ct = + grpc_create_cronet_transport(engine, target, args, reserved); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; return grpc_channel_create(&exec_ctx, target, args, - GRPC_CLIENT_DIRECT_CHANNEL, (grpc_transport *)ct); + GRPC_CLIENT_DIRECT_CHANNEL, ct); } diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 447f3f31ec1..cba5365badc 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -112,6 +112,7 @@ struct grpc_cronet_transport { grpc_transport base; /* must be first element in this structure */ stream_engine *engine; char *host; + bool use_packet_coalescing; }; typedef struct grpc_cronet_transport grpc_cronet_transport; @@ -150,10 +151,8 @@ struct op_state { bool state_callback_received[OP_NUM_OPS]; bool fail_state; bool flush_read; -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING bool flush_cronet_when_ready; bool pending_write_for_trailer; -#endif bool unprocessed_send_message; grpc_error *cancel_error; /* data structure for storing data coming from server */ @@ -178,7 +177,7 @@ struct op_storage { struct stream_obj { struct op_and_state *oas; grpc_transport_stream_op *curr_op; - grpc_cronet_transport curr_ct; + grpc_cronet_transport *curr_ct; grpc_stream *curr_gs; bidirectional_stream *cbs; bidirectional_stream_header_array header_array; @@ -415,6 +414,7 @@ static void on_succeeded(bidirectional_stream *stream) { static void on_stream_ready(bidirectional_stream *stream) { CRONET_LOG(GPR_DEBUG, "W: on_stream_ready(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; + grpc_cronet_transport *t = (grpc_cronet_transport *)s->curr_ct; gpr_mu_lock(&s->mu); s->state.state_op_done[OP_SEND_INITIAL_METADATA] = true; s->state.state_callback_received[OP_SEND_INITIAL_METADATA] = true; @@ -425,12 +425,12 @@ static void on_stream_ready(bidirectional_stream *stream) { } /* Send the initial metadata on wire if there is no SEND_MESSAGE or * SEND_TRAILING_METADATA ops pending */ -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - if (s->state.flush_cronet_when_ready) { - CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_flush (%p)", s->cbs); - bidirectional_stream_flush(stream); + if (t->use_packet_coalescing) { + if (s->state.flush_cronet_when_ready) { + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_flush (%p)", s->cbs); + bidirectional_stream_flush(stream); + } } -#endif gpr_mu_unlock(&s->mu); execute_from_storage(s); } @@ -540,6 +540,7 @@ static void on_response_trailers_received( CRONET_LOG(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, trailers); stream_obj *s = (stream_obj *)stream->annotation; + grpc_cronet_transport *t = (grpc_cronet_transport *)s->curr_ct; gpr_mu_lock(&s->mu); memset(&s->state.rs.trailing_metadata, 0, sizeof(s->state.rs.trailing_metadata)); @@ -568,10 +569,10 @@ static void on_response_trailers_received( CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, 0)", s->cbs); s->state.state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, "", 0, true); -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); - bidirectional_stream_flush(s->cbs); -#endif + if (t->use_packet_coalescing) { + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); + bidirectional_stream_flush(s->cbs); + } s->state.state_op_done[OP_SEND_TRAILING_METADATA] = true; gpr_mu_unlock(&s->mu); @@ -695,8 +696,10 @@ static bool header_has_authority(grpc_linked_mdelem *head) { executed. This is the heart of the state machine. */ static bool op_can_be_run(grpc_transport_stream_op *curr_op, - struct op_state *stream_state, - struct op_state *op_state, enum e_op_id op_id) { + struct stream_obj *s, struct op_state *op_state, + enum e_op_id op_id) { + struct op_state *stream_state = &s->state; + grpc_cronet_transport *t = s->curr_ct; bool result = true; /* When call is canceled, every op can be run, except under following conditions @@ -768,11 +771,9 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, result = false; /* we haven't got on_write_completed for the send yet */ else if (stream_state->state_op_done[OP_SEND_MESSAGE] && - !stream_state->state_callback_received[OP_SEND_MESSAGE] -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - && !stream_state->pending_write_for_trailer -#endif - ) + !stream_state->state_callback_received[OP_SEND_MESSAGE] && + !(t->use_packet_coalescing && + stream_state->pending_write_for_trailer)) result = false; } else if (op_id == OP_CANCEL_ERROR) { /* already executed */ @@ -845,42 +846,41 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, struct op_and_state *oas) { grpc_transport_stream_op *stream_op = &oas->op; struct stream_obj *s = oas->s; + grpc_cronet_transport *t = (grpc_cronet_transport *)s->curr_ct; struct op_state *stream_state = &s->state; enum e_op_result result = NO_ACTION_POSSIBLE; if (stream_op->send_initial_metadata && - op_can_be_run(stream_op, stream_state, &oas->state, - OP_SEND_INITIAL_METADATA)) { + op_can_be_run(stream_op, s, &oas->state, OP_SEND_INITIAL_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_INITIAL_METADATA", oas); /* Start new cronet stream. It is destroyed in on_succeeded, on_canceled, * on_failed */ GPR_ASSERT(s->cbs == NULL); GPR_ASSERT(!stream_state->state_op_done[OP_SEND_INITIAL_METADATA]); - s->cbs = bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, + s->cbs = bidirectional_stream_create(t->engine, s->curr_gs, &cronet_callbacks); CRONET_LOG(GPR_DEBUG, "%p = bidirectional_stream_create()", s->cbs); -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - bidirectional_stream_disable_auto_flush(s->cbs, true); - bidirectional_stream_delay_request_headers_until_flush(s->cbs, true); -#endif + if (t->use_packet_coalescing) { + bidirectional_stream_disable_auto_flush(s->cbs, true); + bidirectional_stream_delay_request_headers_until_flush(s->cbs, true); + } char *url = NULL; const char *method = "POST"; s->header_array.headers = NULL; convert_metadata_to_cronet_headers( - stream_op->send_initial_metadata->list.head, s->curr_ct.host, &url, + stream_op->send_initial_metadata->list.head, t->host, &url, &s->header_array.headers, &s->header_array.count, &method); s->header_array.capacity = s->header_array.count; CRONET_LOG(GPR_DEBUG, "bidirectional_stream_start(%p, %s)", s->cbs, url); bidirectional_stream_start(s->cbs, url, 0, method, &s->header_array, false); stream_state->state_op_done[OP_SEND_INITIAL_METADATA] = true; -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - if (!stream_op->send_message && !stream_op->send_trailing_metadata) { - s->state.flush_cronet_when_ready = true; + if (t->use_packet_coalescing) { + if (!stream_op->send_message && !stream_op->send_trailing_metadata) { + s->state.flush_cronet_when_ready = true; + } } -#endif result = ACTION_TAKEN_WITH_CALLBACK; } else if (stream_op->send_message && - op_can_be_run(stream_op, stream_state, &oas->state, - OP_SEND_MESSAGE)) { + op_can_be_run(stream_op, s, &oas->state, OP_SEND_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_MESSAGE", oas); stream_state->unprocessed_send_message = false; if (stream_state->state_callback_received[OP_FAILED]) { @@ -913,19 +913,18 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, (int)write_buffer_size, false); -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - if (!stream_op->send_trailing_metadata) { - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", - s->cbs); - bidirectional_stream_flush(s->cbs); - result = ACTION_TAKEN_WITH_CALLBACK; + if (t->use_packet_coalescing) { + if (!stream_op->send_trailing_metadata) { + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); + bidirectional_stream_flush(s->cbs); + result = ACTION_TAKEN_WITH_CALLBACK; + } else { + stream_state->pending_write_for_trailer = true; + result = ACTION_TAKEN_NO_CALLBACK; + } } else { - stream_state->pending_write_for_trailer = true; - result = ACTION_TAKEN_NO_CALLBACK; + result = ACTION_TAKEN_WITH_CALLBACK; } -#else - result = ACTION_TAKEN_WITH_CALLBACK; -#endif } else { result = NO_ACTION_POSSIBLE; } @@ -933,7 +932,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->state_op_done[OP_SEND_MESSAGE] = true; oas->state.state_op_done[OP_SEND_MESSAGE] = true; } else if (stream_op->send_trailing_metadata && - op_can_be_run(stream_op, stream_state, &oas->state, + op_can_be_run(stream_op, s, &oas->state, OP_SEND_TRAILING_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_TRAILING_METADATA", oas); if (stream_state->state_callback_received[OP_FAILED]) { @@ -944,15 +943,15 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, s->cbs); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, "", 0, true); -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); - bidirectional_stream_flush(s->cbs); -#endif + if (t->use_packet_coalescing) { + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); + bidirectional_stream_flush(s->cbs); + } result = ACTION_TAKEN_WITH_CALLBACK; } stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true; } else if (stream_op->recv_initial_metadata && - op_can_be_run(stream_op, stream_state, &oas->state, + op_can_be_run(stream_op, s, &oas->state, OP_RECV_INITIAL_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_INITIAL_METADATA", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { @@ -971,8 +970,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; } else if (stream_op->recv_message && - op_can_be_run(stream_op, stream_state, &oas->state, - OP_RECV_MESSAGE)) { + op_can_be_run(stream_op, s, &oas->state, OP_RECV_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_MESSAGE", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { CRONET_LOG(GPR_DEBUG, "Stream is cancelled."); @@ -1084,7 +1082,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_op->recv_trailing_metadata && - op_can_be_run(stream_op, stream_state, &oas->state, + op_can_be_run(stream_op, s, &oas->state, OP_RECV_TRAILING_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_TRAILING_METADATA", oas); if (oas->s->state.rs.trailing_metadata_valid) { @@ -1096,8 +1094,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->state_op_done[OP_RECV_TRAILING_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; } else if (stream_op->cancel_error && - op_can_be_run(stream_op, stream_state, &oas->state, - OP_CANCEL_ERROR)) { + op_can_be_run(stream_op, s, &oas->state, OP_CANCEL_ERROR)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_CANCEL_ERROR", oas); CRONET_LOG(GPR_DEBUG, "W: bidirectional_stream_cancel(%p)", s->cbs); if (s->cbs) { @@ -1111,8 +1108,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->cancel_error = GRPC_ERROR_REF(stream_op->cancel_error); } } else if (stream_op->on_complete && - op_can_be_run(stream_op, stream_state, &oas->state, - OP_ON_COMPLETE)) { + op_can_be_run(stream_op, s, &oas->state, OP_ON_COMPLETE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_ON_COMPLETE", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { grpc_closure_sched(exec_ctx, stream_op->on_complete, @@ -1176,10 +1172,12 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, sizeof(s->state.state_callback_received)); s->state.fail_state = s->state.flush_read = false; s->state.cancel_error = NULL; -#ifdef GRPC_CRONET_WITH_PACKET_COALESCING s->state.flush_cronet_when_ready = s->state.pending_write_for_trailer = false; -#endif s->state.unprocessed_send_message = false; + + s->curr_gs = gs; + s->curr_ct = (grpc_cronet_transport *)gt; + gpr_mu_init(&s->mu); return 0; } @@ -1195,8 +1193,6 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_transport_stream_op *op) { CRONET_LOG(GPR_DEBUG, "perform_stream_op"); stream_obj *s = (stream_obj *)gs; - s->curr_gs = gs; - memcpy(&s->curr_ct, gt, sizeof(grpc_cronet_transport)); add_to_storage(s, op); if (op->send_initial_metadata && header_has_authority(op->send_initial_metadata->list.head)) { @@ -1244,14 +1240,55 @@ static grpc_endpoint *get_endpoint(grpc_exec_ctx *exec_ctx, static void perform_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_transport_op *op) {} -const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj), - "cronet_http", - init_stream, - set_pollset_do_nothing, - set_pollset_set_do_nothing, - perform_stream_op, - perform_op, - destroy_stream, - destroy_transport, - get_peer, - get_endpoint}; +static const grpc_transport_vtable grpc_cronet_vtable = { + sizeof(stream_obj), + "cronet_http", + init_stream, + set_pollset_do_nothing, + set_pollset_set_do_nothing, + perform_stream_op, + perform_op, + destroy_stream, + destroy_transport, + get_peer, + get_endpoint}; + +grpc_transport *grpc_create_cronet_transport(void *engine, const char *target, + const grpc_channel_args *args, + void *reserved) { + grpc_cronet_transport *ct = gpr_malloc(sizeof(grpc_cronet_transport)); + if (!ct) { + goto error; + } + ct->base.vtable = &grpc_cronet_vtable; + ct->engine = engine; + ct->host = gpr_malloc(strlen(target) + 1); + if (!ct->host) { + goto error; + } + strcpy(ct->host, target); + + ct->use_packet_coalescing = true; + for (size_t i = 0; i < args->num_args; i++) { + if (0 == strcmp(args->args[i].key, GRPC_ARG_USE_CRONET_PACKET_COALESCING)) { + if (args->args[i].type != GRPC_ARG_INTEGER) { + gpr_log(GPR_ERROR, "%s ignored: it must be an integer", + GRPC_ARG_USE_CRONET_PACKET_COALESCING); + } else { + ct->use_packet_coalescing = (args->args[i].value.integer != 0); + } + } + } + + return &ct->base; + +error: + if (ct) { + if (ct->host) { + gpr_free(ct->host); + } + gpr_free(ct); + } + + return NULL; +} \ No newline at end of file diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.h b/src/core/ext/transport/cronet/transport/cronet_transport.h new file mode 100644 index 00000000000..169ce31fd7d --- /dev/null +++ b/src/core/ext/transport/cronet/transport/cronet_transport.h @@ -0,0 +1,43 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_TRANSPORT_CRONET_TRANSPORT_CRONET_TRANSPORT_H +#define GRPC_CORE_EXT_TRANSPORT_CRONET_TRANSPORT_CRONET_TRANSPORT_H + +#include "src/core/lib/transport/transport.h" + +grpc_transport *grpc_create_cronet_transport(void *engine, const char *target, + const grpc_channel_args *args, + void *reserved); + +#endif /* GRPC_CORE_EXT_TRANSPORT_CRONET_TRANSPORT_CRONET_TRANSPORT_H */ diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m index dcd7f2fa8db..d06fe7767fa 100644 --- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m +++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m @@ -269,7 +269,12 @@ unsigned int parse_h2_length(const char *field) { grpc_completion_queue_destroy(cq); } -- (void)testPacketCoalescing { +- (void)PacketCoalescing:(bool)use_coalescing { + grpc_arg arg; + arg.key = GRPC_ARG_USE_CRONET_PACKET_COALESCING; + arg.type = GRPC_ARG_INTEGER; + arg.value.integer = use_coalescing ? 1:0; + grpc_channel_args *args = grpc_channel_args_copy_and_add(NULL, &arg, 1); grpc_call *c; grpc_slice request_payload_slice = grpc_slice_from_copied_string("hello world"); @@ -285,8 +290,8 @@ unsigned int parse_h2_length(const char *field) { gpr_join_host_port(&addr, "127.0.0.1", port); grpc_completion_queue *cq = grpc_completion_queue_create(NULL); stream_engine *cronetEngine = [Cronet getGlobalEngine]; - grpc_channel *client = grpc_cronet_secure_channel_create(cronetEngine, addr, - NULL, NULL); + grpc_channel *client = + grpc_cronet_secure_channel_create(cronetEngine, addr, args, NULL); cq_verifier *cqv = cq_verifier_create(cq); grpc_op ops[6]; @@ -379,7 +384,7 @@ unsigned int parse_h2_length(const char *field) { long len; bool coalesced = false; while ((len = SSL_read(ssl, buf, sizeof(buf))) > 0) { - NSLog(@"Read len: %ld", len); + gpr_log(GPR_DEBUG, "Read len: %ld", len); // Analyze the HTTP/2 frames in the same TLS PDU to identify if // coalescing is successful @@ -404,7 +409,7 @@ unsigned int parse_h2_length(const char *field) { } } - XCTAssert(coalesced); + XCTAssert(coalesced == use_coalescing); SSL_free(ssl); SSL_CTX_free(ctx); close(s); @@ -433,4 +438,9 @@ unsigned int parse_h2_length(const char *field) { grpc_completion_queue_destroy(cq); } +- (void)testPacketCoalescing { + [self PacketCoalescing:true]; + [self PacketCoalescing:false]; +} + @end diff --git a/src/objective-c/tests/Podfile b/src/objective-c/tests/Podfile index 462c6a8e0eb..3760330be9a 100644 --- a/src/objective-c/tests/Podfile +++ b/src/objective-c/tests/Podfile @@ -97,7 +97,6 @@ post_install do |installer| # GPR_UNREACHABLE_CODE causes "Control may reach end of non-void # function" warning config.build_settings['GCC_WARN_ABOUT_RETURN_TYPE'] = 'NO' - config.build_settings['GCC_PREPROCESSOR_DEFINITIONS'] = '$(inherited) COCOAPODS=1 GRPC_CRONET_WITH_PACKET_COALESCING=1' end end diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index 1b97d18f163..0738b7221b5 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -161,7 +161,8 @@ s.subspec 'Cronet-Interface' do |ss| ss.header_mappings_dir = 'include/grpc' - ss.source_files = 'include/grpc/grpc_cronet.h' + ss.source_files = 'include/grpc/grpc_cronet.h', + 'src/core/ext/transport/cronet/transport/cronet_transport.h' end s.subspec 'Cronet-Implementation' do |ss| diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 9bc82486d2b..7f594110197 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7873,6 +7873,7 @@ "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", + "src/core/ext/transport/cronet/transport/cronet_transport.h", "third_party/Cronet/bidirectional_stream_c.h" ], "is_filegroup": true, @@ -7884,7 +7885,8 @@ "include/grpc/grpc_security_constants.h", "src/core/ext/transport/cronet/client/secure/cronet_channel_create.c", "src/core/ext/transport/cronet/transport/cronet_api_dummy.c", - "src/core/ext/transport/cronet/transport/cronet_transport.c" + "src/core/ext/transport/cronet/transport/cronet_transport.c", + "src/core/ext/transport/cronet/transport/cronet_transport.h" ], "third_party": false, "type": "filegroup" From 1a5f90fa8a1a9c647e151575fa9651ae1715045f Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 1 Feb 2017 15:46:21 -0800 Subject: [PATCH 028/167] Fix and clang-format Cronet unit tests --- .../tests/CronetUnitTests/CronetUnitTests.m | 283 +++++++++--------- 1 file changed, 146 insertions(+), 137 deletions(-) diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m index a50bf4728f8..9589926d8b5 100644 --- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m +++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m @@ -32,13 +32,13 @@ */ #import -#import #import +#import #import -#import -#import #import +#import +#import #import "test/core/end2end/cq_verifier.h" #import "test/core/util/port.h" @@ -49,19 +49,19 @@ #import "src/core/lib/support/env.h" #import "src/core/lib/support/string.h" #import "src/core/lib/support/tmpfile.h" -#import "test/core/util/test_config.h" #import "test/core/end2end/data/ssl_test_data.h" +#import "test/core/util/test_config.h" #import static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, grpc_timeout_seconds_to_deadline(5), NULL); + ev = grpc_completion_queue_next(cq, grpc_timeout_seconds_to_deadline(5), + NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } - @interface CronetUnitTests : XCTestCase @end @@ -71,8 +71,8 @@ static void drain_cq(grpc_completion_queue *cq) { + (void)setUp { [super setUp]; -/*** FILE *roots_file; - size_t roots_size = strlen(test_root_cert);*/ + /*** FILE *roots_file; + size_t roots_size = strlen(test_root_cert);*/ char *argv[] = {"CoreCronetEnd2EndTests"}; grpc_test_init(1, argv); @@ -83,8 +83,8 @@ static void drain_cq(grpc_completion_queue *cq) { [Cronet setSslKeyLogFileName:@"Documents/key"]; [Cronet enableTestCertVerifierForTesting]; NSURL *url = [[[NSFileManager defaultManager] - URLsForDirectory:NSDocumentDirectory - inDomains:NSUserDomainMask] lastObject]; + URLsForDirectory:NSDocumentDirectory + inDomains:NSUserDomainMask] lastObject]; NSLog(@"Documents directory: %@", url); [Cronet start]; [Cronet startNetLogToFile:@"Documents/cronet_netlog.json" logBytes:YES]; @@ -104,28 +104,28 @@ void init_ssl(void) { OpenSSL_add_ssl_algorithms(); } -void cleanup_ssl(void) { - EVP_cleanup(); -} +void cleanup_ssl(void) { EVP_cleanup(); } int alpn_cb(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg) { // Always select "h2" as the ALPN protocol to be used - *out = (const unsigned char*)"h2"; + *out = (const unsigned char *)"h2"; *outlen = 2; return SSL_TLSEXT_ERR_OK; } void init_ctx(SSL_CTX *ctx) { // Install server certificate - BIO *pem = BIO_new_mem_buf((void*)test_server1_cert, (int)strlen(test_server1_cert)); + BIO *pem = BIO_new_mem_buf((void *)test_server1_cert, + (int)strlen(test_server1_cert)); X509 *cert = PEM_read_bio_X509_AUX(pem, NULL, NULL, ""); SSL_CTX_use_certificate(ctx, cert); X509_free(cert); BIO_free(pem); // Install server private key - pem = BIO_new_mem_buf((void *)test_server1_key, (int)strlen(test_server1_key)); + pem = + BIO_new_mem_buf((void *)test_server1_key, (int)strlen(test_server1_key)); EVP_PKEY *key = PEM_read_bio_PrivateKey(pem, NULL, NULL, ""); SSL_CTX_use_PrivateKey(ctx, key); EVP_PKEY_free(key); @@ -139,29 +139,34 @@ void init_ctx(SSL_CTX *ctx) { } unsigned int parse_h2_length(const char *field) { - return ((unsigned int)(unsigned char)(field[0])) * 65536 + - ((unsigned int)(unsigned char)(field[1])) * 256 + - ((unsigned int)(unsigned char)(field[2])); + return ((unsigned int)(unsigned char)(field[0])) * 65536 + + ((unsigned int)(unsigned char)(field[1])) * 256 + + ((unsigned int)(unsigned char)(field[2])); } - (void)testInternalError { grpc_call *c; grpc_slice request_payload_slice = - grpc_slice_from_copied_string("hello world"); + grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = - grpc_raw_byte_buffer_create(&request_payload_slice, 1); + grpc_raw_byte_buffer_create(&request_payload_slice, 1); gpr_timespec deadline = grpc_timeout_seconds_to_deadline(5); - grpc_metadata meta_c[2] = { - {"key1", "val1", 4, 0, {{NULL, NULL, NULL, NULL}}}, - {"key2", "val2", 4, 0, {{NULL, NULL, NULL, NULL}}}}; + grpc_metadata meta_c[2] = {{grpc_slice_from_static_string("key1"), + grpc_slice_from_static_string("val1"), + 0, + {{NULL, NULL, NULL, NULL}}}, + {grpc_slice_from_static_string("key2"), + grpc_slice_from_static_string("val2"), + 0, + {{NULL, NULL, NULL, NULL}}}}; int port = grpc_pick_unused_port_or_die(); char *addr; gpr_join_host_port(&addr, "127.0.0.1", port); grpc_completion_queue *cq = grpc_completion_queue_create(NULL); stream_engine *cronetEngine = [Cronet getGlobalEngine]; - grpc_channel *client = grpc_cronet_secure_channel_create(cronetEngine, addr, - NULL, NULL); + grpc_channel *client = + grpc_cronet_secure_channel_create(cronetEngine, addr, NULL, NULL); cq_verifier *cqv = cq_verifier_create(cq); grpc_op ops[6]; @@ -173,12 +178,11 @@ unsigned int parse_h2_length(const char *field) { grpc_call_details call_details; grpc_status_code status; grpc_call_error error; - char *details = NULL; - size_t details_capacity = 0; + grpc_slice details; - c = grpc_channel_create_call( - client, NULL, GRPC_PROPAGATE_DEFAULTS, cq, "/foo", - NULL, deadline, NULL); + c = grpc_channel_create_call(client, NULL, GRPC_PROPAGATE_DEFAULTS, cq, + grpc_slice_from_static_string("/foo"), NULL, + deadline, NULL); GPR_ASSERT(c); grpc_metadata_array_init(&initial_metadata_recv); @@ -217,40 +221,40 @@ unsigned int parse_h2_length(const char *field) { op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; op->data.recv_status_on_client.status = &status; op->data.recv_status_on_client.status_details = &details; - op->data.recv_status_on_client.status_details_capacity = &details_capacity; op->flags = 0; op->reserved = NULL; op++; - error = grpc_call_start_batch(c, ops, (size_t)(op - ops), (void*)1, NULL); + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), (void *)1, NULL); GPR_ASSERT(GRPC_CALL_OK == error); - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - int sl = socket(AF_INET, SOCK_STREAM, 0); - GPR_ASSERT(sl >= 0); - - // Make and TCP endpoint to accept the connection - struct sockaddr_in s_addr; - memset(&s_addr, 0, sizeof(s_addr)); - s_addr.sin_family = AF_INET; - s_addr.sin_addr.s_addr = htonl(INADDR_ANY); - s_addr.sin_port = htons(port); - GPR_ASSERT(0 == bind(sl, (struct sockaddr*)&s_addr, sizeof(s_addr))); - GPR_ASSERT(0 == listen(sl, 5)); - int s = accept(sl, NULL, NULL); - GPR_ASSERT(s >= 0); - - // Close the connection after 1 second to trigger Cronet's on_failed() - sleep(1); - close(s); - close(sl); - }); - - CQ_EXPECT_COMPLETION(cqv, (void*)1, 1); + dispatch_async( + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + int sl = socket(AF_INET, SOCK_STREAM, 0); + GPR_ASSERT(sl >= 0); + + // Make and TCP endpoint to accept the connection + struct sockaddr_in s_addr; + memset(&s_addr, 0, sizeof(s_addr)); + s_addr.sin_family = AF_INET; + s_addr.sin_addr.s_addr = htonl(INADDR_ANY); + s_addr.sin_port = htons(port); + GPR_ASSERT(0 == bind(sl, (struct sockaddr *)&s_addr, sizeof(s_addr))); + GPR_ASSERT(0 == listen(sl, 5)); + int s = accept(sl, NULL, NULL); + GPR_ASSERT(s >= 0); + + // Close the connection after 1 second to trigger Cronet's on_failed() + sleep(1); + close(s); + close(sl); + }); + + CQ_EXPECT_COMPLETION(cqv, (void *)1, 1); cq_verify(cqv); GPR_ASSERT(status == GRPC_STATUS_UNAVAILABLE); - gpr_free(details); + grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); grpc_metadata_array_destroy(&trailing_metadata_recv); grpc_metadata_array_destroy(&request_metadata_recv); @@ -262,7 +266,7 @@ unsigned int parse_h2_length(const char *field) { grpc_byte_buffer_destroy(request_payload); grpc_byte_buffer_destroy(response_payload_recv); - + grpc_channel_destroy(client); grpc_completion_queue_shutdown(cq); drain_cq(cq); @@ -273,17 +277,22 @@ unsigned int parse_h2_length(const char *field) { grpc_arg arg; arg.key = GRPC_ARG_USE_CRONET_PACKET_COALESCING; arg.type = GRPC_ARG_INTEGER; - arg.value.integer = use_coalescing ? 1:0; + arg.value.integer = use_coalescing ? 1 : 0; grpc_channel_args *args = grpc_channel_args_copy_and_add(NULL, &arg, 1); grpc_call *c; grpc_slice request_payload_slice = - grpc_slice_from_copied_string("hello world"); + grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = - grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5); - grpc_metadata meta_c[2] = { - {"key1", "val1", 4, 0, {{NULL, NULL, NULL, NULL}}}, - {"key2", "val2", 4, 0, {{NULL, NULL, NULL, NULL}}}}; + grpc_raw_byte_buffer_create(&request_payload_slice, 1); + gpr_timespec deadline = grpc_timeout_seconds_to_deadline(5); + grpc_metadata meta_c[2] = {{grpc_slice_from_static_string("key1"), + grpc_slice_from_static_string("val1"), + 0, + {{NULL, NULL, NULL, NULL}}}, + {grpc_slice_from_static_string("key2"), + grpc_slice_from_static_string("val2"), + 0, + {{NULL, NULL, NULL, NULL}}}}; int port = grpc_pick_unused_port_or_die(); char *addr; @@ -303,12 +312,11 @@ unsigned int parse_h2_length(const char *field) { grpc_call_details call_details; grpc_status_code status; grpc_call_error error; - char *details = NULL; - size_t details_capacity = 0; + grpc_slice details; - c = grpc_channel_create_call( - client, NULL, GRPC_PROPAGATE_DEFAULTS, cq, "/foo", - NULL, deadline, NULL); + c = grpc_channel_create_call(client, NULL, GRPC_PROPAGATE_DEFAULTS, cq, + grpc_slice_from_static_string("/foo"), NULL, + deadline, NULL); GPR_ASSERT(c); grpc_metadata_array_init(&initial_metadata_recv); @@ -325,7 +333,7 @@ unsigned int parse_h2_length(const char *field) { op->reserved = NULL; op++; op->op = GRPC_OP_SEND_MESSAGE; - op->data.send_message = request_payload; + op->data.send_message.send_message = request_payload; op->flags = 0; op->reserved = NULL; op++; @@ -334,12 +342,12 @@ unsigned int parse_h2_length(const char *field) { op->reserved = NULL; op++; op->op = GRPC_OP_RECV_INITIAL_METADATA; - op->data.recv_initial_metadata = &initial_metadata_recv; + op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; op->flags = 0; op->reserved = NULL; op++; op->op = GRPC_OP_RECV_MESSAGE; - op->data.recv_message = &response_payload_recv; + op->data.recv_message.recv_message = &response_payload_recv; op->flags = 0; op->reserved = NULL; op++; @@ -347,79 +355,80 @@ unsigned int parse_h2_length(const char *field) { op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; op->data.recv_status_on_client.status = &status; op->data.recv_status_on_client.status_details = &details; - op->data.recv_status_on_client.status_details_capacity = &details_capacity; op->flags = 0; op->reserved = NULL; op++; - error = grpc_call_start_batch(c, ops, (size_t)(op - ops), (void*)1, NULL); + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), (void *)1, NULL); GPR_ASSERT(GRPC_CALL_OK == error); - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - int sl = socket(AF_INET, SOCK_STREAM, 0); - GPR_ASSERT(sl >= 0); - struct sockaddr_in s_addr; - memset(&s_addr, 0, sizeof(s_addr)); - s_addr.sin_family = AF_INET; - s_addr.sin_addr.s_addr = htonl(INADDR_ANY); - s_addr.sin_port = htons(port); - GPR_ASSERT(0 == bind(sl, (struct sockaddr*)&s_addr, sizeof(s_addr))); - GPR_ASSERT(0 == listen(sl, 5)); - int s = accept(sl, NULL, NULL); - GPR_ASSERT(s >= 0); - struct timeval tv; - tv.tv_sec = 2; - tv.tv_usec = 0; - setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); - - // Make an TLS endpoint to receive Cronet's transmission - SSL_CTX *ctx = SSL_CTX_new(TLSv1_2_server_method()); - init_ctx(ctx); - SSL *ssl = SSL_new(ctx); - SSL_set_fd(ssl, s); - SSL_accept(ssl); - - const char magic[] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; - - char buf[4096]; - long len; - bool coalesced = false; - while ((len = SSL_read(ssl, buf, sizeof(buf))) > 0) { - gpr_log(GPR_DEBUG, "Read len: %ld", len); - - // Analyze the HTTP/2 frames in the same TLS PDU to identify if - // coalescing is successful - unsigned int p = 0; - while (p < len) { - if (len - p >= 24 && 0 == memcmp(&buf[p], magic, 24)) { - p += 24; - continue; + dispatch_async( + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + int sl = socket(AF_INET, SOCK_STREAM, 0); + GPR_ASSERT(sl >= 0); + struct sockaddr_in s_addr; + memset(&s_addr, 0, sizeof(s_addr)); + s_addr.sin_family = AF_INET; + s_addr.sin_addr.s_addr = htonl(INADDR_ANY); + s_addr.sin_port = htons(port); + GPR_ASSERT(0 == bind(sl, (struct sockaddr *)&s_addr, sizeof(s_addr))); + GPR_ASSERT(0 == listen(sl, 5)); + int s = accept(sl, NULL, NULL); + GPR_ASSERT(s >= 0); + struct timeval tv; + tv.tv_sec = 2; + tv.tv_usec = 0; + setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + + // Make an TLS endpoint to receive Cronet's transmission + SSL_CTX *ctx = SSL_CTX_new(TLSv1_2_server_method()); + init_ctx(ctx); + SSL *ssl = SSL_new(ctx); + SSL_set_fd(ssl, s); + SSL_accept(ssl); + + const char magic[] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; + + char buf[4096]; + long len; + bool coalesced = false; + while ((len = SSL_read(ssl, buf, sizeof(buf))) > 0) { + gpr_log(GPR_DEBUG, "Read len: %ld", len); + + // Analyze the HTTP/2 frames in the same TLS PDU to identify if + // coalescing is successful + unsigned int p = 0; + while (p < len) { + if (len - p >= 24 && 0 == memcmp(&buf[p], magic, 24)) { + p += 24; + continue; + } + + if (buf[p + 3] == 0 && // Type is DATA + parse_h2_length(&buf[p]) == 0x10 && // Length is correct + (buf[p + 4] & 1) != 0 && // EOS bit is set + 0 == memcmp("hello world", &buf[p + 14], + 11)) { // Message is correct + coalesced = true; + break; + } + p += (parse_h2_length(&buf[p]) + 9); + } + if (coalesced) { + break; + } } - if (buf[p+3] == 0 && // Type is DATA - parse_h2_length(&buf[p]) == 0x10 && // Length is correct - (buf[p+4] & 1) != 0 && // EOS bit is set - 0 == memcmp("hello world", &buf[p+14], 11)) { // Message is correct - coalesced = true; - break; - } - p += (parse_h2_length(&buf[p]) + 9); - } - if (coalesced) { - break; - } - } - - XCTAssert(coalesced == use_coalescing); - SSL_free(ssl); - SSL_CTX_free(ctx); - close(s); - close(sl); - }); - - CQ_EXPECT_COMPLETION(cqv, (void*)1, 1); + XCTAssert(coalesced == use_coalescing); + SSL_free(ssl); + SSL_CTX_free(ctx); + close(s); + close(sl); + }); + + CQ_EXPECT_COMPLETION(cqv, (void *)1, 1); cq_verify(cqv); - gpr_free(details); + grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); grpc_metadata_array_destroy(&trailing_metadata_recv); grpc_metadata_array_destroy(&request_metadata_recv); @@ -431,7 +440,7 @@ unsigned int parse_h2_length(const char *field) { grpc_byte_buffer_destroy(request_payload); grpc_byte_buffer_destroy(response_payload_recv); - + grpc_channel_destroy(client); grpc_completion_queue_shutdown(cq); drain_cq(cq); From 4c94e52f003b9beb999b2a9b73386f4746f55ffc Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 1 Feb 2017 15:46:39 -0800 Subject: [PATCH 029/167] bug fix for creating Cronet transport context --- .../cronet/transport/cronet_transport.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 1be6550bd3a..fac0a7a234f 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -1287,13 +1287,15 @@ grpc_transport *grpc_create_cronet_transport(void *engine, const char *target, strcpy(ct->host, target); ct->use_packet_coalescing = true; - for (size_t i = 0; i < args->num_args; i++) { - if (0 == strcmp(args->args[i].key, GRPC_ARG_USE_CRONET_PACKET_COALESCING)) { - if (args->args[i].type != GRPC_ARG_INTEGER) { - gpr_log(GPR_ERROR, "%s ignored: it must be an integer", - GRPC_ARG_USE_CRONET_PACKET_COALESCING); - } else { - ct->use_packet_coalescing = (args->args[i].value.integer != 0); + if (args) { + for (size_t i = 0; i < args->num_args; i++) { + if (0 == strcmp(args->args[i].key, GRPC_ARG_USE_CRONET_PACKET_COALESCING)) { + if (args->args[i].type != GRPC_ARG_INTEGER) { + gpr_log(GPR_ERROR, "%s ignored: it must be an integer", + GRPC_ARG_USE_CRONET_PACKET_COALESCING); + } else { + ct->use_packet_coalescing = (args->args[i].value.integer != 0); + } } } } From 2ee9751cbbe57d2b4bfece8926d8b3112388c4eb Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 1 Feb 2017 16:00:50 -0800 Subject: [PATCH 030/167] clang-format --- .../cronet/transport/cronet_transport.c | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index fac0a7a234f..01a03533daf 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -425,8 +425,8 @@ static void on_stream_ready(bidirectional_stream *stream) { gpr_free(s->header_array.headers); s->header_array.headers = NULL; } -/* Send the initial metadata on wire if there is no SEND_MESSAGE or - * SEND_TRAILING_METADATA ops pending */ + /* Send the initial metadata on wire if there is no SEND_MESSAGE or + * SEND_TRAILING_METADATA ops pending */ if (t->use_packet_coalescing) { if (s->state.flush_cronet_when_ready) { CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_flush (%p)", s->cbs); @@ -868,8 +868,8 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, * on_failed */ GPR_ASSERT(s->cbs == NULL); GPR_ASSERT(!stream_state->state_op_done[OP_SEND_INITIAL_METADATA]); - s->cbs = bidirectional_stream_create(t->engine, s->curr_gs, - &cronet_callbacks); + s->cbs = + bidirectional_stream_create(t->engine, s->curr_gs, &cronet_callbacks); CRONET_LOG(GPR_DEBUG, "%p = bidirectional_stream_create()", s->cbs); if (t->use_packet_coalescing) { bidirectional_stream_disable_auto_flush(s->cbs, true); @@ -957,8 +957,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, result = NO_ACTION_POSSIBLE; CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed."); } else { - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, 0)", - s->cbs); + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, 0)", s->cbs); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, "", 0, true); if (t->use_packet_coalescing) { @@ -969,8 +968,8 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, } stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true; } else if (stream_op->recv_initial_metadata && - op_can_be_run(stream_op, s, &oas->state, - OP_RECV_INITIAL_METADATA)) { + op_can_be_run(stream_op, s, &oas->state, + OP_RECV_INITIAL_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_INITIAL_METADATA", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { grpc_closure_sched(exec_ctx, stream_op->recv_initial_metadata_ready, @@ -980,10 +979,10 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, GRPC_ERROR_NONE); } else { grpc_chttp2_incoming_metadata_buffer_publish( - exec_ctx, &oas->s->state.rs.initial_metadata, - stream_op->recv_initial_metadata); + exec_ctx, &oas->s->state.rs.initial_metadata, + stream_op->recv_initial_metadata); grpc_closure_sched(exec_ctx, stream_op->recv_initial_metadata_ready, - GRPC_ERROR_NONE); + GRPC_ERROR_NONE); } stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; @@ -1052,7 +1051,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->state_op_done[OP_READ_REQ_MADE] = true; /* Indicates that at least one read request has been made */ bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, - stream_state->rs.remaining_bytes); + stream_state->rs.remaining_bytes); result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_state->rs.remaining_bytes == 0) { @@ -1289,7 +1288,8 @@ grpc_transport *grpc_create_cronet_transport(void *engine, const char *target, ct->use_packet_coalescing = true; if (args) { for (size_t i = 0; i < args->num_args; i++) { - if (0 == strcmp(args->args[i].key, GRPC_ARG_USE_CRONET_PACKET_COALESCING)) { + if (0 == + strcmp(args->args[i].key, GRPC_ARG_USE_CRONET_PACKET_COALESCING)) { if (args->args[i].type != GRPC_ARG_INTEGER) { gpr_log(GPR_ERROR, "%s ignored: it must be an integer", GRPC_ARG_USE_CRONET_PACKET_COALESCING); From 5862f76bd14be059b1ffcd27d72548adede7e7b3 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 1 Feb 2017 17:00:13 -0800 Subject: [PATCH 031/167] GRPC_TIMEOUT_MILLIS_TO_DEADLINE now renamed to grpc_timeout_milliseconds_to_deadline --- test/core/iomgr/pollset_set_test.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index f1a56ecafb9..40fa8586020 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -268,7 +268,7 @@ static void pollset_set_test_basic() { make_test_fds_readable(tfds, num_fds); gpr_mu_lock(pollsets[i].mu); - deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(2); + deadline = grpc_timeout_milliseconds_to_deadline(2); GPR_ASSERT(GRPC_ERROR_NONE == grpc_pollset_work(&exec_ctx, pollsets[i].ps, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline)); @@ -350,7 +350,7 @@ void pollset_set_test_dup_fds() { make_test_fds_readable(tfds, num_fds); gpr_mu_lock(pollset.mu); - deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(2); + deadline = grpc_timeout_milliseconds_to_deadline(2); GPR_ASSERT(GRPC_ERROR_NONE == grpc_pollset_work(&exec_ctx, pollset.ps, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline)); @@ -419,7 +419,7 @@ void pollset_set_test_empty_pollset() { make_test_fds_readable(tfds, num_fds); gpr_mu_lock(pollsets[0].mu); - deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(2); + deadline = grpc_timeout_milliseconds_to_deadline(2); GPR_ASSERT(GRPC_ERROR_NONE == grpc_pollset_work(&exec_ctx, pollsets[0].ps, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline)); From 39401798c36dbd29eef41e0ff06239435502c821 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 2 Feb 2017 12:22:07 -0800 Subject: [PATCH 032/167] Fix links --- tools/profiling/microbenchmark/bm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmark/bm.py b/tools/profiling/microbenchmark/bm.py index 064bb7af1b4..a9277379c7b 100755 --- a/tools/profiling/microbenchmark/bm.py +++ b/tools/profiling/microbenchmark/bm.py @@ -74,7 +74,7 @@ for bm_name in sys.argv[1:]: 'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()]) for line in subprocess.check_output(['bins/basicprof/%s' % bm_name, '--benchmark_list_tests']).splitlines(): - link(line, 'reports/%s.txt' % fnize(line)) + link(line, '%s.txt' % fnize(line)) with open('reports/%s.txt' % fnize(line), 'w') as f: f.write(subprocess.check_output(['bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' % line])) @@ -99,7 +99,7 @@ for bm_name in sys.argv[1:]: with open('/tmp/bm.folded', 'w') as f: f.write(subprocess.check_output([ '%s/stackcollapse-perf.pl' % flamegraph_dir, '/tmp/bm.perf'])) - link(line, 'reports/%s.svg' % fnize(line)) + link(line, '%s.svg' % fnize(line)) with open('reports/%s.svg' % fnize(line), 'w') as f: f.write(subprocess.check_output([ '%s/flamegraph.pl' % flamegraph_dir, '/tmp/bm.folded'])) From 95ca017ce85ecdabe6e39275acfe9db736f97735 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 2 Feb 2017 12:27:11 -0800 Subject: [PATCH 033/167] Fix perf cmdline --- tools/profiling/microbenchmark/bm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmark/bm.py b/tools/profiling/microbenchmark/bm.py index a9277379c7b..b4f0de841fd 100755 --- a/tools/profiling/microbenchmark/bm.py +++ b/tools/profiling/microbenchmark/bm.py @@ -90,7 +90,7 @@ for bm_name in sys.argv[1:]: 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()]) for line in subprocess.check_output(['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines(): - subprocess.check_call(['sudo', 'perf', 'record', '-g', '-F', '99', + subprocess.check_call(['sudo', 'perf', 'record', '-g', '-c', '1000', 'bins/mutrace/%s' % bm_name, '--benchmark_filter=^%s$' % line, '--benchmark_min_time=20']) From 7dc4ea66f0688c30aa7ddd865074a808fd20f076 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 2 Feb 2017 16:08:05 -0800 Subject: [PATCH 034/167] Make the microbenchmark profile gatherer run some in parallel --- src/core/lib/profiling/basic_timers.c | 18 ++- .../latency_profile/profile_analyzer.py | 18 ++- .../python_utils/start_port_server.py | 128 ++++++++++++++++++ .../bm.py => run_tests/run_microbenchmark.py} | 36 +++-- tools/run_tests/run_tests.py | 94 +------------ 5 files changed, 182 insertions(+), 112 deletions(-) create mode 100644 tools/run_tests/python_utils/start_port_server.py rename tools/{profiling/microbenchmark/bm.py => run_tests/run_microbenchmark.py} (78%) diff --git a/src/core/lib/profiling/basic_timers.c b/src/core/lib/profiling/basic_timers.c index bdf9af2339c..c4f27728227 100644 --- a/src/core/lib/profiling/basic_timers.c +++ b/src/core/lib/profiling/basic_timers.c @@ -44,6 +44,8 @@ #include #include +#include "src/core/lib/support/env.h" + typedef enum { BEGIN = '{', END = '}', MARK = '.' } marker_type; typedef struct gpr_timer_entry { @@ -74,7 +76,7 @@ typedef struct gpr_timer_log_list { static __thread gpr_timer_log *g_thread_log; static gpr_once g_once_init = GPR_ONCE_INIT; static FILE *output_file; -static const char *output_filename = "latency_trace.txt"; +static const char *output_filename_or_null = NULL; static pthread_mutex_t g_mu; static pthread_cond_t g_cv; static gpr_timer_log_list g_in_progress_logs; @@ -85,6 +87,16 @@ static __thread int g_thread_id; static int g_next_thread_id; static int g_writing_enabled = 1; +static const char *output_filename() { + if (output_filename_or_null == NULL) { + output_filename_or_null = gpr_getenv("LATENCY_TRACE"); + if (output_filename_or_null == NULL) { + output_filename_or_null = "latency_trace.txt"; + } + } + return output_filename_or_null; +} + static int timer_log_push_back(gpr_timer_log_list *list, gpr_timer_log *log) { if (list->head == NULL) { list->head = list->tail = log; @@ -134,7 +146,7 @@ static void timer_log_remove(gpr_timer_log_list *list, gpr_timer_log *log) { static void write_log(gpr_timer_log *log) { size_t i; if (output_file == NULL) { - output_file = fopen(output_filename, "w"); + output_file = fopen(output_filename(), "w"); } for (i = 0; i < log->num_entries; i++) { gpr_timer_entry *entry = &(log->log[i]); @@ -198,7 +210,7 @@ static void finish_writing(void) { } void gpr_timers_set_log_filename(const char *filename) { - output_filename = filename; + output_filename_or_null = filename; } static void init_output() { diff --git a/tools/profiling/latency_profile/profile_analyzer.py b/tools/profiling/latency_profile/profile_analyzer.py index 48b8e9b950f..2087cd2793e 100755 --- a/tools/profiling/latency_profile/profile_analyzer.py +++ b/tools/profiling/latency_profile/profile_analyzer.py @@ -34,6 +34,7 @@ import hashlib import itertools import json import math +import sys import tabulate import time @@ -49,6 +50,7 @@ TIME_FROM_LAST_IMPORTANT = object() argp = argparse.ArgumentParser(description='Process output of basic_prof builds') argp.add_argument('--source', default='latency_trace.txt', type=str) argp.add_argument('--fmt', choices=tabulate.tabulate_formats, default='simple') +argp.add_argument('--out', default='-', type=str) args = argp.parse_args() class LineItem(object): @@ -246,16 +248,20 @@ FORMAT = [ ('TO_SCOPE_END', time_format(TIME_TO_SCOPE_END)), ] +out = sys.stdout +if args.out != '-': + out = open(args.out, 'w') + if args.fmt == 'html': - print '' - print '' - print 'Profile Report' - print '' + print >>out, '' + print >>out, '' + print >>out, 'Profile Report' + print >>out, '' accounted_for = 0 for cs in call_stacks: if args.fmt in BANNER: - print BANNER[args.fmt] % { + print >>out, BANNER[args.fmt] % { 'count': cs.count, } header, _ = zip(*FORMAT) @@ -265,7 +271,7 @@ for cs in call_stacks: for _, fn in FORMAT: fields.append(fn(line)) table.append(fields) - print tabulate.tabulate(table, header, tablefmt=args.fmt) + print >>out, tabulate.tabulate(table, header, tablefmt=args.fmt) accounted_for += cs.count if accounted_for > .99 * total_stacks: break diff --git a/tools/run_tests/python_utils/start_port_server.py b/tools/run_tests/python_utils/start_port_server.py new file mode 100644 index 00000000000..4103eb0534b --- /dev/null +++ b/tools/run_tests/python_utils/start_port_server.py @@ -0,0 +1,128 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import print_function + +from six.moves import urllib +import os +import subprocess +import tempfile +import sys +import time + +def start_port_server(port_server_port): + # check if a compatible port server is running + # if incompatible (version mismatch) ==> start a new one + # if not running ==> start a new one + # otherwise, leave it up + try: + version = int(urllib.request.urlopen( + 'http://localhost:%d/version_number' % port_server_port, + timeout=10).read()) + print('detected port server running version %d' % version) + running = True + except Exception as e: + print('failed to detect port server: %s' % sys.exc_info()[0]) + print(e.strerror) + running = False + if running: + current_version = int(subprocess.check_output( + [sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), + 'dump_version'])) + print('my port server is version %d' % current_version) + running = (version >= current_version) + if not running: + print('port_server version mismatch: killing the old one') + urllib.request.urlopen('http://localhost:%d/quitquitquit' % port_server_port).read() + time.sleep(1) + if not running: + fd, logfile = tempfile.mkstemp() + os.close(fd) + print('starting port_server, with log file %s' % logfile) + args = [sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), + '-p', '%d' % port_server_port, '-l', logfile] + env = dict(os.environ) + env['BUILD_ID'] = 'pleaseDontKillMeJenkins' + if platform_string() == 'windows': + # Working directory of port server needs to be outside of Jenkins + # workspace to prevent file lock issues. + tempdir = tempfile.mkdtemp() + port_server = subprocess.Popen( + args, + env=env, + cwd=tempdir, + creationflags = 0x00000008, # detached process + close_fds=True) + else: + port_server = subprocess.Popen( + args, + env=env, + preexec_fn=os.setsid, + close_fds=True) + time.sleep(1) + # ensure port server is up + waits = 0 + while True: + if waits > 10: + print('killing port server due to excessive start up waits') + port_server.kill() + if port_server.poll() is not None: + print('port_server failed to start') + # try one final time: maybe another build managed to start one + time.sleep(1) + try: + urllib.request.urlopen('http://localhost:%d/get' % port_server_port, + timeout=1).read() + print('last ditch attempt to contact port server succeeded') + break + except: + traceback.print_exc() + port_log = open(logfile, 'r').read() + print(port_log) + sys.exit(1) + try: + urllib.request.urlopen('http://localhost:%d/get' % port_server_port, + timeout=1).read() + print('port server is up and ready') + break + except socket.timeout: + print('waiting for port_server: timeout') + traceback.print_exc(); + time.sleep(1) + waits += 1 + except urllib.error.URLError: + print('waiting for port_server: urlerror') + traceback.print_exc(); + time.sleep(1) + waits += 1 + except: + traceback.print_exc() + port_server.kill() + raise + diff --git a/tools/profiling/microbenchmark/bm.py b/tools/run_tests/run_microbenchmark.py similarity index 78% rename from tools/profiling/microbenchmark/bm.py rename to tools/run_tests/run_microbenchmark.py index b4f0de841fd..42a31a622f7 100755 --- a/tools/profiling/microbenchmark/bm.py +++ b/tools/run_tests/run_microbenchmark.py @@ -33,8 +33,18 @@ import os import subprocess import sys +import python_utils.jobset as jobset +import python_utils.start_port_server as start_port_server + flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph') +os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +if not os.path.exists('reports'): + os.makedirs('reports') + +port_server_port = 32766 +start_port_server.start_port_server(port_server_port) + def fnize(s): out = '' for c in s: @@ -45,10 +55,6 @@ def fnize(s): out += c return out -os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) -if not os.path.exists('reports'): - os.makedirs('reports') - # index html index_html = """ @@ -66,6 +72,9 @@ def link(txt, tgt): global index_html index_html += "

%s

\n" % (tgt, txt) +benchmarks = [] +profile_analysis = [] + for bm_name in sys.argv[1:]: # generate latency profiles heading('Latency Profiles: %s' % bm_name) @@ -75,13 +84,18 @@ for bm_name in sys.argv[1:]: for line in subprocess.check_output(['bins/basicprof/%s' % bm_name, '--benchmark_list_tests']).splitlines(): link(line, '%s.txt' % fnize(line)) - with open('reports/%s.txt' % fnize(line), 'w') as f: - f.write(subprocess.check_output(['bins/basicprof/%s' % bm_name, - '--benchmark_filter=^%s$' % line])) - f.write('\n***********************************************************\n') - f.write(subprocess.check_output([ - sys.executable, 'tools/profiling/latency_profile/profile_analyzer.py', - '--source', 'latency_trace.txt', '--fmt', 'simple'])) + benchmarks.append( + jobset.JobSpec(['bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' % line], + environ={'LATENCY_TRACE': '%s.trace' % fnize(line)})) + profile_analysis.append( + jobset.JobSpec([sys.executable, + 'tools/profiling/latency_profile/profile_analyzer.py', + '--source', '%s.trace' % fnize(line), '--fmt', 'simple', + '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None)) + + jobset.run(benchmarks, maxjobs=multiprocessing.cpu_count()/2, + add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) # generate flamegraphs heading('Flamegraphs: %s' % bm_name) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 9d767258102..999ebf87eb7 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -57,6 +57,7 @@ import uuid import python_utils.jobset as jobset import python_utils.report_utils as report_utils import python_utils.watch_dirs as watch_dirs +import python_utils.start_port_server as start_port_server _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) @@ -1322,97 +1323,6 @@ def _shut_down_legacy_server(legacy_server_port): 'http://localhost:%d/quitquitquit' % legacy_server_port).read() -def _start_port_server(port_server_port): - # check if a compatible port server is running - # if incompatible (version mismatch) ==> start a new one - # if not running ==> start a new one - # otherwise, leave it up - try: - version = int(urllib.request.urlopen( - 'http://localhost:%d/version_number' % port_server_port, - timeout=10).read()) - print('detected port server running version %d' % version) - running = True - except Exception as e: - print('failed to detect port server: %s' % sys.exc_info()[0]) - print(e.strerror) - running = False - if running: - current_version = int(subprocess.check_output( - [sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), - 'dump_version'])) - print('my port server is version %d' % current_version) - running = (version >= current_version) - if not running: - print('port_server version mismatch: killing the old one') - urllib.request.urlopen('http://localhost:%d/quitquitquit' % port_server_port).read() - time.sleep(1) - if not running: - fd, logfile = tempfile.mkstemp() - os.close(fd) - print('starting port_server, with log file %s' % logfile) - args = [sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), - '-p', '%d' % port_server_port, '-l', logfile] - env = dict(os.environ) - env['BUILD_ID'] = 'pleaseDontKillMeJenkins' - if platform_string() == 'windows': - # Working directory of port server needs to be outside of Jenkins - # workspace to prevent file lock issues. - tempdir = tempfile.mkdtemp() - port_server = subprocess.Popen( - args, - env=env, - cwd=tempdir, - creationflags = 0x00000008, # detached process - close_fds=True) - else: - port_server = subprocess.Popen( - args, - env=env, - preexec_fn=os.setsid, - close_fds=True) - time.sleep(1) - # ensure port server is up - waits = 0 - while True: - if waits > 10: - print('killing port server due to excessive start up waits') - port_server.kill() - if port_server.poll() is not None: - print('port_server failed to start') - # try one final time: maybe another build managed to start one - time.sleep(1) - try: - urllib.request.urlopen('http://localhost:%d/get' % port_server_port, - timeout=1).read() - print('last ditch attempt to contact port server succeeded') - break - except: - traceback.print_exc() - port_log = open(logfile, 'r').read() - print(port_log) - sys.exit(1) - try: - urllib.request.urlopen('http://localhost:%d/get' % port_server_port, - timeout=1).read() - print('port server is up and ready') - break - except socket.timeout: - print('waiting for port_server: timeout') - traceback.print_exc(); - time.sleep(1) - waits += 1 - except urllib.error.URLError: - print('waiting for port_server: urlerror') - traceback.print_exc(); - time.sleep(1) - waits += 1 - except: - traceback.print_exc() - port_server.kill() - raise - - def _calculate_num_runs_failures(list_of_results): """Caculate number of runs and failures for a particular test. @@ -1460,7 +1370,7 @@ def _build_and_run( antagonists = [subprocess.Popen(['tools/run_tests/python_utils/antagonist.py']) for _ in range(0, args.antagonists)] port_server_port = 32766 - _start_port_server(port_server_port) + start_port_server.start_port_server(port_server_port) resultset = None num_test_failures = 0 try: From b0fd02ed069c67df3d045fdf1131e6f429941263 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Feb 2017 15:58:24 -0800 Subject: [PATCH 035/167] Add examples from fuzzer server --- .../039c25bc070936901fc95f63ce9cc3058158fb6d | Bin 0 -> 89 bytes .../0f01df12331467c4eed400465254eda05eaeb110 | Bin 0 -> 276 bytes .../174ea33060bccea880dfdcfa12c5349e8eb4cb2a | Bin 0 -> 144 bytes .../25c05b84c4b4f1a93f0f72d368c3c3644b564881 | Bin 0 -> 296 bytes .../2d743ec0a0826177cfa7ffb335c0034f482e70e5 | Bin 0 -> 85 bytes .../30738789e20323323196dd3e6435fa278e73279e | Bin 0 -> 85 bytes .../33e01fe9738c887b159ca5add342b22c13e526cf | Bin 0 -> 230 bytes .../3a3346314bb9ddaf14877b653cfd506b6ad34fab | Bin 0 -> 358 bytes .../42231300ca5cf30d18f55b66020926882c64248c | Bin 0 -> 81 bytes .../428cce92c42645f4cc4060a8cb9cef3a803c0341 | Bin 0 -> 230 bytes .../43cdc82b082bbdc4d7d23437a7f761f1ca32ca73 | Bin 0 -> 253 bytes .../448fc1dc939aa7f398f1577e418630abecc0a1d7 | Bin 0 -> 227 bytes .../46171f477d11338f4cc948915350772d54319200 | Bin 0 -> 116 bytes .../51ea84d5a790d3d2495be453f5341c41b6153644 | Bin 0 -> 90 bytes .../5547a3544fc5c634024d546366704547dd72cc2b | Bin 0 -> 85 bytes .../64656ddf81738f914ead8003c19d0148c54f34d6 | Bin 0 -> 87 bytes .../681b758cad3bbce4bde2d1a78a2ec4600c59b05c | Bin 0 -> 127 bytes .../72031f24261c32d2e3bb2c7909a9315227172730 | Bin 0 -> 85 bytes .../783484ad9e15085e9039c7504aac71af1ad549a2 | Bin 0 -> 144 bytes .../78e43d163fc8226d72b979c0fe6e1593ef3cb542 | Bin 0 -> 132 bytes .../7fb64b5785ebe699ca50327c88c1d8b99432fa23 | Bin 0 -> 89 bytes .../82731abc38788755495b1bac7b58bc0f12e4bdd1 | Bin 0 -> 111 bytes .../83776278a4997b0d178602c8419f3e6481dec01d | Bin 0 -> 226 bytes .../92c816d98f9f8669f43b46b22d5da21464d9ef41 | Bin 0 -> 516 bytes .../9ca8f5c67662fa6726db2680978e443d80785a9f | Bin 0 -> 109 bytes .../9cc4eeecbb2df8b130cca5e455a0f6b8a6e00660 | Bin 0 -> 114 bytes .../a270e4304cc0dcd2c67b78c0495dedb10419f0af | Bin 0 -> 275 bytes .../bc42e00a7d67fb68df3cb5893908c04884b6ad5e | Bin 0 -> 252 bytes .../c1e5524945b3e3eabedfbb675be9e9ea99a36b94 | Bin 0 -> 85 bytes .../c9f81864507c264369dd22c72aeb16f1cb1742b0 | Bin 0 -> 228 bytes ...h-177af631195e806f4056847cea4d09b5eb28cf8a | Bin 0 -> 345 bytes ...h-5fc15c2ee9c70fd834588cbd256cfb52cdcbcb8d | Bin 0 -> 332 bytes ...h-b5ae6881c767a7769bb957ac379f22aafe4ef05e | Bin 0 -> 230 bytes .../d0d622fa3916e800e959a3fc2c90e213d518e5f4 | Bin 0 -> 365 bytes .../dd229da166c3bec675e882d17092238cf7d245f3 | Bin 0 -> 229 bytes .../de45c55043f63ec680e990b1edf1f0cc60ebbf4e | Bin 0 -> 109 bytes .../e709e8861c09e29cdae73e337587a63fb0ccf76d | Bin 0 -> 86 bytes .../e839090caa1b1bfa8898eea683f5d5c9f1ed6dd1 | Bin 0 -> 200 bytes .../e9932127668f9de0743fc639dca31acedbfc68fd | Bin 0 -> 112 bytes .../eacf4905e489566a3e5fcaaeac9fe91cbf916e06 | Bin 0 -> 85 bytes .../ec55cbebe6db506acf7af9e5d26386630319623d | Bin 0 -> 87 bytes .../f09c5cabe569b5c22a16d7d584074d54d9343edc | Bin 0 -> 365 bytes .../f9048c4c18e729b6f49e929876ec30866deb16a9 | Bin 0 -> 84 bytes .../f92897ee60bd24634aa1582f162c1c8f4b249148 | Bin 0 -> 127 bytes .../fd05ad1a9d183c2a25d820aca9940caacbaa0660 | Bin 0 -> 129 bytes tools/run_tests/generated/tests.json | 1090 ++++++++++++++++- 46 files changed, 1040 insertions(+), 50 deletions(-) create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/039c25bc070936901fc95f63ce9cc3058158fb6d create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/0f01df12331467c4eed400465254eda05eaeb110 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/174ea33060bccea880dfdcfa12c5349e8eb4cb2a create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/25c05b84c4b4f1a93f0f72d368c3c3644b564881 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/2d743ec0a0826177cfa7ffb335c0034f482e70e5 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/30738789e20323323196dd3e6435fa278e73279e create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/33e01fe9738c887b159ca5add342b22c13e526cf create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/3a3346314bb9ddaf14877b653cfd506b6ad34fab create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/42231300ca5cf30d18f55b66020926882c64248c create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/428cce92c42645f4cc4060a8cb9cef3a803c0341 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/43cdc82b082bbdc4d7d23437a7f761f1ca32ca73 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/448fc1dc939aa7f398f1577e418630abecc0a1d7 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/46171f477d11338f4cc948915350772d54319200 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/51ea84d5a790d3d2495be453f5341c41b6153644 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/5547a3544fc5c634024d546366704547dd72cc2b create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/64656ddf81738f914ead8003c19d0148c54f34d6 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/681b758cad3bbce4bde2d1a78a2ec4600c59b05c create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/72031f24261c32d2e3bb2c7909a9315227172730 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/783484ad9e15085e9039c7504aac71af1ad549a2 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/78e43d163fc8226d72b979c0fe6e1593ef3cb542 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/7fb64b5785ebe699ca50327c88c1d8b99432fa23 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/82731abc38788755495b1bac7b58bc0f12e4bdd1 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/83776278a4997b0d178602c8419f3e6481dec01d create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/92c816d98f9f8669f43b46b22d5da21464d9ef41 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/9ca8f5c67662fa6726db2680978e443d80785a9f create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/9cc4eeecbb2df8b130cca5e455a0f6b8a6e00660 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/a270e4304cc0dcd2c67b78c0495dedb10419f0af create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/bc42e00a7d67fb68df3cb5893908c04884b6ad5e create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/c1e5524945b3e3eabedfbb675be9e9ea99a36b94 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/c9f81864507c264369dd22c72aeb16f1cb1742b0 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/crash-177af631195e806f4056847cea4d09b5eb28cf8a create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/crash-5fc15c2ee9c70fd834588cbd256cfb52cdcbcb8d create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/crash-b5ae6881c767a7769bb957ac379f22aafe4ef05e create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/d0d622fa3916e800e959a3fc2c90e213d518e5f4 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/dd229da166c3bec675e882d17092238cf7d245f3 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/de45c55043f63ec680e990b1edf1f0cc60ebbf4e create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/e709e8861c09e29cdae73e337587a63fb0ccf76d create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/e839090caa1b1bfa8898eea683f5d5c9f1ed6dd1 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/e9932127668f9de0743fc639dca31acedbfc68fd create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/eacf4905e489566a3e5fcaaeac9fe91cbf916e06 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/ec55cbebe6db506acf7af9e5d26386630319623d create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/f09c5cabe569b5c22a16d7d584074d54d9343edc create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/f9048c4c18e729b6f49e929876ec30866deb16a9 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/f92897ee60bd24634aa1582f162c1c8f4b249148 create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/fd05ad1a9d183c2a25d820aca9940caacbaa0660 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/039c25bc070936901fc95f63ce9cc3058158fb6d b/test/core/end2end/fuzzers/api_fuzzer_corpus/039c25bc070936901fc95f63ce9cc3058158fb6d new file mode 100644 index 0000000000000000000000000000000000000000..fd96a329756d715cb856d11ea4034df9495cb971 GIT binary patch literal 89 zcmZQ7PAw`+En;9`Vc=rWPs`5-^BL2N3X=7Ti*w>j5{uGPOXBkq*XE|i=L1z1Wu~Mu k^#A|=zh0P|hlzoKff0oN{|7M`e|0i2aPu&-GcafZ03>x9!vFvP literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/0f01df12331467c4eed400465254eda05eaeb110 b/test/core/end2end/fuzzers/api_fuzzer_corpus/0f01df12331467c4eed400465254eda05eaeb110 new file mode 100644 index 0000000000000000000000000000000000000000..09bf4db5853b1dd44e72c0e6c1e4d189403ad84e GIT binary patch literal 276 zcmZvXKMny=5Ql&BR-Uy)Bod8AM-&{vQ8pXKpM+gzS*1oIG!msdsNBF&Brc(Op0x#u zsb(@W-#4RTmV^b!!9#Z#MQKeVnfE(!JoT3g8S&B&gBkmg9h?jpM`JeCVu^qX83WK> zgHZI^wStQ4|9J8SenEp@XiCj|@7C$#TUp;V6K2N#I0!?Y0t(Rd+ym(yPF`u)@YdXu sM3o!RFznhCB+w7VB1sE$zo1F7-NCq^!wtA^?b5|wZ=Y!a4)X39FO$np+W-In literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/174ea33060bccea880dfdcfa12c5349e8eb4cb2a b/test/core/end2end/fuzzers/api_fuzzer_corpus/174ea33060bccea880dfdcfa12c5349e8eb4cb2a new file mode 100644 index 0000000000000000000000000000000000000000..da32e950137574065ac207a8cf453f654e9066d8 GIT binary patch literal 144 zcmWek%*kP3Do!mbOD$qxU}4~5&`-mNOA?FHQ%mCW64&OY#^(c7 z7GfP=Tl^IErPF0U{Z%nKh<_!V-ceJ$Inu2002+E|F%l zSe8OYQ_S!8W|UJm(r++wO7h^U8zxCifF_Zty8rsM`wYGlo*61~Y^Hq{%oB|5a^m790GlI6# z@D}ZOCZ+22u_}rjJ_HH$w_>Gty_`cSXAUCe5Olv{Fp=sA7lOVH0ad;dR`+uay84G6 LJ_G`^!wY@^>2g-K literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/2d743ec0a0826177cfa7ffb335c0034f482e70e5 b/test/core/end2end/fuzzers/api_fuzzer_corpus/2d743ec0a0826177cfa7ffb335c0034f482e70e5 new file mode 100644 index 0000000000000000000000000000000000000000..5449b346c0100433e517cddd5fc469deb9a12623 GIT binary patch literal 85 zcmZQ7PAw`+En;9`Vc=rWPs`5-^BL2N3X+S9<4Y2Y(o;*~^AgwQrpD(3)fHu?h@~?0 j|Nmbv%+15dz`(%x|38orXJGu*$-p4O!^oV&z^(}Ztj5{uGPOXBkq*XE|i=L1z1WvZky i^#A|=zh0P|hmnDSf${%;Ai==+tCN8ND8k0Tpa}rK+!}xY literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/33e01fe9738c887b159ca5add342b22c13e526cf b/test/core/end2end/fuzzers/api_fuzzer_corpus/33e01fe9738c887b159ca5add342b22c13e526cf new file mode 100644 index 0000000000000000000000000000000000000000..9b3060c5175d2ae06f17f53f7191affb91848e3f GIT binary patch literal 230 zcmW-au}%U(6h-e01W}2NpJ0JfKy0y4J9}gOGA=Lcgx%TAo7puq8WRfr7`}mupRlCy z4>U0(<6v=bag%e8_AD(6l**;f#1>nOeJjPP$_VkjnQjyj< z7xM%SFQ-L~9Hmm(3gO#Y`+jwH?)eAX6KOLhwE02Ch#q){BhTH51GdUg=k0)wC<^4# zsRm*Pgs;??Y7WD~Sgdoy#oDA&WhTyL`=H|6d+Txt-R`a3`7jzmzyd-^h`-mb;RD0P zO_YkIOY8wC`lQ~U;RIZZ>rlBAtP OTl$$0l%imL2dIDg6LoX| literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/42231300ca5cf30d18f55b66020926882c64248c b/test/core/end2end/fuzzers/api_fuzzer_corpus/42231300ca5cf30d18f55b66020926882c64248c new file mode 100644 index 0000000000000000000000000000000000000000..0119532390b97d956b7848e72509c14494a10466 GIT binary patch literal 81 zcmZQ7PAw`+En;9`Vc=rWPs;~$8PkgjlJ$yM~PO8T$YK guNUU#VPs$sVEq4|fq}7;je(JYn~Rx&fq|JF04XyULI3~& literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/428cce92c42645f4cc4060a8cb9cef3a803c0341 b/test/core/end2end/fuzzers/api_fuzzer_corpus/428cce92c42645f4cc4060a8cb9cef3a803c0341 new file mode 100644 index 0000000000000000000000000000000000000000..39faa0d693f5dbc35cedcce5d8c46193e1b7b22f GIT binary patch literal 230 zcmZvWO$x#=5QSe_f53LCjKDck>f_xsI-25;p0F|J6fFX5su=+@D<6O6H4<9)X+T4PSCAyvtJ@5zA!B8FLp MJeeYehz-;E3m>;lhyVZp literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/43cdc82b082bbdc4d7d23437a7f761f1ca32ca73 b/test/core/end2end/fuzzers/api_fuzzer_corpus/43cdc82b082bbdc4d7d23437a7f761f1ca32ca73 new file mode 100644 index 0000000000000000000000000000000000000000..c8250c8ccf5d930986a6d389428781844a04059e GIT binary patch literal 253 zcmZvUK?=e!5Je}kinZN{&?|_WdH@d-VmqyYwkgxJP&a}*S9$|aAl^@q80ta=XYpbF ze>~nGbw~gn%wU~qJ^X%7N!&DrFp_dKqLQ}~1^>soa{nc`zXil{s*~vY<0w}h3R0LN zoUOHoMKEOm{&B({Cm3!ni=}opQXeLC3@~e5zyJmz8x8t6#Y}n;pSbC1EqC0et|?AP dm6DLgJ?Zw6incI9wq~bE8PeMxR0RTn=Py?9Q7r%f literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/448fc1dc939aa7f398f1577e418630abecc0a1d7 b/test/core/end2end/fuzzers/api_fuzzer_corpus/448fc1dc939aa7f398f1577e418630abecc0a1d7 new file mode 100644 index 0000000000000000000000000000000000000000..d5f631429e598b6f49f9d2a44da1cff0216d573b GIT binary patch literal 227 zcmW-aF-`+95JkV85CTLr9DxQ>5b8|-9-Fet#cvnJXVh|;+qy*e_Bm6pXD6fH0n~24a4UiO{TRvJ z^wB%i1JdW$)0>CMJL|#oDGmAMV!W3pwm#N@lZoUSE>pHdtGJWT)z21Pm#LM)c8#MU H*0{$XZB9vh literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/46171f477d11338f4cc948915350772d54319200 b/test/core/end2end/fuzzers/api_fuzzer_corpus/46171f477d11338f4cc948915350772d54319200 new file mode 100644 index 0000000000000000000000000000000000000000..df032667c6d82427e91bae4f3590a3c22d1f13ef GIT binary patch literal 116 zcmZQ7PAw`+En;9`Vc=rWPs`5-^BL2N3X=7Ti*w>j5{uGPOXBkq*XE|i=L1z1Wu~Mu z^#A|=zh0P|hmnDSf${%;Ai==+tCN9&n}>-Rq!%d60MwtFoS&DHSX3Eb3N$r7F+DY} I1Sp{i0I@S9?EnA( literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/51ea84d5a790d3d2495be453f5341c41b6153644 b/test/core/end2end/fuzzers/api_fuzzer_corpus/51ea84d5a790d3d2495be453f5341c41b6153644 new file mode 100644 index 0000000000000000000000000000000000000000..df8ee081bfb74c566acfe0a36041e663543f8c33 GIT binary patch literal 90 zcmZQ7PAw`+En;9`Vc=rWPs`5-^BL2N3X=7Ti*w>j5{uGPOPm-P7~=C1*XE|i=L1z2 mWu~MuX!QU8|G!?Cn+GVz!1(_^kYHf^)ycpB6k%sz&;$S<6&ox7 literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/5547a3544fc5c634024d546366704547dd72cc2b b/test/core/end2end/fuzzers/api_fuzzer_corpus/5547a3544fc5c634024d546366704547dd72cc2b new file mode 100644 index 0000000000000000000000000000000000000000..9e6f5b687d42fa7b59a11a1284b99f860bbdedd3 GIT binary patch literal 85 zcmZQ7PAw`+En;9`Vc=rWPs`5-^BL2N3X=7Ti*w>j5{uGPOXBkq*XE|i=L1z1Wu~Mu i^#A|=zh0P|hmnDSf${%;Ai==+tCN8ND8kObzy$!vK^md} literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/64656ddf81738f914ead8003c19d0148c54f34d6 b/test/core/end2end/fuzzers/api_fuzzer_corpus/64656ddf81738f914ead8003c19d0148c54f34d6 new file mode 100644 index 0000000000000000000000000000000000000000..daf5bc14b1c80f229a90efa7cb46770e8a64f830 GIT binary patch literal 87 zcmZQ7PAw`+En;9`Vc=rWPs`5-^BL2N3X=7Ti*w>j5{uGPOXBkq*XE|i=L1z1Wu~Mu l^#A|=zh0P|hmnDSf${%;Ai==+tCN9&mw}syiJ5^x69E0Z8p{9x literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/681b758cad3bbce4bde2d1a78a2ec4600c59b05c b/test/core/end2end/fuzzers/api_fuzzer_corpus/681b758cad3bbce4bde2d1a78a2ec4600c59b05c new file mode 100644 index 0000000000000000000000000000000000000000..d54a2af33deaa60e9f3c116697e91fea4b9bfce0 GIT binary patch literal 127 zcmZQ7PAw`+En;9`Vc=p&OV0;!({l35lS^_K%GEhiIW&sOG_;cSii>l;$1}0=IM!rVMSRsjR! Zk4^>##j5{uGPOXBkq*XE|i=L1z1Wu~Mu j^#A|=zh0P|hmnDSf${%;Ai==+tCN9&n}>;+fk6`h$W>SiqLaz{JQ<&X>A}gQF--<3CWi<&oR}wJZ#*?3_#-44hI-#i>Q* zeBZ#*TK^fC6jRleBNQ1JlQRix}Qnw>mNOA?FHQ%mCW64&OY#^(c-7O7jM nq%!pX|Np;Un45=@fq{YX|9>FC!1$|^fq|Qqhmo0qfn5^-9}^nr literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/82731abc38788755495b1bac7b58bc0f12e4bdd1 b/test/core/end2end/fuzzers/api_fuzzer_corpus/82731abc38788755495b1bac7b58bc0f12e4bdd1 new file mode 100644 index 0000000000000000000000000000000000000000..20c7c70f83590364a0c65a49a4482cdc87d1ae0d GIT binary patch literal 111 zcmZQ7PAw`+En;9`Vc=rWPs`5-^BL2N3X=7Ti*w>j5{uGPOXBkq*XE|i=L1z1Wu~Mu z^#A|=zh0P|hmnDSf${%;Ai==+tCN8ND8dZV%fOJVmy(*6SejE3pIBOwkzbTqQVCS6 F2>=HZBgFs! literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/83776278a4997b0d178602c8419f3e6481dec01d b/test/core/end2end/fuzzers/api_fuzzer_corpus/83776278a4997b0d178602c8419f3e6481dec01d new file mode 100644 index 0000000000000000000000000000000000000000..2570ea8bc6cc06db8153843d2d5513c75a091aa2 GIT binary patch literal 226 zcmW-aF;W6S5JmqC2#A!4M<5XcFhLj<`_)Hp zs0XB{H;dcH`8(^u`kY4mdUJ;fs={k8ZJ||L94iv&+YFXU6-k?!hVO-5q5aM EAHKs#bpQYW literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/92c816d98f9f8669f43b46b22d5da21464d9ef41 b/test/core/end2end/fuzzers/api_fuzzer_corpus/92c816d98f9f8669f43b46b22d5da21464d9ef41 new file mode 100644 index 0000000000000000000000000000000000000000..c000647a860f717a4d0d16ac558586b553700ccd GIT binary patch literal 516 zcmZvYF;4jlhOev^3ae&5R|>`I z@}$77ivrhJJfU%8Tzk4?Ys#w>wsEwf^S;(wk+LPQg{#FX5^fJ0xc|U!FV8oWP?r4w literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/9ca8f5c67662fa6726db2680978e443d80785a9f b/test/core/end2end/fuzzers/api_fuzzer_corpus/9ca8f5c67662fa6726db2680978e443d80785a9f new file mode 100644 index 0000000000000000000000000000000000000000..4954b7255c19957e4f6350ae31ad1da055c28943 GIT binary patch literal 109 zcmZQ7PAw`+En;9`Vc=rWPs`5-^BL2N3X=7Ti*w>j5{uGPOXBkq*XE|i=L1z1Wu~Mu z^#A|=zh0P|hmnDSf${%;Ai==+tCN8ND8dZUo06KASejE3pIBOwkzbTqQVCSa1OU$= BBc1>N literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/9cc4eeecbb2df8b130cca5e455a0f6b8a6e00660 b/test/core/end2end/fuzzers/api_fuzzer_corpus/9cc4eeecbb2df8b130cca5e455a0f6b8a6e00660 new file mode 100644 index 0000000000000000000000000000000000000000..86f991ed002eea2ba938fea2f153f80d98f69509 GIT binary patch literal 114 zcmZQ7PAw`+En;9`Vc=rWPs`7T@EOyK3X=7Ti*w>j5{uGPORVDa5_41I^MOhgi!xKJ m82bPJ|6ec6&BM%qt`wmXs4$fQ2$v3Cb6H*ge*OW3qFnFO_} zO4a+`D`ix(mX+-D>3st@mn8UHggXm0nj+IZnD_sb23(>O_Z4k$p=>j-2q3MHlCE8Oxx sQdMh1!*SqSkU&2bi=m~WBM8VNEZfn zq_PB8#e+|>{rt{Yu{11z#^h3_Hx;MBXCw<-YsXT~J^}(j0>>~Og`iahre8XJbs(H2 z$vIk2bGAX3r7z??J6lCcU+r^UxJ(T+w|>Hn6d9ETwC&8S&dxcGk37ZLxzax&;3 zm_(iXFuxxI&7TQ$j%B1w`Kk;Ot=>>e5f`JzNUQAn UXRE$Re{Ep9fqoA*@SrUG0b*27#Q*>R literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/c1e5524945b3e3eabedfbb675be9e9ea99a36b94 b/test/core/end2end/fuzzers/api_fuzzer_corpus/c1e5524945b3e3eabedfbb675be9e9ea99a36b94 new file mode 100644 index 0000000000000000000000000000000000000000..12bb8f2d6492baea865a6c1117c6db1d5d5ad611 GIT binary patch literal 85 zcmZQ7PAw`+En;9`Vc=rWPs`5-^BL2N3X=7Ti*w>j5{uGPOXBkq*XE|i=L1z1Wu~Mu i^#A|=zh0P|hmnDSf${%;Ai==+tCN8ND8j_Rpa}rTOd7lZ literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/c9f81864507c264369dd22c72aeb16f1cb1742b0 b/test/core/end2end/fuzzers/api_fuzzer_corpus/c9f81864507c264369dd22c72aeb16f1cb1742b0 new file mode 100644 index 0000000000000000000000000000000000000000..0331bad1a67ff859eeb3944004bd137cd0bf95a6 GIT binary patch literal 228 zcmW-aJx&8L6okjlj{-zA9DxEtL8wCkbo7YYdTgE+i(T*P*=v*rniO!DoPmmqq@krE z5<)Ugikafm{AkY-S)f!YZE@a;tw|Q9%h*uKEFZu%yA_h+)W@@RhUSV&9 H6>jkl{VYng literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/crash-177af631195e806f4056847cea4d09b5eb28cf8a b/test/core/end2end/fuzzers/api_fuzzer_corpus/crash-177af631195e806f4056847cea4d09b5eb28cf8a new file mode 100644 index 0000000000000000000000000000000000000000..344825c5718c64f75fdb61fd8a2508ab7c268ac9 GIT binary patch literal 345 zcmXw#y-veG5QOK>0Rc(JBT%3;33Vudjvi5;4tp^czO%Ld5vgd>AR+N2eg-ODBn>SU zNC@dz+t6$=+S&PbbVb|}L#dR@cu{iZEU!%?vau=o8XCX^2Us2#g{b`V6+%lF@3KE1 zJXMt&%PG~Su-;JXh~qo!J>{mVZG3NTnk-Kw!#d+v)N+>CGDCR1R*#3%S)`#p`jB(r z8BpI(GnsD_L_63eeROhqcD2Nib)W(UQ{F;$$wK@V>|3i MTYKnmiCR|Xzm>*u?EnA( literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/crash-5fc15c2ee9c70fd834588cbd256cfb52cdcbcb8d b/test/core/end2end/fuzzers/api_fuzzer_corpus/crash-5fc15c2ee9c70fd834588cbd256cfb52cdcbcb8d new file mode 100644 index 0000000000000000000000000000000000000000..ce2ffbb6d521decab532d53df1e95ece851feae6 GIT binary patch literal 332 zcmYL^u}T9$6h+_6h8WEvgrGklqNXs`vCwq(Vm*wTmvwQnyS$m*kj69-w2D8HZ?N(w zEYk>nfP$#wTZ34yw21b`ZBKzA?8Sf$T* z?@R2zu;Jj>WUWz}Vst9iBUBmZMOQja(J1Ts(Y3lP5;NcuJyK~8OG*YhPryH3h-teS z1`@<=RdUJ);^RK#|5SruP1aHF937vWckf8n$D2rvo6dfx3U{Pan`cg0j0#HJ1WG}k zjlz4d@@=(h#XV&1-D>5~WaG#gU|y%=2f}|~X6ntBGXcv5tUn{X`njJrLs@Zw_~PVN XLmLBIe;LpQ@y+r!v~&OqI5YAGsa$N- literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/crash-b5ae6881c767a7769bb957ac379f22aafe4ef05e b/test/core/end2end/fuzzers/api_fuzzer_corpus/crash-b5ae6881c767a7769bb957ac379f22aafe4ef05e new file mode 100644 index 0000000000000000000000000000000000000000..3518dffafcdae38f7f74427001738c54e37a1602 GIT binary patch literal 230 zcmY+8I}XAy5JblbABm*n5I_?oPC&yEs4bIhJ|e`CZ6QSz)TuZL7o;(3s?cn)J3H@1 zmdu?DFvAM_QtQocAu7}7nYFES%2X#OJN3Si)EqTUL3nxZpX($Q0EG8EAU-3Iq){Ax z`Rw_ha@`fm4F7c$>ZWa$6-0Gr;)96|6Sp)?*uZ1lF7S@RqYQSnbNwM}wZ5vFR3R^e SBa7fNE)t~C95F-?jF(^0CrxDl literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/d0d622fa3916e800e959a3fc2c90e213d518e5f4 b/test/core/end2end/fuzzers/api_fuzzer_corpus/d0d622fa3916e800e959a3fc2c90e213d518e5f4 new file mode 100644 index 0000000000000000000000000000000000000000..0de93e6a0d2f68b2b13d7615b284bdd36a57b444 GIT binary patch literal 365 zcmZutK}th05PdT(Y5Fa0yg+vq7xf4pq6heVG!{YqWq$d8Sr-AR8~5TBa;boq&_$f2 z2rZb9VJ7qDy~l9pq;~?0QQ?|RHMkq;dB$dNR+DoxBu&_^mTWTCx}(;8+m?A+%jS03 zb&vJ7k+s{_&rNu8b*s$saOd!zQrcad%?JRcqyhB32lQtGR9T_t6DNiFQ|B{ve$6TU zF?;uKtLXk5bVn*Y)~kojpE59Vi3E(ueaXn<1(w9l5j~QclsOCz_Owta$cF+!G+%jh fm!LQeSX`0#S0t;Yj{$GGPIy98<$#HW{Q!OeWdCU# literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/dd229da166c3bec675e882d17092238cf7d245f3 b/test/core/end2end/fuzzers/api_fuzzer_corpus/dd229da166c3bec675e882d17092238cf7d245f3 new file mode 100644 index 0000000000000000000000000000000000000000..2a994d9ddaced7c7f44ef0e93d50634477a5f987 GIT binary patch literal 229 zcmZvWO$x#=5QSe_f53La zWG`fa2^QEKj2U-P+pa8}Yo%AVCNH&moJpFHw8x5YXXe{7%LM?z90tU50J1zuV>g^R z^rj5{uGPOXBkq*XE|i=L1z1Wu~Mu z^#A|=zh0P|hmnDSf${%;Ai==+tCN8ND8dZUo06KASejE3pIBOwkzbTqQVCS42>{PX BBgFs! literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/e709e8861c09e29cdae73e337587a63fb0ccf76d b/test/core/end2end/fuzzers/api_fuzzer_corpus/e709e8861c09e29cdae73e337587a63fb0ccf76d new file mode 100644 index 0000000000000000000000000000000000000000..a4815427911a633677da9cf7d6c9d71eeff54e57 GIT binary patch literal 86 zcmZQ7PAw`+En;9`Vc=rWPs`5-^BL2N3X=7Ti*w>j5{uGPOXBkq*XE|i=L1z1Wu~Mu j^#A|=zh0P|hmnDSf${%;Ai==+tCN8ND8kObz@-HM;|&_j literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/e839090caa1b1bfa8898eea683f5d5c9f1ed6dd1 b/test/core/end2end/fuzzers/api_fuzzer_corpus/e839090caa1b1bfa8898eea683f5d5c9f1ed6dd1 new file mode 100644 index 0000000000000000000000000000000000000000..af332baf9a0591c2ecd8d61d4fa1be6cff691264 GIT binary patch literal 200 zcmYMuOAZ1d3mv}yZQCW;q| zzyu3yb|HLf;o_-Zr*tscMn~Cr`#h2fJQ?=`;ZaJtFO?PmL@AsgUL&Bi?CLJJ!TZz) vKiGIRXIam380*OA=JCvzj5{uGPOXBkq*XE|i=L1z1Wu~Mu z^#A|=zh0P|hmnDSf${%;Ai==+tCN9&n}>-Rsy8(`KQAS*s4~73XkvU~dTL$?P_ZTe DLANCB literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/eacf4905e489566a3e5fcaaeac9fe91cbf916e06 b/test/core/end2end/fuzzers/api_fuzzer_corpus/eacf4905e489566a3e5fcaaeac9fe91cbf916e06 new file mode 100644 index 0000000000000000000000000000000000000000..44f3e00f9863c3b1a14fc6bbbb7824be2ce07d7d GIT binary patch literal 85 zcmZQ7PAw`+En;9`Vc=rWPs`5-^BL2N3X=7Ti*w>j5{uGPOXBkq*XE|i=L1z1Wu~Mu i^#A|=zh0P|hmnDSf${%;Ai==+tCN8ND8kObpa}rTU>d;y literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/ec55cbebe6db506acf7af9e5d26386630319623d b/test/core/end2end/fuzzers/api_fuzzer_corpus/ec55cbebe6db506acf7af9e5d26386630319623d new file mode 100644 index 0000000000000000000000000000000000000000..14ebab0cf35e33b520b8517b798b5c20b7f8f9dd GIT binary patch literal 87 zcmZQ7PAw`+En;9`Vc=rWPs`6Q267qGiwctUs*7{tOA?FHQ}YsYQ{(f2s){mGiW&O< l|F0M3=3!)DaA5rZpMim~lZ_k5;pJgsWngAt1Tq@;QQIUXn@-Bie$fp8+LJzTL z7R3b<5~|bHRYf?6cA~%(6;9Yyy_{*sb9TLW1ADG72H2mUvKzQw{fIBeosVN_5|BL&%g h1j}~_$~y@MN2Kuy#akJ3!lQjBA|ab|!py_J0^hRVX Date: Mon, 6 Feb 2017 19:30:09 +0100 Subject: [PATCH 036/167] Fix issue #9597 Generate consistently named PDBs for libraries across generators and install those PDBs if present. --- templates/CMakeLists.txt.template | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 444fb6a2dc3..52a060a8808 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -442,7 +442,19 @@ % endif % endfor ) - + + if(WIN32 AND MSVC) + set_target_properties(${lib.name} PROPERTIES COMPILE_PDB_NAME "${lib.name}" + COMPILE_PDB_OUTPUT_DIRECTORY <%text>"${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES <%text>${CMAKE_CURRENT_BINARY_DIR}/${lib.name}.pdb + DESTINATION <%text>${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() + endif() + + % for src in lib.src: % if proto_re.match(src): protobuf_generate_grpc_cpp( From d5923578c0c43a5716dd78c6467ccdd5199feff2 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 6 Feb 2017 11:25:59 -0800 Subject: [PATCH 037/167] nit fixes --- gRPC-Core.podspec | 1 + .../tests/CronetUnitTests/CronetUnitTests.m | 10 +++++----- templates/gRPC-Core.podspec.template | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 840d8132c66..0ef61442a4b 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -883,6 +883,7 @@ Pod::Spec.new do |s| 'test/core/end2end/end2end_test_utils.c', 'test/core/end2end/tests/*.{c,h}', 'test/core/end2end/data/*.{c,h}', + 'test/core/util/debugger_macros.c', 'test/core/util/test_config.{c,h}', 'test/core/util/port.h', 'test/core/util/port_posix.c', diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m index 9589926d8b5..408952a091b 100644 --- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m +++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m @@ -273,7 +273,7 @@ unsigned int parse_h2_length(const char *field) { grpc_completion_queue_destroy(cq); } -- (void)PacketCoalescing:(bool)use_coalescing { +- (void)packetCoalescing:(BOOL)use_coalescing { grpc_arg arg; arg.key = GRPC_ARG_USE_CRONET_PACKET_COALESCING; arg.type = GRPC_ARG_INTEGER; @@ -390,7 +390,7 @@ unsigned int parse_h2_length(const char *field) { char buf[4096]; long len; - bool coalesced = false; + BOOL coalesced = NO; while ((len = SSL_read(ssl, buf, sizeof(buf))) > 0) { gpr_log(GPR_DEBUG, "Read len: %ld", len); @@ -408,7 +408,7 @@ unsigned int parse_h2_length(const char *field) { (buf[p + 4] & 1) != 0 && // EOS bit is set 0 == memcmp("hello world", &buf[p + 14], 11)) { // Message is correct - coalesced = true; + coalesced = YES; break; } p += (parse_h2_length(&buf[p]) + 9); @@ -448,8 +448,8 @@ unsigned int parse_h2_length(const char *field) { } - (void)testPacketCoalescing { - [self PacketCoalescing:true]; - [self PacketCoalescing:false]; + [self packetCoalescing:YES]; + [self packetCoalescing:NO]; } @end diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index 0738b7221b5..0c1ff64a002 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -179,6 +179,7 @@ 'test/core/end2end/end2end_test_utils.c', 'test/core/end2end/tests/*.{c,h}', 'test/core/end2end/data/*.{c,h}', + 'test/core/util/debugger_macros.c', 'test/core/util/test_config.{c,h}', 'test/core/util/port.h', 'test/core/util/port_posix.c', From 93727aaaa767db37f1f30c1eec444b44091c4959 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Feb 2017 13:05:39 -0800 Subject: [PATCH 038/167] Fix memory leak when re-attempting send_status after it failed --- src/core/lib/surface/call.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 70bab4c0796..a8152964fc4 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -1483,6 +1483,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, } bctl->send_final_op = 1; call->sent_final_op = 1; + GPR_ASSERT(call->send_extra_metadata_count == 0); call->send_extra_metadata_count = 1; call->send_extra_metadata[0].md = grpc_channel_get_reffed_status_elem( exec_ctx, call->channel, op->data.send_status_from_server.status); @@ -1511,6 +1512,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, (int)op->data.send_status_from_server.trailing_metadata_count, op->data.send_status_from_server.trailing_metadata, 1, 1, NULL, 0)) { + for (int n = 0; n < call->send_extra_metadata_count; n++) { + GRPC_MDELEM_UNREF(exec_ctx, call->send_extra_metadata[n].md); + } + call->send_extra_metadata_count = 0; error = GRPC_CALL_ERROR_INVALID_METADATA; goto done_with_error; } From e1cf1b691d3e82742214c8772a71e79e3209c2ce Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 6 Feb 2017 13:31:05 -0800 Subject: [PATCH 039/167] nit fix 2 --- src/objective-c/tests/CronetUnitTests/CronetUnitTests.m | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m index 408952a091b..b6237c950bf 100644 --- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m +++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m @@ -273,11 +273,11 @@ unsigned int parse_h2_length(const char *field) { grpc_completion_queue_destroy(cq); } -- (void)packetCoalescing:(BOOL)use_coalescing { +- (void)packetCoalescing:(BOOL)useCoalescing { grpc_arg arg; arg.key = GRPC_ARG_USE_CRONET_PACKET_COALESCING; arg.type = GRPC_ARG_INTEGER; - arg.value.integer = use_coalescing ? 1 : 0; + arg.value.integer = useCoalescing ? 1 : 0; grpc_channel_args *args = grpc_channel_args_copy_and_add(NULL, &arg, 1); grpc_call *c; grpc_slice request_payload_slice = @@ -418,7 +418,7 @@ unsigned int parse_h2_length(const char *field) { } } - XCTAssert(coalesced == use_coalescing); + XCTAssert(coalesced == useCoalescing); SSL_free(ssl); SSL_CTX_free(ctx); close(s); From 5ff4bfb651ebb061d76e6b14af1d6b02050ffc97 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 27 Jan 2017 17:20:20 -0800 Subject: [PATCH 040/167] Advance protoc version to 3.1.0 --- src/objective-c/!ProtoCompiler.podspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index b55f6c93c62..dc4d8e964e6 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.0.2' + v = '3.1.0' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC From d81c6b6d0127ea4c649aa7e97a9d38a86ff0ade0 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Sat, 28 Jan 2017 20:34:05 -0800 Subject: [PATCH 041/167] Advance protoc dependency version as well --- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index bcc2bb61265..0c96784e685 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -98,7 +98,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.0.2' + s.dependency '!ProtoCompiler', '3.1.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.1' s.osx.deployment_target = '10.9' From 50bd995a9839ae26217dc6f8635b8aba9f379a83 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 6 Feb 2017 14:05:01 -0800 Subject: [PATCH 042/167] Advance BoringSSL version --- gRPC-Core.podspec | 2 +- src/objective-c/BoringSSL.podspec | 382 +++++++++++++-------------- templates/gRPC-Core.podspec.template | 2 +- 3 files changed, 190 insertions(+), 196 deletions(-) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 02336eec814..f4b32dea897 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -191,7 +191,7 @@ Pod::Spec.new do |s| ss.header_mappings_dir = '.' ss.libraries = 'z' ss.dependency "#{s.name}/Interface", version - ss.dependency 'BoringSSL', '~> 7.0' + ss.dependency 'BoringSSL', '~> 8.0' # To save you from scrolling, this is the last part of the podspec. ss.source_files = 'src/core/lib/profiling/timers.h', diff --git a/src/objective-c/BoringSSL.podspec b/src/objective-c/BoringSSL.podspec index 47b5b1a2e7b..908bb0b5e5e 100644 --- a/src/objective-c/BoringSSL.podspec +++ b/src/objective-c/BoringSSL.podspec @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.name = 'BoringSSL' - version = '7.0' + version = '8.0' s.version = version s.summary = 'BoringSSL is a fork of OpenSSL that is designed to meet Google’s needs.' # Adapted from the homepage: @@ -388,42 +388,42 @@ Pod::Spec.new do |s| 0x28340c19, 0x283480ac, 0x283500ea, - 0x2c322910, - 0x2c32a91e, - 0x2c332930, - 0x2c33a942, - 0x2c342956, - 0x2c34a968, - 0x2c352983, - 0x2c35a995, - 0x2c3629a8, + 0x2c3228ca, + 0x2c32a8d8, + 0x2c3328ea, + 0x2c33a8fc, + 0x2c342910, + 0x2c34a922, + 0x2c35293d, + 0x2c35a94f, + 0x2c362962, 0x2c36832d, - 0x2c3729b5, - 0x2c37a9c7, - 0x2c3829da, - 0x2c38a9f1, - 0x2c3929ff, - 0x2c39aa0f, - 0x2c3a2a21, - 0x2c3aaa35, - 0x2c3b2a46, - 0x2c3baa65, - 0x2c3c2a79, - 0x2c3caa8f, - 0x2c3d2aa8, - 0x2c3daac5, - 0x2c3e2ad6, - 0x2c3eaae4, - 0x2c3f2afc, - 0x2c3fab14, - 0x2c402b21, + 0x2c37296f, + 0x2c37a981, + 0x2c382994, + 0x2c38a9ab, + 0x2c3929b9, + 0x2c39a9c9, + 0x2c3a29db, + 0x2c3aa9ef, + 0x2c3b2a00, + 0x2c3baa1f, + 0x2c3c2a33, + 0x2c3caa49, + 0x2c3d2a62, + 0x2c3daa7f, + 0x2c3e2a90, + 0x2c3eaa9e, + 0x2c3f2ab6, + 0x2c3faace, + 0x2c402adb, 0x2c4090e7, - 0x2c412b32, - 0x2c41ab45, + 0x2c412aec, + 0x2c41aaff, 0x2c4210c0, - 0x2c42ab56, + 0x2c42ab10, 0x2c430720, - 0x2c43aa57, + 0x2c43aa11, 0x30320000, 0x30328015, 0x3033001f, @@ -639,74 +639,74 @@ Pod::Spec.new do |s| 0x405b1e9e, 0x405b9eaf, 0x405c1ec2, - 0x405c9ee3, - 0x405d1ef0, - 0x405d9f07, - 0x405e1f27, + 0x405c9ed3, + 0x405d1ee0, + 0x405d9ef7, + 0x405e1f17, 0x405e8a95, - 0x405f1f48, - 0x405f9f55, - 0x40601f63, - 0x40609f85, - 0x40611fad, - 0x40619fc2, - 0x40621fd9, - 0x40629fea, - 0x40631ffb, - 0x4063a010, - 0x40642027, - 0x4064a053, - 0x4065206e, - 0x4065a085, - 0x4066209d, - 0x4066a0c7, - 0x406720f2, - 0x4067a113, - 0x40682126, - 0x4068a147, - 0x40692179, - 0x4069a1a7, - 0x406a21c8, - 0x406aa1e8, - 0x406b2370, - 0x406ba393, - 0x406c23a9, - 0x406ca60b, - 0x406d263a, - 0x406da662, - 0x406e2690, - 0x406ea6a8, - 0x406f26c7, - 0x406fa6dc, - 0x407026ef, - 0x4070a70c, + 0x405f1f38, + 0x405f9f45, + 0x40601f53, + 0x40609f75, + 0x40611f9d, + 0x40619fb2, + 0x40621fc9, + 0x40629fda, + 0x40631feb, + 0x4063a000, + 0x40642017, + 0x4064a043, + 0x4065205e, + 0x4065a075, + 0x4066208d, + 0x4066a0b7, + 0x406720e2, + 0x4067a103, + 0x40682116, + 0x4068a137, + 0x40692169, + 0x4069a197, + 0x406a21b8, + 0x406aa1d8, + 0x406b2360, + 0x406ba383, + 0x406c2399, + 0x406ca5c5, + 0x406d25f4, + 0x406da61c, + 0x406e264a, + 0x406ea662, + 0x406f2681, + 0x406fa696, + 0x407026a9, + 0x4070a6c6, 0x40710800, - 0x4071a71e, - 0x40722731, - 0x4072a74a, - 0x40732762, + 0x4071a6d8, + 0x407226eb, + 0x4072a704, + 0x4073271c, 0x4073936d, - 0x40742776, - 0x4074a790, - 0x407527a1, - 0x4075a7b5, - 0x407627c3, + 0x40742730, + 0x4074a74a, + 0x4075275b, + 0x4075a76f, + 0x4076277d, 0x407691aa, - 0x407727e8, - 0x4077a80a, - 0x40782825, - 0x4078a85e, - 0x40792875, - 0x4079a88b, - 0x407a2897, - 0x407aa8aa, - 0x407b28bf, - 0x407ba8d1, - 0x407c28e6, - 0x407ca8ef, - 0x407d2162, + 0x407727a2, + 0x4077a7c4, + 0x407827df, + 0x4078a818, + 0x4079282f, + 0x4079a845, + 0x407a2851, + 0x407aa864, + 0x407b2879, + 0x407ba88b, + 0x407c28a0, + 0x407ca8a9, + 0x407d2152, 0x407d9c57, - 0x407e283a, + 0x407e27f4, 0x407e9e16, 0x407f1a67, 0x407f9887, @@ -714,45 +714,42 @@ Pod::Spec.new do |s| 0x40809a8f, 0x40811cd9, 0x40819c08, - 0x4082267b, + 0x40822635, 0x4082986d, 0x40831df1, - 0x4083a038, + 0x4083a028, 0x40841aa3, 0x40849e4e, - 0x40851ed3, - 0x41f4229b, - 0x41f9232d, - 0x41fe2220, - 0x41fea3fc, - 0x41ff24ed, - 0x420322b4, - 0x420822d6, - 0x4208a312, - 0x42092204, - 0x4209a34c, - 0x420a225b, - 0x420aa23b, - 0x420b227b, - 0x420ba2f4, - 0x420c2509, - 0x420ca3c9, - 0x420d23e3, - 0x420da41a, - 0x42122434, - 0x421724d0, - 0x4217a476, - 0x421c2498, - 0x421f2453, - 0x42212520, - 0x422624b3, - 0x422b25ef, - 0x422ba59d, - 0x422c25d7, - 0x422ca55c, - 0x422d253b, - 0x422da5bc, - 0x422e2582, + 0x41f4228b, + 0x41f9231d, + 0x41fe2210, + 0x41fea3ec, + 0x41ff24dd, + 0x420322a4, + 0x420822c6, + 0x4208a302, + 0x420921f4, + 0x4209a33c, + 0x420a224b, + 0x420aa22b, + 0x420b226b, + 0x420ba2e4, + 0x420c24f9, + 0x420ca3b9, + 0x420d23d3, + 0x420da40a, + 0x42122424, + 0x421724c0, + 0x4217a466, + 0x421c2488, + 0x421f2443, + 0x42212510, + 0x422624a3, + 0x422b25a9, + 0x422ba572, + 0x422c2591, + 0x422ca54c, + 0x422d252b, 0x4432072b, 0x4432873a, 0x44330746, @@ -795,69 +792,69 @@ Pod::Spec.new do |s| 0x4c3d136d, 0x4c3d937c, 0x4c3e1389, - 0x50322b68, - 0x5032ab77, - 0x50332b82, - 0x5033ab92, - 0x50342bab, - 0x5034abc5, - 0x50352bd3, - 0x5035abe9, - 0x50362bfb, - 0x5036ac11, - 0x50372c2a, - 0x5037ac3d, - 0x50382c55, - 0x5038ac66, - 0x50392c7b, - 0x5039ac8f, - 0x503a2caf, - 0x503aacc5, - 0x503b2cdd, - 0x503bacef, - 0x503c2d0b, - 0x503cad22, - 0x503d2d3b, - 0x503dad51, - 0x503e2d5e, - 0x503ead74, - 0x503f2d86, + 0x50322b22, + 0x5032ab31, + 0x50332b3c, + 0x5033ab4c, + 0x50342b65, + 0x5034ab7f, + 0x50352b8d, + 0x5035aba3, + 0x50362bb5, + 0x5036abcb, + 0x50372be4, + 0x5037abf7, + 0x50382c0f, + 0x5038ac20, + 0x50392c35, + 0x5039ac49, + 0x503a2c69, + 0x503aac7f, + 0x503b2c97, + 0x503baca9, + 0x503c2cc5, + 0x503cacdc, + 0x503d2cf5, + 0x503dad0b, + 0x503e2d18, + 0x503ead2e, + 0x503f2d40, 0x503f8382, - 0x50402d99, - 0x5040ada9, - 0x50412dc3, - 0x5041add2, - 0x50422dec, - 0x5042ae09, - 0x50432e19, - 0x5043ae29, - 0x50442e38, + 0x50402d53, + 0x5040ad63, + 0x50412d7d, + 0x5041ad8c, + 0x50422da6, + 0x5042adc3, + 0x50432dd3, + 0x5043ade3, + 0x50442df2, 0x5044843f, - 0x50452e4c, - 0x5045ae6a, - 0x50462e7d, - 0x5046ae93, - 0x50472ea5, - 0x5047aeba, - 0x50482ee0, - 0x5048aeee, - 0x50492f01, - 0x5049af16, - 0x504a2f2c, - 0x504aaf3c, - 0x504b2f5c, - 0x504baf6f, - 0x504c2f92, - 0x504cafc0, - 0x504d2fd2, - 0x504dafef, - 0x504e300a, - 0x504eb026, - 0x504f3038, - 0x504fb04f, - 0x5050305e, + 0x50452e06, + 0x5045ae24, + 0x50462e37, + 0x5046ae4d, + 0x50472e5f, + 0x5047ae74, + 0x50482e9a, + 0x5048aea8, + 0x50492ebb, + 0x5049aed0, + 0x504a2ee6, + 0x504aaef6, + 0x504b2f16, + 0x504baf29, + 0x504c2f4c, + 0x504caf7a, + 0x504d2f8c, + 0x504dafa9, + 0x504e2fc4, + 0x504eafe0, + 0x504f2ff2, + 0x504fb009, + 0x50503018, 0x505086ef, - 0x50513071, + 0x5051302b, 0x58320ec9, 0x68320e8b, 0x68328c25, @@ -1292,7 +1289,6 @@ Pod::Spec.new do |s| "NO_RENEGOTIATION\\0" "NO_REQUIRED_DIGEST\\0" "NO_SHARED_CIPHER\\0" - "NO_SHARED_GROUP\\0" "NULL_SSL_CTX\\0" "NULL_SSL_METHOD_PASSED\\0" "OLD_SESSION_CIPHER_NOT_RETURNED\\0" @@ -1353,9 +1349,7 @@ Pod::Spec.new do |s| "TLSV1_ALERT_USER_CANCELLED\\0" "TLSV1_BAD_CERTIFICATE_HASH_VALUE\\0" "TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE\\0" - "TLSV1_CERTIFICATE_REQUIRED\\0" "TLSV1_CERTIFICATE_UNOBTAINABLE\\0" - "TLSV1_UNKNOWN_PSK_IDENTITY\\0" "TLSV1_UNRECOGNIZED_NAME\\0" "TLSV1_UNSUPPORTED_EXTENSION\\0" "TLS_PEER_DID_NOT_RESPOND_WITH_CERTIFICATE_LIST\\0" diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index d43bcb93ef2..84720d2cd96 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -151,7 +151,7 @@ ss.header_mappings_dir = '.' ss.libraries = 'z' ss.dependency "#{s.name}/Interface", version - ss.dependency 'BoringSSL', '~> 7.0' + ss.dependency 'BoringSSL', '~> 8.0' # To save you from scrolling, this is the last part of the podspec. ss.source_files = ${ruby_multiline_list(grpc_private_files(libs), 22)} From cc08a4f24f26f649b6af10b45ae5a0d141fd118c Mon Sep 17 00:00:00 2001 From: Piotr Pietraszkiewicz Date: Tue, 7 Feb 2017 11:02:58 +0100 Subject: [PATCH 043/167] Fix issue #9602 This is achieved by putting `grpc++_codegegen_proto` on the `filegroup` of `grpc++`. --- build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/build.yaml b/build.yaml index 456db28cef4..a453e6b4373 100644 --- a/build.yaml +++ b/build.yaml @@ -1063,6 +1063,7 @@ libs: filegroups: - grpc++_base - grpc++_codegen_base + - grpc++_codegen_proto - grpc++_codegen_base_src secure: check vs_project_guid: '{C187A093-A0FE-489D-A40A-6E33DE0F9FEB}' From 78e6451dae37a91d9a77941bf836776a1807cad0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Feb 2017 07:03:26 -0800 Subject: [PATCH 044/167] Add a hook to disable BDP probing --- include/grpc/impl/codegen/grpc_types.h | 2 + .../chttp2/transport/chttp2_transport.c | 58 +++++++++++-------- .../ext/transport/chttp2/transport/internal.h | 3 + 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 7fbaf56e75b..bc0b2bd7f86 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -179,6 +179,8 @@ typedef struct { Larger values give lower CPU usage for large messages, but more head of line blocking for small messages. */ #define GRPC_ARG_HTTP2_MAX_FRAME_SIZE "grpc.http2.max_frame_size" +/** Should BDP probing be performed? */ +#define GRPC_ARG_HTTP2_BDP_PROBE "grpc.http2.bdp_probe" /** Minimum time (in milliseconds) between successive ping frames being sent */ #define GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS \ "grpc.http2.min_time_between_pings_ms" diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index fa18f5a725a..8a9eaa8b6a2 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -292,6 +292,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->force_send_settings = 1 << GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; t->sent_local_settings = 0; t->write_buffer_size = DEFAULT_WINDOW; + t->enable_bdp_probe = true; if (is_client) { grpc_slice_buffer_add(&t->outbuf, grpc_slice_from_copied_string( @@ -358,6 +359,10 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->write_buffer_size = (uint32_t)grpc_channel_arg_get_integer( &channel_args->args[i], (grpc_integer_options){0, 0, MAX_WRITE_BUFFER_SIZE}); + } else if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_BDP_PROBE)) { + t->enable_bdp_probe = grpc_channel_arg_get_integer( + &channel_args->args[i], (grpc_integer_options){1, 0, 1}); } else { static const struct { const char *channel_arg_name; @@ -1908,33 +1913,36 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_endpoint_read(exec_ctx, t->ep, &t->read_buffer, &t->read_action_locked); - if (need_bdp_ping) { - GRPC_CHTTP2_REF_TRANSPORT(t, "bdp_ping"); - grpc_bdp_estimator_schedule_ping(&t->bdp_estimator); - send_ping_locked(exec_ctx, t, - GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE, - &t->start_bdp_ping_locked, &t->finish_bdp_ping_locked); - } - - int64_t estimate = -1; - if (grpc_bdp_estimator_get_estimate(&t->bdp_estimator, &estimate)) { - double target = 1 + log2((double)estimate); - double memory_pressure = grpc_resource_quota_get_memory_pressure( - grpc_resource_user_quota(grpc_endpoint_get_resource_user(t->ep))); - if (memory_pressure > 0.8) { - target *= 1 - GPR_MIN(1, (memory_pressure - 0.8) / 0.1); + if (t->enable_bdp_probe) { + if (need_bdp_ping) { + GRPC_CHTTP2_REF_TRANSPORT(t, "bdp_ping"); + grpc_bdp_estimator_schedule_ping(&t->bdp_estimator); + send_ping_locked(exec_ctx, t, + GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE, + &t->start_bdp_ping_locked, &t->finish_bdp_ping_locked); } - double bdp_error = target - grpc_pid_controller_last(&t->pid_controller); - gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); - gpr_timespec dt_timespec = gpr_time_sub(now, t->last_pid_update); - double dt = (double)dt_timespec.tv_sec + dt_timespec.tv_nsec * 1e-9; - if (dt > 0.1) { - dt = 0.1; + + int64_t estimate = -1; + if (grpc_bdp_estimator_get_estimate(&t->bdp_estimator, &estimate)) { + double target = 1 + log2((double)estimate); + double memory_pressure = grpc_resource_quota_get_memory_pressure( + grpc_resource_user_quota(grpc_endpoint_get_resource_user(t->ep))); + if (memory_pressure > 0.8) { + target *= 1 - GPR_MIN(1, (memory_pressure - 0.8) / 0.1); + } + double bdp_error = + target - grpc_pid_controller_last(&t->pid_controller); + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + gpr_timespec dt_timespec = gpr_time_sub(now, t->last_pid_update); + double dt = (double)dt_timespec.tv_sec + dt_timespec.tv_nsec * 1e-9; + if (dt > 0.1) { + dt = 0.1; + } + double log2_bdp_guess = + grpc_pid_controller_update(&t->pid_controller, bdp_error, dt); + update_bdp(exec_ctx, t, pow(2, log2_bdp_guess)); + t->last_pid_update = now; } - double log2_bdp_guess = - grpc_pid_controller_update(&t->pid_controller, bdp_error, dt); - update_bdp(exec_ctx, t, pow(2, log2_bdp_guess)); - t->last_pid_update = now; } GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "keep_reading"); } else { diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 1dabf9edba2..075d421dd46 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -227,6 +227,9 @@ struct grpc_chttp2_transport { /** is there a read request to the endpoint outstanding? */ uint8_t endpoint_reading; + /** should we probe bdp? */ + bool enable_bdp_probe; + /** various lists of streams */ grpc_chttp2_stream_list lists[STREAM_LIST_COUNT]; From 49124e7ae16bd7423a1fa238181743a797d72437 Mon Sep 17 00:00:00 2001 From: Robbie Shade Date: Tue, 7 Feb 2017 10:42:22 -0500 Subject: [PATCH 045/167] Adds an exec_ctx argument to orphan_cb in udp_server --- src/core/lib/iomgr/udp_server.c | 4 ++-- src/core/lib/iomgr/udp_server.h | 3 ++- test/core/iomgr/udp_server_test.c | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index 02a194c9825..2a1c8d39fae 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -178,7 +178,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { /* Call the orphan_cb to signal that the FD is about to be closed and * should no longer be used. */ GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(sp->emfd); + sp->orphan_cb(exec_ctx, sp->emfd); grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL, "udp_listener_shutdown"); @@ -204,7 +204,7 @@ void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, if (s->active_ports) { for (sp = s->head; sp; sp = sp->next) { GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(sp->emfd); + sp->orphan_cb(exec_ctx, sp->emfd); grpc_fd_shutdown(exec_ctx, sp->emfd, GRPC_ERROR_CREATE("Server destroyed")); } diff --git a/src/core/lib/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h index ce068cbf04c..ed63fa7d817 100644 --- a/src/core/lib/iomgr/udp_server.h +++ b/src/core/lib/iomgr/udp_server.h @@ -54,7 +54,8 @@ typedef void (*grpc_udp_server_write_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd); /* Called when the grpc_fd is about to be orphaned (and the FD closed). */ -typedef void (*grpc_udp_server_orphan_cb)(grpc_fd *emfd); +typedef void (*grpc_udp_server_orphan_cb)(grpc_exec_ctx *exec_ctx, + grpc_fd *emfd); /* Create a server, initially not bound to any ports */ grpc_udp_server *grpc_udp_server_create(void); diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index ba7a52ea21c..ba572017a22 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -88,7 +88,7 @@ static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd) { gpr_mu_unlock(g_mu); } -static void on_fd_orphaned(grpc_fd *emfd) { +static void on_fd_orphaned(grpc_exec_ctx *exec_ctx, grpc_fd *emfd) { gpr_log(GPR_INFO, "gRPC FD about to be orphaned: %d", grpc_fd_wrapped_fd(emfd)); g_number_of_orphan_calls++; From 5f3a1430682976f94bccc58719a08a45ca493ff6 Mon Sep 17 00:00:00 2001 From: Carl Mastrangelo Date: Tue, 7 Feb 2017 08:14:29 -0800 Subject: [PATCH 046/167] remove extra space in Csharp code --- src/csharp/ext/grpc_csharp_ext.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/ext/grpc_csharp_ext.c b/src/csharp/ext/grpc_csharp_ext.c index e1f4d7cdf31..6a241190b27 100644 --- a/src/csharp/ext/grpc_csharp_ext.c +++ b/src/csharp/ext/grpc_csharp_ext.c @@ -761,7 +761,7 @@ GPR_EXPORT grpc_call_error GPR_CALLTYPE grpcsharp_call_send_status_from_server( ops[nops].data.send_message.send_message = ctx->send_message; ops[nops].flags = write_flags; ops[nops].reserved = NULL; - nops ++; + nops++; } if (send_empty_initial_metadata) { ops[nops].op = GRPC_OP_SEND_INITIAL_METADATA; From b038beb7245877b71e30e782ddd6c86d82d11d17 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Feb 2017 08:28:51 -0800 Subject: [PATCH 047/167] Add counters for mutex acquisitions, expose in bm_fullstack --- Makefile | 8 +++++ build.yaml | 3 ++ src/core/lib/support/sync_posix.c | 7 ++++ test/cpp/microbenchmarks/bm_fullstack.cc | 46 +++++++++++++++++++----- tools/run_tests/generated/configs.json | 3 ++ tools/run_tests/generated/tests.json | 34 ++++++++++++++++++ 6 files changed, 92 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 0413cc0e009..3d8f7e6a432 100644 --- a/Makefile +++ b/Makefile @@ -212,6 +212,14 @@ CPPFLAGS_mutrace = -O3 -fno-omit-frame-pointer LDFLAGS_mutrace = -rdynamic DEFINES_mutrace = NDEBUG +VALID_CONFIG_counters = 1 +CC_counters = $(DEFAULT_CC) +CXX_counters = $(DEFAULT_CXX) +LD_counters = $(DEFAULT_CC) +LDXX_counters = $(DEFAULT_CXX) +CPPFLAGS_counters = -O2 -DGPR_MU_COUNTERS +DEFINES_counters = NDEBUG + # General settings. diff --git a/build.yaml b/build.yaml index 456db28cef4..a13ba05b990 100644 --- a/build.yaml +++ b/build.yaml @@ -3825,6 +3825,9 @@ configs: basicprof: CPPFLAGS: -O2 -DGRPC_BASIC_PROFILER -DGRPC_TIMERS_RDTSC DEFINES: NDEBUG + counters: + CPPFLAGS: -O2 -DGPR_MU_COUNTERS + DEFINES: NDEBUG dbg: CPPFLAGS: -O0 DEFINES: _DEBUG DEBUG diff --git a/src/core/lib/support/sync_posix.c b/src/core/lib/support/sync_posix.c index dcb0969a4ea..de0f0484b57 100644 --- a/src/core/lib/support/sync_posix.c +++ b/src/core/lib/support/sync_posix.c @@ -42,11 +42,18 @@ #include #include "src/core/lib/profiling/timers.h" +#ifdef GPR_MU_COUNTERS +gpr_atm grpc_mu_locks = 0; +#endif + void gpr_mu_init(gpr_mu* mu) { GPR_ASSERT(pthread_mutex_init(mu, NULL) == 0); } void gpr_mu_destroy(gpr_mu* mu) { GPR_ASSERT(pthread_mutex_destroy(mu) == 0); } void gpr_mu_lock(gpr_mu* mu) { +#ifdef GPR_MU_COUNTERS + gpr_atm_no_barrier_fetch_add(&grpc_mu_locks, 1); +#endif GPR_TIMER_BEGIN("gpr_mu_lock", 0); GPR_ASSERT(pthread_mutex_lock(mu) == 0); GPR_TIMER_END("gpr_mu_lock", 0); diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index c3e96c572c8..c5ace53ff9b 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -94,7 +94,37 @@ static void ApplyCommonChannelArguments(ChannelArguments* c) { c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX); } -class FullstackFixture { +#ifdef GPR_MU_COUNTERS +extern "C" gpr_atm grpc_mu_locks; +#endif + +class BaseFixture { + public: + void Finish(benchmark::State& s) { + std::ostringstream out; + this->AddToLabel(out, s); +#ifdef GPR_MU_COUNTERS + out << " locks/iteration:" + << ((double)(gpr_atm_no_barrier_load(&grpc_mu_locks) - + mu_locks_at_start_) / + (double)s.iterations()); +#endif + auto label = out.str(); + if (label.length() && label[0] == ' ') { + label = label.substr(1); + } + s.SetLabel(label); + } + + virtual void AddToLabel(std::ostream& out, benchmark::State& s) = 0; + + private: +#ifdef GPR_MU_COUNTERS + const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&grpc_mu_locks); +#endif +}; + +class FullstackFixture : public BaseFixture { public: FullstackFixture(Service* service, const grpc::string& address) { ServerBuilder b; @@ -130,7 +160,7 @@ class TCP : public FullstackFixture { public: TCP(Service* service) : FullstackFixture(service, MakeAddress()) {} - void Finish(benchmark::State& state) {} + void AddToLabel(std::ostream& out, benchmark::State& state) {} private: static grpc::string MakeAddress() { @@ -145,7 +175,7 @@ class UDS : public FullstackFixture { public: UDS(Service* service) : FullstackFixture(service, MakeAddress()) {} - void Finish(benchmark::State& state) {} + void AddToLabel(std::ostream& out, benchmark::State& state) override {} private: static grpc::string MakeAddress() { @@ -157,7 +187,7 @@ class UDS : public FullstackFixture { } }; -class EndpointPairFixture { +class EndpointPairFixture : public BaseFixture { public: EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) { ServerBuilder b; @@ -233,7 +263,7 @@ class SockPair : public EndpointPairFixture { "test", initialize_stuff.rq(), 8192)) { } - void Finish(benchmark::State& state) {} + void AddToLabel(std::ostream& out, benchmark::State& state) {} }; class InProcessCHTTP2 : public EndpointPairFixture { @@ -241,11 +271,9 @@ class InProcessCHTTP2 : public EndpointPairFixture { InProcessCHTTP2(Service* service) : EndpointPairFixture(service, MakeEndpoints()) {} - void Finish(benchmark::State& state) { - std::ostringstream out; - out << "writes/iteration:" + void AddToLabel(std::ostream& out, benchmark::State& state) { + out << " writes/iteration:" << ((double)stats_.num_writes / (double)state.iterations()); - state.SetLabel(out.str()); } private: diff --git a/tools/run_tests/generated/configs.json b/tools/run_tests/generated/configs.json index 091f5d98239..9173bd7c19c 100644 --- a/tools/run_tests/generated/configs.json +++ b/tools/run_tests/generated/configs.json @@ -66,5 +66,8 @@ }, { "config": "mutrace" + }, + { + "config": "counters" } ] diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 4cc543962e4..34f777de166 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -39242,6 +39242,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39277,6 +39278,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39312,6 +39314,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39347,6 +39350,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39384,6 +39388,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39421,6 +39426,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39456,6 +39462,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39491,6 +39498,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39526,6 +39534,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39561,6 +39570,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39596,6 +39606,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39631,6 +39642,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39666,6 +39678,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39701,6 +39714,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39736,6 +39750,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39771,6 +39786,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39806,6 +39822,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39841,6 +39858,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39876,6 +39894,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39911,6 +39930,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39946,6 +39966,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -39983,6 +40004,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40020,6 +40042,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40055,6 +40078,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40090,6 +40114,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40125,6 +40150,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40160,6 +40186,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40195,6 +40222,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40230,6 +40258,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40265,6 +40294,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40300,6 +40330,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40335,6 +40366,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40370,6 +40402,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", @@ -40405,6 +40438,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "counters", "dbg", "gcov", "helgrind", From 01d7d9b2e52e5945616610025527908c4101df03 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Feb 2017 08:39:05 -0800 Subject: [PATCH 048/167] Add allocations/iteration counters to bm_fullstack --- test/cpp/microbenchmarks/bm_fullstack.cc | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index c5ace53ff9b..32fd5351659 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -54,6 +54,7 @@ extern "C" { #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/completion_queue.h" #include "src/core/lib/surface/server.h" +#include "test/core/util/memory_counters.h" #include "test/core/util/passthru_endpoint.h" #include "test/core/util/port.h" } @@ -67,6 +68,7 @@ namespace testing { static class InitializeStuff { public: InitializeStuff() { + grpc_memory_counters_init(); init_lib_.init(); rq_ = grpc_resource_quota_create("bm"); } @@ -104,11 +106,15 @@ class BaseFixture { std::ostringstream out; this->AddToLabel(out, s); #ifdef GPR_MU_COUNTERS - out << " locks/iteration:" - << ((double)(gpr_atm_no_barrier_load(&grpc_mu_locks) - - mu_locks_at_start_) / - (double)s.iterations()); + out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&grpc_mu_locks) - + mu_locks_at_start_) / + (double)s.iterations()); #endif + grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); + out << " allocs/iter:" + << ((double)(counters_at_end.total_allocs_absolute - + counters_at_start_.total_allocs_absolute) / + (double)s.iterations()); auto label = out.str(); if (label.length() && label[0] == ' ') { label = label.substr(1); @@ -122,6 +128,7 @@ class BaseFixture { #ifdef GPR_MU_COUNTERS const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&grpc_mu_locks); #endif + grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot(); }; class FullstackFixture : public BaseFixture { @@ -272,7 +279,7 @@ class InProcessCHTTP2 : public EndpointPairFixture { : EndpointPairFixture(service, MakeEndpoints()) {} void AddToLabel(std::ostream& out, benchmark::State& state) { - out << " writes/iteration:" + out << " writes/iter:" << ((double)stats_.num_writes / (double)state.iterations()); } From 6911d08388a7958e159d96481770caf8e7bb8a9e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Feb 2017 10:30:44 -0800 Subject: [PATCH 049/167] Fix platform_string() usage, cut temporary space usage --- .../python_utils/start_port_server.py | 3 ++- tools/run_tests/run_microbenchmark.py | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/python_utils/start_port_server.py b/tools/run_tests/python_utils/start_port_server.py index 4103eb0534b..d521aa6a9d9 100644 --- a/tools/run_tests/python_utils/start_port_server.py +++ b/tools/run_tests/python_utils/start_port_server.py @@ -35,6 +35,7 @@ import subprocess import tempfile import sys import time +import jobset def start_port_server(port_server_port): # check if a compatible port server is running @@ -69,7 +70,7 @@ def start_port_server(port_server_port): '-p', '%d' % port_server_port, '-l', logfile] env = dict(os.environ) env['BUILD_ID'] = 'pleaseDontKillMeJenkins' - if platform_string() == 'windows': + if jobset.platform_string() == 'windows': # Working directory of port server needs to be outside of Jenkins # workspace to prevent file lock issues. tempdir = tempfile.mkdtemp() diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 42a31a622f7..d51388bbf05 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -74,6 +74,7 @@ def link(txt, tgt): benchmarks = [] profile_analysis = [] +cleanup = [] for bm_name in sys.argv[1:]: # generate latency profiles @@ -92,10 +93,20 @@ for bm_name in sys.argv[1:]: 'tools/profiling/latency_profile/profile_analyzer.py', '--source', '%s.trace' % fnize(line), '--fmt', 'simple', '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None)) - - jobset.run(benchmarks, maxjobs=multiprocessing.cpu_count()/2, - add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) - jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) + cleanup.append('rm', '%s.trace' % fnize(line)) + if len(benchmarks) >= 2 * multiprocessing.cpu_count(): + jobset.run(benchmarks, maxjobs=multiprocessing.cpu_count()/2, + add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) + jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) + benchmarks = [] + profile_analysis = [] + cleanup = [] + if len(benchmarks): + jobset.run(benchmarks, maxjobs=multiprocessing.cpu_count()/2, + add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) + jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) # generate flamegraphs heading('Flamegraphs: %s' % bm_name) From 715e43bc81227d86f6a8a102aab488d47db801f4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Feb 2017 11:13:16 -0800 Subject: [PATCH 050/167] Make a jobspec --- tools/run_tests/run_microbenchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index d51388bbf05..852b83967a9 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -93,7 +93,7 @@ for bm_name in sys.argv[1:]: 'tools/profiling/latency_profile/profile_analyzer.py', '--source', '%s.trace' % fnize(line), '--fmt', 'simple', '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None)) - cleanup.append('rm', '%s.trace' % fnize(line)) + cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)])) if len(benchmarks) >= 2 * multiprocessing.cpu_count(): jobset.run(benchmarks, maxjobs=multiprocessing.cpu_count()/2, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) From 0a8e32ab7f9666c45fb5651cec10f61dc538a79d Mon Sep 17 00:00:00 2001 From: ncteisen Date: Tue, 7 Feb 2017 12:25:19 -0800 Subject: [PATCH 051/167] Add more verbose status msg to rst_stream case --- .../ext/transport/chttp2/transport/frame_rst_stream.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index 7d5beed09dd..2e93ae2b857 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -109,8 +109,14 @@ grpc_error *grpc_chttp2_rst_stream_parser_parse(grpc_exec_ctx *exec_ctx, (((uint32_t)p->reason_bytes[3])); grpc_error *error = GRPC_ERROR_NONE; if (reason != GRPC_HTTP2_NO_ERROR || s->header_frames_received < 2) { - error = grpc_error_set_int(GRPC_ERROR_CREATE("RST_STREAM"), - GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); + char *message; + gpr_asprintf(&message, "Received RST_STREAM with error code %d", + reason); + error = grpc_error_set_int( + grpc_error_set_str(GRPC_ERROR_CREATE("RST_STREAM"), + GRPC_ERROR_STR_GRPC_MESSAGE, message), + GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); + gpr_free(message); } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, error); } From dc9bee7ce4dd64392acd62b447e06b1cd0f4cd6b Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 7 Feb 2017 12:29:14 -0800 Subject: [PATCH 052/167] Add proxy mapper method for overriding the name to resolve. --- src/core/ext/client_channel/client_channel.c | 44 +++++----------- .../client_channel/client_channel_plugin.c | 2 + src/core/ext/client_channel/http_proxy.c | 51 +++++++++++++++++- src/core/ext/client_channel/http_proxy.h | 4 +- src/core/ext/client_channel/proxy_mapper.c | 25 ++++++--- src/core/ext/client_channel/proxy_mapper.h | 36 +++++++++---- .../client_channel/proxy_mapper_registry.c | 52 +++++++++++++------ .../client_channel/proxy_mapper_registry.h | 16 ++++-- src/core/ext/client_channel/subchannel.c | 4 +- 9 files changed, 159 insertions(+), 75 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 4ab467329a6..06038bb5ba9 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -44,8 +44,8 @@ #include #include "src/core/ext/client_channel/http_connect_handshaker.h" -#include "src/core/ext/client_channel/http_proxy.h" #include "src/core/ext/client_channel/lb_policy_registry.h" +#include "src/core/ext/client_channel/proxy_mapper_registry.h" #include "src/core/ext/client_channel/resolver_registry.h" #include "src/core/ext/client_channel/subchannel.h" #include "src/core/lib/channel/channel_args.h" @@ -153,10 +153,6 @@ static void *method_parameters_create_from_json(const grpc_json *json) { */ typedef struct client_channel_channel_data { - /** server name */ - char *server_name; - /** HTTP CONNECT proxy to use, if any */ - char *proxy_name; /** resolver for this channel */ grpc_resolver *resolver; /** have we started resolving this channel */ @@ -317,17 +313,6 @@ static void on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg, // Use pick_first if nothing was specified and we didn't select grpclb // above. if (lb_policy_name == NULL) lb_policy_name = "pick_first"; - // If using a proxy, add channel arg for server in HTTP CONNECT request. - if (chand->proxy_name != NULL) { - grpc_arg new_arg; - new_arg.key = GRPC_ARG_HTTP_CONNECT_SERVER; - new_arg.type = GRPC_ARG_STRING; - new_arg.value.string = chand->server_name; - grpc_channel_args *tmp_args = chand->resolver_result; - chand->resolver_result = - grpc_channel_args_copy_and_add(chand->resolver_result, &new_arg, 1); - grpc_channel_args_destroy(exec_ctx, tmp_args); - } // Instantiate LB policy. grpc_lb_policy_args lb_policy_args; lb_policy_args.args = chand->resolver_result; @@ -542,24 +527,21 @@ static grpc_error *cc_init_channel_elem(grpc_exec_ctx *exec_ctx, GPR_ASSERT(arg->type == GRPC_ARG_POINTER); grpc_client_channel_factory_ref(arg->value.pointer.p); chand->client_channel_factory = arg->value.pointer.p; - // Instantiate resolver. + // Get server name to resolve, using proxy mapper if needed. arg = grpc_channel_args_find(args->channel_args, GRPC_ARG_SERVER_URI); GPR_ASSERT(arg != NULL); GPR_ASSERT(arg->type == GRPC_ARG_STRING); - grpc_uri *uri = grpc_uri_parse(arg->value.string, true); - if (uri == NULL) return GRPC_ERROR_CREATE("cannot parse server URI"); - if (uri->path[0] == '\0') { - grpc_uri_destroy(uri); - return GRPC_ERROR_CREATE("server URI is missing path"); - } - chand->server_name = - gpr_strdup(uri->path[0] == '/' ? uri->path + 1 : uri->path); - grpc_uri_destroy(uri); - chand->proxy_name = grpc_get_http_proxy_server(); - char *name_to_resolve = - chand->proxy_name == NULL ? arg->value.string : chand->proxy_name; + char *proxy_name = NULL; + grpc_channel_args *new_args = NULL; + grpc_proxy_mappers_map_name(exec_ctx, arg->value.string, args->channel_args, + &proxy_name, &new_args); + // Instantiate resolver. chand->resolver = grpc_resolver_create( - exec_ctx, name_to_resolve, args->channel_args, chand->interested_parties); + exec_ctx, proxy_name != NULL ? proxy_name : arg->value.string, + new_args != NULL ? new_args : args->channel_args, + chand->interested_parties); + if (proxy_name != NULL) gpr_free(proxy_name); + if (new_args != NULL) grpc_channel_args_destroy(exec_ctx, new_args); if (chand->resolver == NULL) { return GRPC_ERROR_CREATE("resolver creation failed"); } @@ -570,8 +552,6 @@ static grpc_error *cc_init_channel_elem(grpc_exec_ctx *exec_ctx, static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem) { channel_data *chand = elem->channel_data; - gpr_free(chand->server_name); - gpr_free(chand->proxy_name); if (chand->resolver != NULL) { grpc_resolver_shutdown(exec_ctx, chand->resolver); GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel"); diff --git a/src/core/ext/client_channel/client_channel_plugin.c b/src/core/ext/client_channel/client_channel_plugin.c index 7f75233727e..6f9df3e386e 100644 --- a/src/core/ext/client_channel/client_channel_plugin.c +++ b/src/core/ext/client_channel/client_channel_plugin.c @@ -39,6 +39,7 @@ #include "src/core/ext/client_channel/client_channel.h" #include "src/core/ext/client_channel/http_connect_handshaker.h" +#include "src/core/ext/client_channel/http_proxy.h" #include "src/core/ext/client_channel/lb_policy_registry.h" #include "src/core/ext/client_channel/proxy_mapper_registry.h" #include "src/core/ext/client_channel/resolver_registry.h" @@ -82,6 +83,7 @@ void grpc_client_channel_init(void) { grpc_lb_policy_registry_init(); grpc_resolver_registry_init(); grpc_proxy_mapper_registry_init(); + grpc_register_http_proxy_mapper(); grpc_subchannel_index_init(); grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MIN, set_default_host_if_unset, NULL); diff --git a/src/core/ext/client_channel/http_proxy.c b/src/core/ext/client_channel/http_proxy.c index 9a6c818c4ee..7daa0714951 100644 --- a/src/core/ext/client_channel/http_proxy.c +++ b/src/core/ext/client_channel/http_proxy.c @@ -40,10 +40,13 @@ #include #include +#include "src/core/ext/client_channel/http_connect_handshaker.h" +#include "src/core/ext/client_channel/proxy_mapper_registry.h" #include "src/core/ext/client_channel/uri_parser.h" +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/support/env.h" -char* grpc_get_http_proxy_server() { +static char* grpc_get_http_proxy_server() { char* uri_str = gpr_getenv("http_proxy"); if (uri_str == NULL) return NULL; grpc_uri* uri = grpc_uri_parse(uri_str, false /* suppress_errors */); @@ -66,3 +69,49 @@ done: grpc_uri_destroy(uri); return proxy_name; } + +static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args) { + *name_to_resolve = grpc_get_http_proxy_server(); + if (*name_to_resolve == NULL) return false; + grpc_uri* uri = grpc_uri_parse(server_uri, false /* suppress_errors */); + if (uri == NULL || uri->path[0] == '\0') { + gpr_log(GPR_ERROR, + "'http_proxy' environment variable set, but cannot " + "parse server URI '%s' -- not using proxy", + server_uri); + if (uri != NULL) grpc_uri_destroy(uri); + return false; + } + grpc_arg new_arg; + new_arg.key = GRPC_ARG_HTTP_CONNECT_SERVER; + new_arg.type = GRPC_ARG_STRING; + new_arg.value.string = uri->path[0] == '/' ? uri->path + 1 : uri->path; + *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1); + grpc_uri_destroy(uri); + return true; +} + +static bool proxy_mapper_map_address(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args) { + return false; +} + +static void proxy_mapper_destroy(grpc_proxy_mapper* mapper) {} + +static const grpc_proxy_mapper_vtable proxy_mapper_vtable = { + proxy_mapper_map_name, proxy_mapper_map_address, proxy_mapper_destroy}; + +static grpc_proxy_mapper proxy_mapper = {&proxy_mapper_vtable}; + +void grpc_register_http_proxy_mapper() { + grpc_proxy_mapper_register(true /* at_start */, &proxy_mapper); +} diff --git a/src/core/ext/client_channel/http_proxy.h b/src/core/ext/client_channel/http_proxy.h index 0d77ae253b0..c8882b1ef18 100644 --- a/src/core/ext/client_channel/http_proxy.h +++ b/src/core/ext/client_channel/http_proxy.h @@ -34,8 +34,6 @@ #ifndef GRPC_CORE_EXT_CLIENT_CHANNEL_HTTP_PROXY_H #define GRPC_CORE_EXT_CLIENT_CHANNEL_HTTP_PROXY_H -/// Returns the name of the proxy to use, or NULL if no proxy is configured. -/// Caller takes ownership of result. -char* grpc_get_http_proxy_server(); +void grpc_register_http_proxy_mapper(); #endif /* GRPC_CORE_EXT_CLIENT_CHANNEL_HTTP_PROXY_H */ diff --git a/src/core/ext/client_channel/proxy_mapper.c b/src/core/ext/client_channel/proxy_mapper.c index 6b6f328d3ce..f92afe847bf 100644 --- a/src/core/ext/client_channel/proxy_mapper.c +++ b/src/core/ext/client_channel/proxy_mapper.c @@ -38,13 +38,24 @@ void grpc_proxy_mapper_init(const grpc_proxy_mapper_vtable* vtable, mapper->vtable = vtable; } -bool grpc_proxy_mapper_map(grpc_exec_ctx* exec_ctx, grpc_proxy_mapper* mapper, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args) { - return mapper->vtable->map(exec_ctx, mapper, address, args, new_address, - new_args); +bool grpc_proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args) { + return mapper->vtable->map_name(exec_ctx, mapper, server_uri, args, + name_to_resolve, new_args); +} + +bool grpc_proxy_mapper_map_address(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args) { + return mapper->vtable->map_address(exec_ctx, mapper, address, args, + new_address, new_args); } void grpc_proxy_mapper_destroy(grpc_proxy_mapper* mapper) { diff --git a/src/core/ext/client_channel/proxy_mapper.h b/src/core/ext/client_channel/proxy_mapper.h index fa930379e70..6e4607fe4d9 100644 --- a/src/core/ext/client_channel/proxy_mapper.h +++ b/src/core/ext/client_channel/proxy_mapper.h @@ -43,15 +43,22 @@ typedef struct grpc_proxy_mapper grpc_proxy_mapper; typedef struct { + /// Determines the proxy name to resolve for \a server_uri. + /// If no proxy is needed, returns false. + /// Otherwise, sets \a name_to_resolve, optionally sets \a new_args, + /// and returns true. + bool (*map_name)(grpc_exec_ctx* exec_ctx, grpc_proxy_mapper* mapper, + const char* server_uri, const grpc_channel_args* args, + char** name_to_resolve, grpc_channel_args** new_args); /// Determines the proxy address to use to contact \a address. /// If no proxy is needed, returns false. /// Otherwise, sets \a new_address, optionally sets \a new_args, and /// returns true. - bool (*map)(grpc_exec_ctx* exec_ctx, grpc_proxy_mapper* mapper, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args); + bool (*map_address)(grpc_exec_ctx* exec_ctx, grpc_proxy_mapper* mapper, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args); /// Destroys \a mapper. void (*destroy)(grpc_proxy_mapper* mapper); } grpc_proxy_mapper_vtable; @@ -63,11 +70,20 @@ struct grpc_proxy_mapper { void grpc_proxy_mapper_init(const grpc_proxy_mapper_vtable* vtable, grpc_proxy_mapper* mapper); -bool grpc_proxy_mapper_map(grpc_exec_ctx* exec_ctx, grpc_proxy_mapper* mapper, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args); +bool grpc_proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args); + +bool grpc_proxy_mapper_map_address(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args); + void grpc_proxy_mapper_destroy(grpc_proxy_mapper* mapper); #endif /* GRPC_CORE_EXT_CLIENT_CHANNEL_PROXY_MAPPER_H */ diff --git a/src/core/ext/client_channel/proxy_mapper_registry.c b/src/core/ext/client_channel/proxy_mapper_registry.c index 0a156c8b1c1..2c44b9d4903 100644 --- a/src/core/ext/client_channel/proxy_mapper_registry.c +++ b/src/core/ext/client_channel/proxy_mapper_registry.c @@ -61,15 +61,28 @@ static void grpc_proxy_mapper_list_register(grpc_proxy_mapper_list* list, ++list->num_mappers; } -static bool grpc_proxy_mapper_list_map(grpc_exec_ctx* exec_ctx, - grpc_proxy_mapper_list* list, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args) { +static bool grpc_proxy_mapper_list_map_name(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper_list* list, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args) { for (size_t i = 0; i < list->num_mappers; ++i) { - if (grpc_proxy_mapper_map(exec_ctx, list->list[i], address, args, - new_address, new_args)) { + if (grpc_proxy_mapper_map_name(exec_ctx, list->list[i], server_uri, args, + name_to_resolve, new_args)) { + return true; + } + } + return false; +} + +static bool grpc_proxy_mapper_list_map_address( + grpc_exec_ctx* exec_ctx, grpc_proxy_mapper_list* list, + const grpc_resolved_address* address, const grpc_channel_args* args, + grpc_resolved_address** new_address, grpc_channel_args** new_args) { + for (size_t i = 0; i < list->num_mappers; ++i) { + if (grpc_proxy_mapper_map_address(exec_ctx, list->list[i], address, args, + new_address, new_args)) { return true; } } @@ -101,11 +114,20 @@ void grpc_proxy_mapper_register(bool at_start, grpc_proxy_mapper* mapper) { grpc_proxy_mapper_list_register(&g_proxy_mapper_list, at_start, mapper); } -bool grpc_proxy_mappers_map(grpc_exec_ctx* exec_ctx, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args) { - return grpc_proxy_mapper_list_map(exec_ctx, &g_proxy_mapper_list, address, - args, new_address, new_args); +bool grpc_proxy_mappers_map_name(grpc_exec_ctx* exec_ctx, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args) { + return grpc_proxy_mapper_list_map_name(exec_ctx, &g_proxy_mapper_list, + server_uri, args, name_to_resolve, + new_args); +} +bool grpc_proxy_mappers_map_address(grpc_exec_ctx* exec_ctx, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args) { + return grpc_proxy_mapper_list_map_address( + exec_ctx, &g_proxy_mapper_list, address, args, new_address, new_args); } diff --git a/src/core/ext/client_channel/proxy_mapper_registry.h b/src/core/ext/client_channel/proxy_mapper_registry.h index b76af8d4564..742b57a2d49 100644 --- a/src/core/ext/client_channel/proxy_mapper_registry.h +++ b/src/core/ext/client_channel/proxy_mapper_registry.h @@ -44,10 +44,16 @@ void grpc_proxy_mapper_registry_shutdown(); /// the list. Otherwise, it will be added to the end. void grpc_proxy_mapper_register(bool at_start, grpc_proxy_mapper* mapper); -bool grpc_proxy_mappers_map(grpc_exec_ctx* exec_ctx, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args); +bool grpc_proxy_mappers_map_name(grpc_exec_ctx* exec_ctx, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args); + +bool grpc_proxy_mappers_map_address(grpc_exec_ctx* exec_ctx, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args); #endif /* GRPC_CORE_EXT_CLIENT_CHANNEL_PROXY_MAPPER_REGISTRY_H */ diff --git a/src/core/ext/client_channel/subchannel.c b/src/core/ext/client_channel/subchannel.c index f1e4e079e21..9396937783a 100644 --- a/src/core/ext/client_channel/subchannel.c +++ b/src/core/ext/client_channel/subchannel.c @@ -336,8 +336,8 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, grpc_set_initial_connect_string(&addr, &c->initial_connect_string); grpc_resolved_address *new_address = NULL; grpc_channel_args *new_args = NULL; - if (grpc_proxy_mappers_map(exec_ctx, addr, args->args, &new_address, - &new_args)) { + if (grpc_proxy_mappers_map_address(exec_ctx, addr, args->args, &new_address, + &new_args)) { GPR_ASSERT(new_address != NULL); gpr_free(addr); addr = new_address; From 5f3cfe960f708a397c4465a3064072dd3b2d7bb7 Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Fri, 20 Jan 2017 11:02:11 -0500 Subject: [PATCH 053/167] Fix read from uninitialized memory bug in GrpcBufferWriter. This commit fixes an issue in which the following sequence of operations leads to use of uninitialized memory: 1. Caller invokes GrpcBufferWriter::Next(), and then makes use of 8191 bytes in the returned buffer (which is 8192 bytes in size). 2. Caller then returns the unused single byte via GrpcBufferWriter::BackUp(). This method invokes g_core_codegen_interface->grpc_slice_split_tail(), which causes backup_slice_ to be a grpc_slice with one byte. 3. At the next invocation of GrpcBufferWriter::Next(), a reference to the single byte grpc_slice is returned to the caller. The problem here is that the returned reference is to the inlined buffer in the grpc_slice, which is resident in slice_, not the location of the buffer inside slice_buffer_ after g_core_codegen_interface->grpc_slice_buffer_add() in GrpcBufferWriter::Next(). As a result, any data the caller writes to the returned void* data is lost. The solution is to avoid inlined backup slices. --- CMakeLists.txt | 32 +++ Makefile | 48 ++++ build.yaml | 12 + include/grpc++/impl/codegen/proto_utils.h | 9 +- test/cpp/codegen/codegen_test_full.cc | 2 +- test/cpp/codegen/proto_utils_test.cc | 93 +++++++ .../generated/sources_and_headers.json | 17 ++ tools/run_tests/generated/tests.json | 22 ++ .../proto_utils_test/proto_utils_test.vcxproj | 248 ++++++++++++++++++ .../proto_utils_test.vcxproj.filters | 200 ++++++++++++++ 10 files changed, 681 insertions(+), 2 deletions(-) create mode 100644 test/cpp/codegen/proto_utils_test.cc create mode 100644 vsprojects/vcxproj/test/proto_utils_test/proto_utils_test.vcxproj create mode 100644 vsprojects/vcxproj/test/proto_utils_test/proto_utils_test.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index d52e199842c..c6a57e5f826 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -611,6 +611,7 @@ add_dependencies(buildtests_cxx metrics_client) add_dependencies(buildtests_cxx mock_test) add_dependencies(buildtests_cxx noop-benchmark) add_dependencies(buildtests_cxx proto_server_reflection_test) +add_dependencies(buildtests_cxx proto_utils_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx qps_interarrival_test) endif() @@ -8484,6 +8485,37 @@ target_link_libraries(proto_server_reflection_test ${_gRPC_GFLAGS_LIBRARIES} ) +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + +add_executable(proto_utils_test + test/cpp/codegen/proto_utils_test.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(proto_utils_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(proto_utils_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++ + grpc + ${_gRPC_GFLAGS_LIBRARIES} +) + endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) diff --git a/Makefile b/Makefile index 0413cc0e009..d6b7a659be3 100644 --- a/Makefile +++ b/Makefile @@ -1072,6 +1072,7 @@ metrics_client: $(BINDIR)/$(CONFIG)/metrics_client mock_test: $(BINDIR)/$(CONFIG)/mock_test noop-benchmark: $(BINDIR)/$(CONFIG)/noop-benchmark proto_server_reflection_test: $(BINDIR)/$(CONFIG)/proto_server_reflection_test +proto_utils_test: $(BINDIR)/$(CONFIG)/proto_utils_test qps_interarrival_test: $(BINDIR)/$(CONFIG)/qps_interarrival_test qps_json_driver: $(BINDIR)/$(CONFIG)/qps_json_driver qps_openloop_test: $(BINDIR)/$(CONFIG)/qps_openloop_test @@ -1469,6 +1470,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/mock_test \ $(BINDIR)/$(CONFIG)/noop-benchmark \ $(BINDIR)/$(CONFIG)/proto_server_reflection_test \ + $(BINDIR)/$(CONFIG)/proto_utils_test \ $(BINDIR)/$(CONFIG)/qps_interarrival_test \ $(BINDIR)/$(CONFIG)/qps_json_driver \ $(BINDIR)/$(CONFIG)/qps_openloop_test \ @@ -1572,6 +1574,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/mock_test \ $(BINDIR)/$(CONFIG)/noop-benchmark \ $(BINDIR)/$(CONFIG)/proto_server_reflection_test \ + $(BINDIR)/$(CONFIG)/proto_utils_test \ $(BINDIR)/$(CONFIG)/qps_interarrival_test \ $(BINDIR)/$(CONFIG)/qps_json_driver \ $(BINDIR)/$(CONFIG)/qps_openloop_test \ @@ -1901,6 +1904,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/noop-benchmark || ( echo test noop-benchmark failed ; exit 1 ) $(E) "[RUN] Testing proto_server_reflection_test" $(Q) $(BINDIR)/$(CONFIG)/proto_server_reflection_test || ( echo test proto_server_reflection_test failed ; exit 1 ) + $(E) "[RUN] Testing proto_utils_test" + $(Q) $(BINDIR)/$(CONFIG)/proto_utils_test || ( echo test proto_utils_test failed ; exit 1 ) $(E) "[RUN] Testing qps_openloop_test" $(Q) $(BINDIR)/$(CONFIG)/qps_openloop_test || ( echo test qps_openloop_test failed ; exit 1 ) $(E) "[RUN] Testing round_robin_end2end_test" @@ -13907,6 +13912,49 @@ endif endif +PROTO_UTILS_TEST_SRC = \ + test/cpp/codegen/proto_utils_test.cc \ + +PROTO_UTILS_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(PROTO_UTILS_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/proto_utils_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/proto_utils_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/proto_utils_test: $(PROTOBUF_DEP) $(PROTO_UTILS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(PROTO_UTILS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/proto_utils_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/codegen/proto_utils_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a + +deps_proto_utils_test: $(PROTO_UTILS_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(PROTO_UTILS_TEST_OBJS:.o=.dep) +endif +endif + + QPS_INTERARRIVAL_TEST_SRC = \ test/cpp/qps/qps_interarrival_test.cc \ diff --git a/build.yaml b/build.yaml index 456db28cef4..cacb5c4790f 100644 --- a/build.yaml +++ b/build.yaml @@ -3442,6 +3442,18 @@ targets: - grpc - gpr_test_util - gpr +- name: proto_utils_test + gtest: true + build: test + language: c++ + src: + - test/cpp/codegen/proto_utils_test.cc + deps: + - grpc++ + - grpc + filegroups: + - grpc++_codegen_base + - grpc++_codegen_proto - name: qps_interarrival_test build: test run: false diff --git a/include/grpc++/impl/codegen/proto_utils.h b/include/grpc++/impl/codegen/proto_utils.h index 2123b62ed9e..6df9de4fd22 100644 --- a/include/grpc++/impl/codegen/proto_utils.h +++ b/include/grpc++/impl/codegen/proto_utils.h @@ -50,6 +50,8 @@ extern CoreCodegenInterface* g_core_codegen_interface; namespace internal { +class GrpcBufferWriterPeer; + const int kGrpcBufferWriterMaxBufferLength = 8192; class GrpcBufferWriter final @@ -91,13 +93,18 @@ class GrpcBufferWriter final &slice_, GRPC_SLICE_LENGTH(slice_) - count); g_core_codegen_interface->grpc_slice_buffer_add(slice_buffer_, slice_); } - have_backup_ = true; + // It's dangerous to keep an inlined grpc_slice as the backup slice, since + // on a following Next() call, a reference will be returned to this slice + // via GRPC_SLICE_START_PTR, which will not be an adddress held by + // slice_buffer_. + have_backup_ = backup_slice_.refcount != NULL; byte_count_ -= count; } grpc::protobuf::int64 ByteCount() const override { return byte_count_; } private: + friend class GrpcBufferWriterPeer; const int block_size_; int64_t byte_count_; grpc_slice_buffer* slice_buffer_; diff --git a/test/cpp/codegen/codegen_test_full.cc b/test/cpp/codegen/codegen_test_full.cc index d6e2416b550..bc19fc9669d 100644 --- a/test/cpp/codegen/codegen_test_full.cc +++ b/test/cpp/codegen/codegen_test_full.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2016, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/cpp/codegen/proto_utils_test.cc b/test/cpp/codegen/proto_utils_test.cc new file mode 100644 index 00000000000..1daa142b50f --- /dev/null +++ b/test/cpp/codegen/proto_utils_test.cc @@ -0,0 +1,93 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include + +namespace grpc { +namespace internal { + +static GrpcLibraryInitializer g_gli_initializer; + +// Provide access to GrpcBufferWriter internals. +class GrpcBufferWriterPeer { + public: + explicit GrpcBufferWriterPeer(internal::GrpcBufferWriter* writer) + : writer_(writer) {} + bool have_backup() const { return writer_->have_backup_; } + const grpc_slice& backup_slice() const { return writer_->backup_slice_; } + const grpc_slice& slice() const { return writer_->slice_; } + + private: + GrpcBufferWriter* writer_; +}; + +class ProtoUtilsTest : public ::testing::Test {}; + +// Regression test for a memory corruption bug where a series of +// GrpcBufferWriter Next()/Backup() invocations could result in a dangling +// pointer returned by Next() due to the interaction between grpc_slice inlining +// and GRPC_SLICE_START_PTR. +TEST_F(ProtoUtilsTest, BackupNext) { + // Ensure the GrpcBufferWriter internals are initialized. + g_gli_initializer.summon(); + + grpc_byte_buffer* bp; + GrpcBufferWriter writer(&bp, 8192); + GrpcBufferWriterPeer peer(&writer); + + void* data; + int size; + // Allocate a slice. + ASSERT_TRUE(writer.Next(&data, &size)); + EXPECT_EQ(8192, size); + // Return a single byte. Before the fix that this test acts as a regression + // for, this would have resulted in an inlined backup slice. + writer.BackUp(1); + EXPECT_TRUE(!peer.have_backup()); + // On the next allocation, the slice is non-inlined. + ASSERT_TRUE(writer.Next(&data, &size)); + EXPECT_TRUE(peer.slice().refcount != NULL); + + // Cleanup. + g_core_codegen_interface->grpc_byte_buffer_destroy(bp); +} + +} // namespace internal +} // namespace grpc + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 52d8ae183b4..def360afc85 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -3048,6 +3048,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "grpc", + "grpc++", + "grpc++_codegen_base", + "grpc++_codegen_proto" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "proto_utils_test", + "src": [ + "test/cpp/codegen/proto_utils_test.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 4cc543962e4..1b636686cd0 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2975,6 +2975,28 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "proto_utils_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/vcxproj/test/proto_utils_test/proto_utils_test.vcxproj b/vsprojects/vcxproj/test/proto_utils_test/proto_utils_test.vcxproj new file mode 100644 index 00000000000..fc2355624f8 --- /dev/null +++ b/vsprojects/vcxproj/test/proto_utils_test/proto_utils_test.vcxproj @@ -0,0 +1,248 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {D55F5F6B-BAB1-C421-7138-CDC08C686DCF} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + proto_utils_test + static + Debug + static + Debug + + + proto_utils_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/proto_utils_test/proto_utils_test.vcxproj.filters b/vsprojects/vcxproj/test/proto_utils_test/proto_utils_test.vcxproj.filters new file mode 100644 index 00000000000..e6b4a924589 --- /dev/null +++ b/vsprojects/vcxproj/test/proto_utils_test/proto_utils_test.vcxproj.filters @@ -0,0 +1,200 @@ + + + + + test\cpp\codegen + + + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen\security + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + + + + {cc3989e4-cb32-e89b-2d05-41ad6dd1dfdf} + + + {59dc67c7-fb18-960b-db77-99ea8e6c0a5f} + + + {6c2c28e0-a57f-8eb7-4897-41ec191373f9} + + + {6740e44a-b709-0cb2-faa4-c1a792d909c5} + + + {cbb79e1d-95bc-e475-8e86-af33a7603af1} + + + {c4a45c0f-ccce-7e26-d0bb-9e54ad676140} + + + {4aff0d7b-5c9d-080c-faa6-e8126853f149} + + + {35a120bf-8b2f-77e1-cab7-24259b5ed1db} + + + {9d60ac43-ff38-4dfc-8c33-9167fb47fa90} + + + {7845357d-2b4b-6f25-c68b-79be9ab587fb} + + + {22e8c70f-796c-f2c0-d87d-e4ef419609a1} + + + + From f2fe4f7dbd00a285f265167bb8fec5c48fd7345b Mon Sep 17 00:00:00 2001 From: yang-g Date: Tue, 7 Feb 2017 12:47:22 -0800 Subject: [PATCH 054/167] Add a PreServerStart in global server callbacks --- include/grpc++/server.h | 2 ++ src/cpp/server/server_cc.cc | 1 + 2 files changed, 3 insertions(+) diff --git a/include/grpc++/server.h b/include/grpc++/server.h index fba9952e6e7..002f252a8f5 100644 --- a/include/grpc++/server.h +++ b/include/grpc++/server.h @@ -88,6 +88,8 @@ class Server final : public ServerInterface, private GrpcLibraryCodegen { virtual void PreSynchronousRequest(ServerContext* context) = 0; /// Called after application callback for each synchronous server request virtual void PostSynchronousRequest(ServerContext* context) = 0; + /// Called before server is started. + virtual void PreServerStart(Server* server) {} }; /// Set the global callback object. Can only be called once. Does not take /// ownership of callbacks, and expects the pointed to object to be alive diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index f6a7f1d755b..29898a4209e 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -478,6 +478,7 @@ int Server::AddListeningPort(const grpc::string& addr, bool Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) { GPR_ASSERT(!started_); + global_callbacks_->PreServerStart(this); started_ = true; grpc_server_start(server_); From 8c58a489a2cf8f81299d97a082995cb77eb0bc52 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Feb 2017 14:52:59 -0800 Subject: [PATCH 055/167] Save allocating a grpc_error if there is no error --- src/core/lib/surface/call.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 70bab4c0796..7f1410a0856 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -897,7 +897,7 @@ static void recv_common_filter(grpc_exec_ctx *exec_ctx, grpc_call *call, error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, msg); gpr_free(msg); grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.grpc_message); - } else { + } else if (error != GRPC_ERROR_NONE) { error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, ""); } From e128e837f0f55c93334a97517e99a03cc2699da2 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Tue, 7 Feb 2017 16:22:14 -0800 Subject: [PATCH 056/167] clang fmt --- src/core/ext/transport/chttp2/transport/frame_rst_stream.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index 2e93ae2b857..cb017e75f04 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -110,11 +110,10 @@ grpc_error *grpc_chttp2_rst_stream_parser_parse(grpc_exec_ctx *exec_ctx, grpc_error *error = GRPC_ERROR_NONE; if (reason != GRPC_HTTP2_NO_ERROR || s->header_frames_received < 2) { char *message; - gpr_asprintf(&message, "Received RST_STREAM with error code %d", - reason); + gpr_asprintf(&message, "Received RST_STREAM with error code %d", reason); error = grpc_error_set_int( - grpc_error_set_str(GRPC_ERROR_CREATE("RST_STREAM"), - GRPC_ERROR_STR_GRPC_MESSAGE, message), + grpc_error_set_str(GRPC_ERROR_CREATE("RST_STREAM"), + GRPC_ERROR_STR_GRPC_MESSAGE, message), GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); gpr_free(message); } From 3a5bba0a2034a2dffbfeb7a30281f8ac43d5ce59 Mon Sep 17 00:00:00 2001 From: Paul Marks Date: Tue, 7 Feb 2017 16:28:09 -0800 Subject: [PATCH 057/167] Connect to "localhost" from interop clients. This is for compatibility with IPv6-only environments; see the Google bug b/32531998. Note that InteropClient.cs and http2_client.cc don't appear to be tested by our internal branch, but I've included them for consistency. --- src/csharp/Grpc.IntegrationTesting/InteropClient.cs | 2 +- src/python/grpcio_tests/tests/interop/client.py | 2 +- test/cpp/interop/client.cc | 2 +- test/cpp/interop/http2_client.cc | 2 +- test/cpp/interop/reconnect_interop_client.cc | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs index 5ba83b143ed..68a8f4879b4 100644 --- a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs +++ b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs @@ -56,7 +56,7 @@ namespace Grpc.IntegrationTesting { private class ClientOptions { - [Option("server_host", Default = "127.0.0.1")] + [Option("server_host", Default = "localhost")] public string ServerHost { get; set; } [Option("server_host_override", Default = TestCredentials.DefaultHostOverride)] diff --git a/src/python/grpcio_tests/tests/interop/client.py b/src/python/grpcio_tests/tests/interop/client.py index 833818e6625..97f6843d3cb 100644 --- a/src/python/grpcio_tests/tests/interop/client.py +++ b/src/python/grpcio_tests/tests/interop/client.py @@ -45,7 +45,7 @@ def _args(): '--server_host', help='the host to which to connect', type=str, - default="127.0.0.1") + default="localhost") parser.add_argument( '--server_port', help='the port to which to connect', type=int) parser.add_argument( diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc index 8a00b61cef1..5688ab79716 100644 --- a/test/cpp/interop/client.cc +++ b/test/cpp/interop/client.cc @@ -51,7 +51,7 @@ DEFINE_bool(use_tls, false, "Whether to use tls."); DEFINE_string(custom_credentials_type, "", "User provided credentials type."); DEFINE_bool(use_test_ca, false, "False to use SSL roots for google"); DEFINE_int32(server_port, 0, "Server port."); -DEFINE_string(server_host, "127.0.0.1", "Server host to connect to"); +DEFINE_string(server_host, "localhost", "Server host to connect to"); DEFINE_string(server_host_override, "foo.test.google.fr", "Override the server host which is sent in HTTP header"); DEFINE_string( diff --git a/test/cpp/interop/http2_client.cc b/test/cpp/interop/http2_client.cc index 38aee43b26a..b96e9fac36e 100644 --- a/test/cpp/interop/http2_client.cc +++ b/test/cpp/interop/http2_client.cc @@ -223,7 +223,7 @@ bool Http2Client::DoMaxStreams() { } // namespace grpc DEFINE_int32(server_port, 0, "Server port."); -DEFINE_string(server_host, "127.0.0.1", "Server host to connect to"); +DEFINE_string(server_host, "localhost", "Server host to connect to"); DEFINE_string(test_case, "rst_after_header", "Configure different test cases. Valid options are:\n\n" "goaway\n" diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 797e52c744f..1c2f6066377 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -48,7 +48,7 @@ DEFINE_int32(server_control_port, 0, "Server port for control rpcs."); DEFINE_int32(server_retry_port, 0, "Server port for testing reconnection."); -DEFINE_string(server_host, "127.0.0.1", "Server host to connect to"); +DEFINE_string(server_host, "localhost", "Server host to connect to"); DEFINE_int32(max_reconnect_backoff_ms, 0, "Maximum backoff time, or 0 for default."); From 012915045fca08c365add95a8e29eb40021ecd64 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 7 Feb 2017 13:26:41 -0800 Subject: [PATCH 058/167] Secure naming support for gRPCLB --- BUILD | 62 +++++-- CMakeLists.txt | 4 + Makefile | 6 + binding.gyp | 2 + build.yaml | 22 ++- config.m4 | 2 + gRPC-Core.podspec | 6 + grpc.gemspec | 4 + package.xml | 4 + src/core/ext/client_channel/subchannel.c | 16 +- src/core/ext/client_channel/subchannel.h | 3 + src/core/ext/lb_policy/grpclb/grpclb.c | 171 +++++++++++------- .../ext/lb_policy/grpclb/grpclb_channel.c | 77 ++++++++ .../ext/lb_policy/grpclb/grpclb_channel.h | 56 ++++++ .../lb_policy/grpclb/grpclb_channel_secure.c | 107 +++++++++++ .../ext/lb_policy/round_robin/round_robin.c | 7 + .../client/secure/secure_channel_create.c | 171 +++++++++++++----- src/core/lib/iomgr/sockaddr_utils.c | 38 ++-- src/core/lib/iomgr/sockaddr_utils.h | 4 + .../lib/security/credentials/credentials.c | 47 +++++ .../lib/security/credentials/credentials.h | 13 ++ .../credentials/fake/fake_credentials.c | 8 +- .../credentials/fake/fake_credentials.h | 15 ++ .../security/transport/client_auth_filter.c | 2 +- .../lib/security/transport/lb_targets_info.c | 70 +++++++ .../lib/security/transport/lb_targets_info.h | 47 +++++ .../security/transport/security_connector.c | 141 +++++++++++++-- .../security/transport/security_connector.h | 7 +- .../security/transport/security_handshaker.c | 4 +- src/core/lib/surface/init_secure.c | 2 +- src/python/grpcio/grpc_core_dependencies.py | 2 + test/core/end2end/fake_resolver.c | 29 ++- .../core/surface/secure_channel_create_test.c | 2 +- test/cpp/grpclb/grpclb_test.cc | 89 ++++++--- tools/doxygen/Doxyfile.core.internal | 4 + .../generated/sources_and_headers.json | 37 +++- vsprojects/vcxproj/grpc/grpc.vcxproj | 6 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 12 ++ .../grpc_unsecure/grpc_unsecure.vcxproj | 3 + .../grpc_unsecure.vcxproj.filters | 6 + 40 files changed, 1098 insertions(+), 210 deletions(-) create mode 100644 src/core/ext/lb_policy/grpclb/grpclb_channel.c create mode 100644 src/core/ext/lb_policy/grpclb/grpclb_channel.h create mode 100644 src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c create mode 100644 src/core/lib/security/transport/lb_targets_info.c create mode 100644 src/core/lib/security/transport/lb_targets_info.h diff --git a/BUILD b/BUILD index 58971ff8e35..c56d9c9746f 100644 --- a/BUILD +++ b/BUILD @@ -63,7 +63,7 @@ grpc_cc_library( deps = [ "census", "grpc_base", - "grpc_lb_policy_grpclb", + "grpc_lb_policy_grpclb_secure", "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", "grpc_load_reporting", @@ -624,9 +624,9 @@ grpc_cc_library( "src/core/lib/surface/completion_queue.h", "src/core/lib/surface/event_string.h", "src/core/lib/surface/init.h", - "src/core/lib/surface/validate_metadata.h", "src/core/lib/surface/lame_client.h", "src/core/lib/surface/server.h", + "src/core/lib/surface/validate_metadata.h", "src/core/lib/transport/byte_stream.h", "src/core/lib/transport/connectivity_state.h", "src/core/lib/transport/error_utils.h", @@ -665,7 +665,6 @@ grpc_cc_library( grpc_cc_library( name = "grpc_client_channel", - language = "c", srcs = [ "src/core/ext/client_channel/channel_connectivity.c", "src/core/ext/client_channel/client_channel.c", @@ -709,6 +708,7 @@ grpc_cc_library( "src/core/ext/client_channel/subchannel_index.h", "src/core/ext/client_channel/uri_parser.h", ], + language = "c", deps = [ "grpc_base", ], @@ -735,11 +735,37 @@ grpc_cc_library( name = "grpc_lb_policy_grpclb", srcs = [ "src/core/ext/lb_policy/grpclb/grpclb.c", + "src/core/ext/lb_policy/grpclb/grpclb_channel.c", + "src/core/ext/lb_policy/grpclb/load_balancer_api.c", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", + ], + hdrs = [ + "src/core/ext/lb_policy/grpclb/grpclb.h", + "src/core/ext/lb_policy/grpclb/grpclb_channel.h", + "src/core/ext/lb_policy/grpclb/load_balancer_api.h", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", + ], + external_deps = [ + "nanopb", + ], + language = "c", + deps = [ + "grpc_base", + "grpc_client_channel", + ], +) + +grpc_cc_library( + name = "grpc_lb_policy_grpclb_secure", + srcs = [ + "src/core/ext/lb_policy/grpclb/grpclb.c", + "src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.c", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", ], hdrs = [ "src/core/ext/lb_policy/grpclb/grpclb.h", + "src/core/ext/lb_policy/grpclb/grpclb_channel.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", ], @@ -750,6 +776,7 @@ grpc_cc_library( deps = [ "grpc_base", "grpc_client_channel", + "grpc_secure", ], ) @@ -837,6 +864,7 @@ grpc_cc_library( "src/core/lib/security/credentials/ssl/ssl_credentials.c", "src/core/lib/security/transport/client_auth_filter.c", "src/core/lib/security/transport/secure_endpoint.c", + "src/core/lib/security/transport/lb_targets_info.c", "src/core/lib/security/transport/security_connector.c", "src/core/lib/security/transport/security_handshaker.c", "src/core/lib/security/transport/server_auth_filter.c", @@ -860,6 +888,7 @@ grpc_cc_library( "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/transport/auth_filters.h", "src/core/lib/security/transport/secure_endpoint.h", + "src/core/lib/security/transport/lb_targets_info.h", "src/core/lib/security/transport/security_connector.h", "src/core/lib/security/transport/security_handshaker.h", "src/core/lib/security/transport/tsi_error.h", @@ -943,22 +972,21 @@ grpc_cc_library( ) grpc_cc_library( - name = "grpc_transport_chttp2_client_connector", - hdrs = [ - "src/core/ext/transport/chttp2/client/chttp2_connector.h", - ], - srcs = [ - "src/core/ext/transport/chttp2/client/chttp2_connector.c", - ], - language = "c", - deps = [ - "grpc_transport_chttp2", - "grpc_base", - "grpc_client_channel", - ], + name = "grpc_transport_chttp2_client_connector", + srcs = [ + "src/core/ext/transport/chttp2/client/chttp2_connector.c", + ], + hdrs = [ + "src/core/ext/transport/chttp2/client/chttp2_connector.h", + ], + language = "c", + deps = [ + "grpc_base", + "grpc_client_channel", + "grpc_transport_chttp2", + ], ) - grpc_cc_library( name = "grpc_transport_chttp2_client_insecure", srcs = [ diff --git a/CMakeLists.txt b/CMakeLists.txt index c6a57e5f826..cf9908b0606 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -953,6 +953,7 @@ add_library(grpc src/core/lib/security/credentials/plugin/plugin_credentials.c src/core/lib/security/credentials/ssl/ssl_credentials.c src/core/lib/security/transport/client_auth_filter.c + src/core/lib/security/transport/lb_targets_info.c src/core/lib/security/transport/secure_endpoint.c src/core/lib/security/transport/security_connector.c src/core/lib/security/transport/security_handshaker.c @@ -993,6 +994,7 @@ add_library(grpc src/core/ext/transport/chttp2/client/insecure/channel_create.c src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c src/core/ext/lb_policy/grpclb/grpclb.c + src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c src/core/ext/lb_policy/grpclb/load_balancer_api.c src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c third_party/nanopb/pb_common.c @@ -1272,6 +1274,7 @@ add_library(grpc_cronet src/core/lib/security/credentials/plugin/plugin_credentials.c src/core/lib/security/credentials/ssl/ssl_credentials.c src/core/lib/security/transport/client_auth_filter.c + src/core/lib/security/transport/lb_targets_info.c src/core/lib/security/transport/secure_endpoint.c src/core/lib/security/transport/security_connector.c src/core/lib/security/transport/security_handshaker.c @@ -1773,6 +1776,7 @@ add_library(grpc_unsecure src/core/ext/load_reporting/load_reporting.c src/core/ext/load_reporting/load_reporting_filter.c src/core/ext/lb_policy/grpclb/grpclb.c + src/core/ext/lb_policy/grpclb/grpclb_channel.c src/core/ext/lb_policy/grpclb/load_balancer_api.c src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c third_party/nanopb/pb_common.c diff --git a/Makefile b/Makefile index d6b7a659be3..d0602def847 100644 --- a/Makefile +++ b/Makefile @@ -2794,6 +2794,7 @@ LIBGRPC_SRC = \ src/core/lib/security/credentials/plugin/plugin_credentials.c \ src/core/lib/security/credentials/ssl/ssl_credentials.c \ src/core/lib/security/transport/client_auth_filter.c \ + src/core/lib/security/transport/lb_targets_info.c \ src/core/lib/security/transport/secure_endpoint.c \ src/core/lib/security/transport/security_connector.c \ src/core/lib/security/transport/security_handshaker.c \ @@ -2834,6 +2835,7 @@ LIBGRPC_SRC = \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c \ src/core/ext/lb_policy/grpclb/grpclb.c \ + src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ @@ -3127,6 +3129,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/security/credentials/plugin/plugin_credentials.c \ src/core/lib/security/credentials/ssl/ssl_credentials.c \ src/core/lib/security/transport/client_auth_filter.c \ + src/core/lib/security/transport/lb_targets_info.c \ src/core/lib/security/transport/secure_endpoint.c \ src/core/lib/security/transport/security_connector.c \ src/core/lib/security/transport/security_handshaker.c \ @@ -3644,6 +3647,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/load_reporting/load_reporting.c \ src/core/ext/load_reporting/load_reporting_filter.c \ src/core/ext/lb_policy/grpclb/grpclb.c \ + src/core/ext/lb_policy/grpclb/grpclb_channel.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ @@ -17705,6 +17709,7 @@ ifneq ($(OPENSSL_DEP),) # This is to ensure the embedded OpenSSL is built beforehand, properly # installing headers to their final destination on the drive. We need this # otherwise parallel compilation will fail if a source is compiled first. +src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c: $(OPENSSL_DEP) src/core/ext/transport/chttp2/client/secure/secure_channel_create.c: $(OPENSSL_DEP) src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c: $(OPENSSL_DEP) src/core/ext/transport/cronet/client/secure/cronet_channel_create.c: $(OPENSSL_DEP) @@ -17726,6 +17731,7 @@ src/core/lib/security/credentials/oauth2/oauth2_credentials.c: $(OPENSSL_DEP) src/core/lib/security/credentials/plugin/plugin_credentials.c: $(OPENSSL_DEP) src/core/lib/security/credentials/ssl/ssl_credentials.c: $(OPENSSL_DEP) src/core/lib/security/transport/client_auth_filter.c: $(OPENSSL_DEP) +src/core/lib/security/transport/lb_targets_info.c: $(OPENSSL_DEP) src/core/lib/security/transport/secure_endpoint.c: $(OPENSSL_DEP) src/core/lib/security/transport/security_connector.c: $(OPENSSL_DEP) src/core/lib/security/transport/security_handshaker.c: $(OPENSSL_DEP) diff --git a/binding.gyp b/binding.gyp index 1815a5a4760..8ff3d8c1a34 100644 --- a/binding.gyp +++ b/binding.gyp @@ -758,6 +758,7 @@ 'src/core/lib/security/credentials/plugin/plugin_credentials.c', 'src/core/lib/security/credentials/ssl/ssl_credentials.c', 'src/core/lib/security/transport/client_auth_filter.c', + 'src/core/lib/security/transport/lb_targets_info.c', 'src/core/lib/security/transport/secure_endpoint.c', 'src/core/lib/security/transport/security_connector.c', 'src/core/lib/security/transport/security_handshaker.c', @@ -798,6 +799,7 @@ 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c', 'src/core/ext/lb_policy/grpclb/grpclb.c', + 'src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', diff --git a/build.yaml b/build.yaml index cacb5c4790f..7fec296ad3d 100644 --- a/build.yaml +++ b/build.yaml @@ -452,10 +452,28 @@ filegroups: - name: grpc_lb_policy_grpclb headers: - src/core/ext/lb_policy/grpclb/grpclb.h + - src/core/ext/lb_policy/grpclb/grpclb_channel.h - src/core/ext/lb_policy/grpclb/load_balancer_api.h - src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h src: - src/core/ext/lb_policy/grpclb/grpclb.c + - src/core/ext/lb_policy/grpclb/grpclb_channel.c + - src/core/ext/lb_policy/grpclb/load_balancer_api.c + - src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c + plugin: grpc_lb_policy_grpclb + uses: + - grpc_base + - grpc_client_channel + - nanopb +- name: grpc_lb_policy_grpclb_secure + headers: + - src/core/ext/lb_policy/grpclb/grpclb.h + - src/core/ext/lb_policy/grpclb/grpclb_channel.h + - src/core/ext/lb_policy/grpclb/load_balancer_api.h + - src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h + src: + - src/core/ext/lb_policy/grpclb/grpclb.c + - src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c - src/core/ext/lb_policy/grpclb/load_balancer_api.c - src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c plugin: grpc_lb_policy_grpclb @@ -518,6 +536,7 @@ filegroups: - src/core/lib/security/credentials/plugin/plugin_credentials.h - src/core/lib/security/credentials/ssl/ssl_credentials.h - src/core/lib/security/transport/auth_filters.h + - src/core/lib/security/transport/lb_targets_info.h - src/core/lib/security/transport/secure_endpoint.h - src/core/lib/security/transport/security_connector.h - src/core/lib/security/transport/security_handshaker.h @@ -541,6 +560,7 @@ filegroups: - src/core/lib/security/credentials/plugin/plugin_credentials.c - src/core/lib/security/credentials/ssl/ssl_credentials.c - src/core/lib/security/transport/client_auth_filter.c + - src/core/lib/security/transport/lb_targets_info.c - src/core/lib/security/transport/secure_endpoint.c - src/core/lib/security/transport/security_connector.c - src/core/lib/security/transport/security_handshaker.c @@ -909,7 +929,7 @@ libs: - grpc_transport_chttp2_client_secure - grpc_transport_chttp2_server_insecure - grpc_transport_chttp2_client_insecure - - grpc_lb_policy_grpclb + - grpc_lb_policy_grpclb_secure - grpc_lb_policy_pick_first - grpc_lb_policy_round_robin - grpc_resolver_dns_native diff --git a/config.m4 b/config.m4 index 2c6c89f3eeb..90536e503ed 100644 --- a/config.m4 +++ b/config.m4 @@ -237,6 +237,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/security/credentials/plugin/plugin_credentials.c \ src/core/lib/security/credentials/ssl/ssl_credentials.c \ src/core/lib/security/transport/client_auth_filter.c \ + src/core/lib/security/transport/lb_targets_info.c \ src/core/lib/security/transport/secure_endpoint.c \ src/core/lib/security/transport/security_connector.c \ src/core/lib/security/transport/security_handshaker.c \ @@ -277,6 +278,7 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/transport/chttp2/client/insecure/channel_create.c \ src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c \ src/core/ext/lb_policy/grpclb/grpclb.c \ + src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 7632174f31c..77743f69bdc 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -388,6 +388,7 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.h', 'src/core/lib/security/credentials/ssl/ssl_credentials.h', 'src/core/lib/security/transport/auth_filters.h', + 'src/core/lib/security/transport/lb_targets_info.h', 'src/core/lib/security/transport/secure_endpoint.h', 'src/core/lib/security/transport/security_connector.h', 'src/core/lib/security/transport/security_handshaker.h', @@ -420,6 +421,7 @@ Pod::Spec.new do |s| 'src/core/ext/client_channel/uri_parser.h', 'src/core/ext/transport/chttp2/client/chttp2_connector.h', 'src/core/ext/lb_policy/grpclb/grpclb.h', + 'src/core/ext/lb_policy/grpclb/grpclb_channel.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', 'third_party/nanopb/pb.h', @@ -596,6 +598,7 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.c', 'src/core/lib/security/credentials/ssl/ssl_credentials.c', 'src/core/lib/security/transport/client_auth_filter.c', + 'src/core/lib/security/transport/lb_targets_info.c', 'src/core/lib/security/transport/secure_endpoint.c', 'src/core/lib/security/transport/security_connector.c', 'src/core/lib/security/transport/security_handshaker.c', @@ -636,6 +639,7 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c', 'src/core/ext/lb_policy/grpclb/grpclb.c', + 'src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', @@ -813,6 +817,7 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.h', 'src/core/lib/security/credentials/ssl/ssl_credentials.h', 'src/core/lib/security/transport/auth_filters.h', + 'src/core/lib/security/transport/lb_targets_info.h', 'src/core/lib/security/transport/secure_endpoint.h', 'src/core/lib/security/transport/security_connector.h', 'src/core/lib/security/transport/security_handshaker.h', @@ -845,6 +850,7 @@ Pod::Spec.new do |s| 'src/core/ext/client_channel/uri_parser.h', 'src/core/ext/transport/chttp2/client/chttp2_connector.h', 'src/core/ext/lb_policy/grpclb/grpclb.h', + 'src/core/ext/lb_policy/grpclb/grpclb_channel.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', 'third_party/nanopb/pb.h', diff --git a/grpc.gemspec b/grpc.gemspec index 7b132686c8f..6e7e1174da4 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -305,6 +305,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.h ) s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.h ) s.files += %w( src/core/lib/security/transport/auth_filters.h ) + s.files += %w( src/core/lib/security/transport/lb_targets_info.h ) s.files += %w( src/core/lib/security/transport/secure_endpoint.h ) s.files += %w( src/core/lib/security/transport/security_connector.h ) s.files += %w( src/core/lib/security/transport/security_handshaker.h ) @@ -337,6 +338,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/client_channel/uri_parser.h ) s.files += %w( src/core/ext/transport/chttp2/client/chttp2_connector.h ) s.files += %w( src/core/ext/lb_policy/grpclb/grpclb.h ) + s.files += %w( src/core/ext/lb_policy/grpclb/grpclb_channel.h ) s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.h ) s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h ) s.files += %w( third_party/nanopb/pb.h ) @@ -513,6 +515,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.c ) s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.c ) s.files += %w( src/core/lib/security/transport/client_auth_filter.c ) + s.files += %w( src/core/lib/security/transport/lb_targets_info.c ) s.files += %w( src/core/lib/security/transport/secure_endpoint.c ) s.files += %w( src/core/lib/security/transport/security_connector.c ) s.files += %w( src/core/lib/security/transport/security_handshaker.c ) @@ -553,6 +556,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/transport/chttp2/client/insecure/channel_create.c ) s.files += %w( src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c ) s.files += %w( src/core/ext/lb_policy/grpclb/grpclb.c ) + s.files += %w( src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c ) s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.c ) s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c ) s.files += %w( third_party/nanopb/pb_common.c ) diff --git a/package.xml b/package.xml index e4c0e83cc71..7aa894a1781 100644 --- a/package.xml +++ b/package.xml @@ -314,6 +314,7 @@ + @@ -346,6 +347,7 @@ + @@ -522,6 +524,7 @@ + @@ -562,6 +565,7 @@ + diff --git a/src/core/ext/client_channel/subchannel.c b/src/core/ext/client_channel/subchannel.c index f1e4e079e21..abbbfdfd644 100644 --- a/src/core/ext/client_channel/subchannel.c +++ b/src/core/ext/client_channel/subchannel.c @@ -788,7 +788,8 @@ grpc_call_stack *grpc_subchannel_call_get_call_stack( return SUBCHANNEL_CALL_TO_CALL_STACK(subchannel_call); } -static void grpc_uri_to_sockaddr(char *uri_str, grpc_resolved_address *addr) { +static void grpc_uri_to_sockaddr(const char *uri_str, + grpc_resolved_address *addr) { grpc_uri *uri = grpc_uri_parse(uri_str, 0 /* suppress_errors */); GPR_ASSERT(uri != NULL); if (strcmp(uri->scheme, "ipv4") == 0) { @@ -803,14 +804,19 @@ static void grpc_uri_to_sockaddr(char *uri_str, grpc_resolved_address *addr) { void grpc_get_subchannel_address_arg(const grpc_channel_args *args, grpc_resolved_address *addr) { + const char *addr_uri_str = grpc_get_subchannel_address_uri_arg(args); + memset(addr, 0, sizeof(*addr)); + if (*addr_uri_str != '\0') { + grpc_uri_to_sockaddr(addr_uri_str, addr); + } +} + +const char *grpc_get_subchannel_address_uri_arg(const grpc_channel_args *args) { const grpc_arg *addr_arg = grpc_channel_args_find(args, GRPC_ARG_SUBCHANNEL_ADDRESS); GPR_ASSERT(addr_arg != NULL); // Should have been set by LB policy. GPR_ASSERT(addr_arg->type == GRPC_ARG_STRING); - memset(addr, 0, sizeof(*addr)); - if (*addr_arg->value.string != '\0') { - grpc_uri_to_sockaddr(addr_arg->value.string, addr); - } + return addr_arg->value.string; } grpc_arg grpc_create_subchannel_address_arg(const grpc_resolved_address *addr) { diff --git a/src/core/ext/client_channel/subchannel.h b/src/core/ext/client_channel/subchannel.h index 9bd35a77049..26ce9544871 100644 --- a/src/core/ext/client_channel/subchannel.h +++ b/src/core/ext/client_channel/subchannel.h @@ -178,6 +178,9 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, void grpc_get_subchannel_address_arg(const grpc_channel_args *args, grpc_resolved_address *addr); +/// Returns the URI string for the address to connect to. +const char *grpc_get_subchannel_address_uri_arg(const grpc_channel_args *args); + /// Returns a new channel arg encoding the subchannel address as a string. /// Caller is responsible for freeing the string. grpc_arg grpc_create_subchannel_address_arg(const grpc_resolved_address *addr); diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 308facb7e7a..ab62e5ed6a1 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -112,11 +112,13 @@ #include "src/core/ext/client_channel/lb_policy_registry.h" #include "src/core/ext/client_channel/parse_address.h" #include "src/core/ext/lb_policy/grpclb/grpclb.h" +#include "src/core/ext/lb_policy/grpclb/grpclb_channel.h" #include "src/core/ext/lb_policy/grpclb/load_balancer_api.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/slice/slice_hash_table.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/backoff.h" @@ -751,6 +753,96 @@ static void glb_rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, GRPC_ERROR_UNREF(error); } +static void destroy_balancer_name(grpc_exec_ctx *exec_ctx, + void *balancer_name) { + gpr_free(balancer_name); +} + +static void *copy_balancer_name(void *balancer_name) { + return gpr_strdup(balancer_name); +} + +static grpc_slice_hash_table_entry targets_info_entry_create( + const char *address, const char *balancer_name) { + static const grpc_slice_hash_table_vtable vtable = {destroy_balancer_name, + copy_balancer_name}; + grpc_slice_hash_table_entry entry; + entry.key = grpc_slice_from_copied_string(address); + entry.value = (void *)balancer_name; + entry.vtable = &vtable; + return entry; +} + +/* Returns the target URI for the LB service whose addresses are in \a + * addresses. Using this URI, a bidirectional streaming channel will be created + * for the reception of load balancing updates. + * + * The output argument \a targets_info will be updated to contain a mapping of + * "LB server address" to "balancer name", as reported by the naming system. + * This mapping will be propagated via the channel arguments of the + * aforementioned LB streaming channel, to be used by the security connector for + * secure naming checks. The user is responsible for freeing \a targets_info. */ +static char *get_lb_uri_target_addresses(grpc_exec_ctx *exec_ctx, + const grpc_lb_addresses *addresses, + grpc_slice_hash_table **targets_info) { + size_t num_grpclb_addrs = 0; + for (size_t i = 0; i < addresses->num_addresses; ++i) { + if (addresses->addresses[i].is_balancer) ++num_grpclb_addrs; + } + /* All input addresses come from a resolver that claims they are LB services. + * It's the resolver's responsibility to make sure this policy is only + * instantiated and used in that case. Otherwise, something has gone wrong. */ + GPR_ASSERT(num_grpclb_addrs > 0); + + grpc_slice_hash_table_entry *targets_info_entries = + gpr_malloc(sizeof(*targets_info_entries) * num_grpclb_addrs); + + /* construct a target ipvX://ip1:port1,ip2:port2,... from the addresses in \a + * addresses */ + /* TODO(dgq): support mixed ip version */ + char **addr_strs = gpr_malloc(sizeof(char *) * num_grpclb_addrs); + size_t addr_index = 0; + + for (size_t i = 0; i < addresses->num_addresses; i++) { + if (addresses->addresses[i].user_data != NULL) { + gpr_log(GPR_ERROR, + "This LB policy doesn't support user data. It will be ignored"); + } + if (addresses->addresses[i].is_balancer) { + char *addr_str; + GPR_ASSERT(grpc_sockaddr_to_string( + &addr_str, &addresses->addresses[i].address, true) > 0); + targets_info_entries[addr_index] = targets_info_entry_create( + addr_str, addresses->addresses[i].balancer_name); + addr_strs[addr_index++] = addr_str; + } + } + GPR_ASSERT(addr_index == num_grpclb_addrs); + + size_t uri_path_len; + char *uri_path = gpr_strjoin_sep((const char **)addr_strs, num_grpclb_addrs, + ",", &uri_path_len); + for (size_t i = 0; i < num_grpclb_addrs; i++) gpr_free(addr_strs[i]); + gpr_free(addr_strs); + + char *target_uri_str = NULL; + /* TODO(dgq): Don't assume all addresses will share the scheme of the first + * one */ + gpr_asprintf(&target_uri_str, "%s:%s", + grpc_sockaddr_get_uri_scheme(&addresses->addresses[0].address), + uri_path); + gpr_free(uri_path); + + *targets_info = + grpc_slice_hash_table_create(num_grpclb_addrs, targets_info_entries); + for (size_t i = 0; i < num_grpclb_addrs; i++) { + grpc_slice_unref_internal(exec_ctx, targets_info_entries[i].key); + } + gpr_free(targets_info_entries); + + return target_uri_str; +} + static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { @@ -788,85 +880,30 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, } grpc_uri_destroy(uri); - /* All input addresses in addresses come from a resolver that claims - * they are LB services. It's the resolver's responsibility to make sure - * this policy is only instantiated and used in that case. - * - * Create a client channel over them to communicate with a LB service */ glb_policy->cc_factory = args->client_channel_factory; glb_policy->args = grpc_channel_args_copy(args->args); GPR_ASSERT(glb_policy->cc_factory != NULL); - /* construct a target from the addresses in args, given in the form - * ipvX://ip1:port1,ip2:port2,... - * TODO(dgq): support mixed ip version */ - char **addr_strs = gpr_malloc(sizeof(char *) * num_grpclb_addrs); - size_t addr_index = 0; - for (size_t i = 0; i < addresses->num_addresses; i++) { - if (addresses->addresses[i].user_data != NULL) { - gpr_log(GPR_ERROR, - "This LB policy doesn't support user data. It will be ignored"); - } - if (addresses->addresses[i].is_balancer) { - if (addr_index == 0) { - addr_strs[addr_index++] = - grpc_sockaddr_to_uri(&addresses->addresses[i].address); - } else { - GPR_ASSERT(grpc_sockaddr_to_string(&addr_strs[addr_index++], - &addresses->addresses[i].address, - true) > 0); - } - } - } - size_t uri_path_len; - char *target_uri_str = gpr_strjoin_sep((const char **)addr_strs, - num_grpclb_addrs, ",", &uri_path_len); - - /* Create a channel to talk to the LBs. - * - * We strip out the channel arg for the LB policy name, since we want - * to use the default (pick_first) in this case. - * - * We also strip out the channel arg for the resolved addresses, since - * that will be generated by the name resolver used in the LB channel. - * Note that the LB channel will use the sockaddr resolver, so this - * won't actually generate a query to DNS (or some other name service). - * However, the addresses returned by the sockaddr resolver will have - * is_balancer=false, whereas our own addresses have is_balancer=true. - * We need the LB channel to return addresses with is_balancer=false - * so that it does not wind up recursively using the grpclb LB policy, - * as per the special case logic in client_channel.c. - * - * Finally, we also strip out the channel arg for the server URI, - * since that will be different for the LB channel than for the parent - * channel. (The client channel factory will re-add this arg with - * the right value.) - */ - static const char *keys_to_remove[] = { - GRPC_ARG_LB_POLICY_NAME, GRPC_ARG_LB_ADDRESSES, GRPC_ARG_SERVER_URI}; - grpc_channel_args *new_args = grpc_channel_args_copy_and_remove( - args->args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove)); - glb_policy->lb_channel = grpc_client_channel_factory_create_channel( - exec_ctx, glb_policy->cc_factory, target_uri_str, - GRPC_CLIENT_CHANNEL_TYPE_LOAD_BALANCING, new_args); - grpc_channel_args_destroy(exec_ctx, new_args); - - gpr_free(target_uri_str); - for (size_t i = 0; i < num_grpclb_addrs; i++) { - gpr_free(addr_strs[i]); - } - gpr_free(addr_strs); - + grpc_slice_hash_table *targets_info = NULL; + /* Create a client channel over them to communicate with a LB service */ + char *lb_service_target_addresses = + get_lb_uri_target_addresses(exec_ctx, addresses, &targets_info); + grpc_channel_args *lb_channel_args = + get_lb_channel_args(exec_ctx, targets_info, args->args); + glb_policy->lb_channel = grpc_lb_policy_grpclb_create_lb_channel( + exec_ctx, lb_service_target_addresses, args->client_channel_factory, + lb_channel_args); + grpc_slice_hash_table_unref(exec_ctx, targets_info); + grpc_channel_args_destroy(exec_ctx, lb_channel_args); + gpr_free(lb_service_target_addresses); if (glb_policy->lb_channel == NULL) { gpr_free(glb_policy); return NULL; } - grpc_lb_policy_init(&glb_policy->base, &glb_lb_policy_vtable); gpr_mu_init(&glb_policy->mu); grpc_connectivity_state_init(&glb_policy->state_tracker, GRPC_CHANNEL_IDLE, "grpclb"); - return &glb_policy->base; } diff --git a/src/core/ext/lb_policy/grpclb/grpclb_channel.c b/src/core/ext/lb_policy/grpclb/grpclb_channel.c new file mode 100644 index 00000000000..1b8bbab1b69 --- /dev/null +++ b/src/core/ext/lb_policy/grpclb/grpclb_channel.c @@ -0,0 +1,77 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include + +#include "src/core/ext/client_channel/client_channel.h" +#include "src/core/ext/lb_policy/grpclb/grpclb_channel.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" +#include "src/core/lib/support/string.h" + +grpc_channel *grpc_lb_policy_grpclb_create_lb_channel( + grpc_exec_ctx *exec_ctx, const char *lb_service_target_addresses, + grpc_client_channel_factory *client_channel_factory, + grpc_channel_args *args) { + grpc_channel *lb_channel = grpc_client_channel_factory_create_channel( + exec_ctx, client_channel_factory, lb_service_target_addresses, + GRPC_CLIENT_CHANNEL_TYPE_LOAD_BALANCING, args); + return lb_channel; +} + +grpc_channel_args *get_lb_channel_args(grpc_exec_ctx *exec_ctx, + grpc_slice_hash_table *targets_info, + const grpc_channel_args *args) { + /* We strip out the channel arg for the LB policy name, since we want + * to use the default (pick_first) in this case. + * + * We also strip out the channel arg for the resolved addresses, since + * that will be generated by the name resolver used in the LB channel. + * Note that the LB channel will use the sockaddr resolver, so this + * won't actually generate a query to DNS (or some other name service). + * However, the addresses returned by the sockaddr resolver will have + * is_balancer=false, whereas our own addresses have is_balancer=true. + * We need the LB channel to return addresses with is_balancer=false + * so that it does not wind up recursively using the grpclb LB policy, + * as per the special case logic in client_channel.c. + * + * Lastly, we also strip out the channel arg for the server URI, + * since that will be different for the LB channel than for the parent + * channel (the client channel factory will re-add this arg with + * the right value). */ + static const char *keys_to_remove[] = { + GRPC_ARG_LB_POLICY_NAME, GRPC_ARG_LB_ADDRESSES, GRPC_ARG_SERVER_URI}; + return grpc_channel_args_copy_and_remove(args, keys_to_remove, + GPR_ARRAY_SIZE(keys_to_remove)); +} diff --git a/src/core/ext/lb_policy/grpclb/grpclb_channel.h b/src/core/ext/lb_policy/grpclb/grpclb_channel.h new file mode 100644 index 00000000000..f66082d78ee --- /dev/null +++ b/src/core/ext/lb_policy/grpclb/grpclb_channel.h @@ -0,0 +1,56 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_LB_POLICY_GRPCLB_GRPCLB_CHANNEL_H +#define GRPC_CORE_EXT_LB_POLICY_GRPCLB_GRPCLB_CHANNEL_H + +#include "src/core/ext/client_channel/lb_policy_factory.h" +#include "src/core/lib/slice/slice_hash_table.h" + +/** Create the channel used for communicating with an LB service. + * Note that an LB *service* may be comprised of several LB *servers*. + * + * \a lb_service_target_addresses is the target URI containing the addresses + * from resolving the LB service's name (eg, ipv4:10.0.0.1:1234,10.2.3.4:9876). + * \a client_channel_factory will be used for the creation of the LB channel, + * alongside the channel args passed in \a args. */ +grpc_channel *grpc_lb_policy_grpclb_create_lb_channel( + grpc_exec_ctx *exec_ctx, const char *lb_service_target_addresses, + grpc_client_channel_factory *client_channel_factory, + grpc_channel_args *args); + +grpc_channel_args *get_lb_channel_args(grpc_exec_ctx *exec_ctx, + grpc_slice_hash_table *targets_info, + const grpc_channel_args *args); + +#endif /* GRPC_CORE_EXT_LB_POLICY_GRPCLB_GRPCLB_CHANNEL_H */ diff --git a/src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c b/src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c new file mode 100644 index 00000000000..2fee5f1b8ea --- /dev/null +++ b/src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c @@ -0,0 +1,107 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include + +#include "src/core/ext/client_channel/client_channel.h" +#include "src/core/ext/lb_policy/grpclb/grpclb_channel.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" +#include "src/core/lib/security/credentials/credentials.h" +#include "src/core/lib/security/transport/lb_targets_info.h" +#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/support/string.h" + +grpc_channel *grpc_lb_policy_grpclb_create_lb_channel( + grpc_exec_ctx *exec_ctx, const char *lb_service_target_addresses, + grpc_client_channel_factory *client_channel_factory, + grpc_channel_args *args) { + grpc_channel_args *new_args = args; + grpc_channel_credentials *channel_credentials = + grpc_channel_credentials_find_in_args(args); + if (channel_credentials != NULL) { + /* Substitute the channel credentials with a version without call + * credentials: the load balancer is not necessarily trusted to handle + * bearer token credentials */ + static const char *keys_to_remove[] = {GRPC_ARG_CHANNEL_CREDENTIALS}; + grpc_channel_credentials *creds_sans_call_creds = + grpc_channel_credentials_duplicate_without_call_credentials( + channel_credentials); + GPR_ASSERT(creds_sans_call_creds != NULL); + grpc_arg args_to_add[] = { + grpc_channel_credentials_to_arg(creds_sans_call_creds)}; + /* Create the new set of channel args */ + new_args = grpc_channel_args_copy_and_add_and_remove( + args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), args_to_add, + GPR_ARRAY_SIZE(args_to_add)); + grpc_channel_credentials_unref(exec_ctx, creds_sans_call_creds); + } + grpc_channel *lb_channel = grpc_client_channel_factory_create_channel( + exec_ctx, client_channel_factory, lb_service_target_addresses, + GRPC_CLIENT_CHANNEL_TYPE_LOAD_BALANCING, new_args); + if (channel_credentials != NULL) { + grpc_channel_args_destroy(exec_ctx, new_args); + } + return lb_channel; +} + +grpc_channel_args *get_lb_channel_args(grpc_exec_ctx *exec_ctx, + grpc_slice_hash_table *targets_info, + const grpc_channel_args *args) { + const grpc_arg targets_info_arg = + grpc_lb_targets_info_create_channel_arg(targets_info); + /* We strip out the channel arg for the LB policy name, since we want + * to use the default (pick_first) in this case. + * + * We also strip out the channel arg for the resolved addresses, since + * that will be generated by the name resolver used in the LB channel. + * Note that the LB channel will use the sockaddr resolver, so this + * won't actually generate a query to DNS (or some other name service). + * However, the addresses returned by the sockaddr resolver will have + * is_balancer=false, whereas our own addresses have is_balancer=true. + * We need the LB channel to return addresses with is_balancer=false + * so that it does not wind up recursively using the grpclb LB policy, + * as per the special case logic in client_channel.c. + * + * Lastly, we also strip out the channel arg for the server URI, + * since that will be different for the LB channel than for the parent + * channel (the client channel factory will re-add this arg with + * the right value). */ + static const char *keys_to_remove[] = { + GRPC_ARG_LB_POLICY_NAME, GRPC_ARG_LB_ADDRESSES, GRPC_ARG_SERVER_URI}; + /* Add the targets info table to be used for secure naming */ + return grpc_channel_args_copy_and_add_and_remove( + args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), &targets_info_arg, + 1); +} diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index d17d8fa057a..3e060d189ab 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -739,6 +739,13 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, sc_args.args = new_args; grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( exec_ctx, args->client_channel_factory, &sc_args); + if (grpc_lb_round_robin_trace) { + char *address_uri = + grpc_sockaddr_to_uri(&addresses->addresses[i].address); + gpr_log(GPR_DEBUG, "Created subchannel %p for address uri %s", + (void *)subchannel, address_uri); + gpr_free(address_uri); + } grpc_channel_args_destroy(exec_ctx, new_args); if (subchannel != NULL) { diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c index d3e53984f2d..d8c18eb1223 100644 --- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c +++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c @@ -40,10 +40,15 @@ #include "src/core/ext/client_channel/client_channel.h" #include "src/core/ext/client_channel/resolver_registry.h" +#include "src/core/ext/client_channel/uri_parser.h" #include "src/core/ext/transport/chttp2/client/chttp2_connector.h" #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/security/credentials/credentials.h" +#include "src/core/lib/security/transport/lb_targets_info.h" #include "src/core/lib/security/transport/security_connector.h" +#include "src/core/lib/slice/slice_hash_table.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" @@ -53,12 +58,114 @@ static void client_channel_factory_ref( static void client_channel_factory_unref( grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory) {} +static grpc_subchannel_args *get_secure_naming_subchannel_args( + grpc_exec_ctx *exec_ctx, const grpc_subchannel_args *args) { + grpc_channel_credentials *channel_credentials = + grpc_channel_credentials_find_in_args(args->args); + if (channel_credentials == NULL) { + gpr_log(GPR_ERROR, + "Can't create subchannel: channel credentials missing for secure " + "channel."); + return NULL; + } + // Make sure security connector does not already exist in args. + if (grpc_security_connector_find_in_args(args->args) != NULL) { + gpr_log(GPR_ERROR, + "Can't create subchannel: security connector already present in " + "channel args."); + return NULL; + } + // To which address are we connecting? By default, use the server URI. + const grpc_arg *server_uri_arg = + grpc_channel_args_find(args->args, GRPC_ARG_SERVER_URI); + GPR_ASSERT(server_uri_arg != NULL); + GPR_ASSERT(server_uri_arg->type == GRPC_ARG_STRING); + const char *server_uri_str = server_uri_arg->value.string; + GPR_ASSERT(server_uri_str != NULL); + grpc_uri *server_uri = + grpc_uri_parse(server_uri_str, true /* supress errors */); + GPR_ASSERT(server_uri != NULL); + const char *server_uri_path; + server_uri_path = + server_uri->path[0] == '/' ? server_uri->path + 1 : server_uri->path; + const grpc_slice_hash_table *targets_info = + grpc_lb_targets_info_find_in_args(args->args); + char *target_name_to_check = NULL; + if (targets_info != NULL) { // LB channel + // Find the balancer name for the target. + const char *target_uri_str = + grpc_get_subchannel_address_uri_arg(args->args); + grpc_uri *target_uri = + grpc_uri_parse(target_uri_str, false /* suppress errors */); + GPR_ASSERT(target_uri != NULL); + if (target_uri->path[0] != '\0') { // "path" may be empty + const grpc_slice key = grpc_slice_from_static_string( + target_uri->path[0] == '/' ? target_uri->path + 1 : target_uri->path); + const char *value = grpc_slice_hash_table_get(targets_info, key); + if (value != NULL) target_name_to_check = gpr_strdup(value); + grpc_slice_unref_internal(exec_ctx, key); + } + if (target_name_to_check == NULL) { + // If the target name to check hasn't already been set, fall back to using + // SERVER_URI + target_name_to_check = gpr_strdup(server_uri_path); + } + grpc_uri_destroy(target_uri); + } else { // regular channel: the secure name is the original server URI. + target_name_to_check = gpr_strdup(server_uri_path); + } + grpc_uri_destroy(server_uri); + GPR_ASSERT(target_name_to_check != NULL); + grpc_channel_security_connector *subchannel_security_connector = NULL; + // Create the security connector using the credentials and target name. + grpc_channel_args *new_args_from_connector = NULL; + const grpc_security_status security_status = + grpc_channel_credentials_create_security_connector( + exec_ctx, channel_credentials, target_name_to_check, args->args, + &subchannel_security_connector, &new_args_from_connector); + if (security_status != GRPC_SECURITY_OK) { + gpr_log(GPR_ERROR, + "Failed to create secure subchannel for secure name '%s'", + target_name_to_check); + gpr_free(target_name_to_check); + return NULL; + } + gpr_free(target_name_to_check); + grpc_arg new_security_connector_arg = + grpc_security_connector_to_arg(&subchannel_security_connector->base); + + grpc_channel_args *new_args = grpc_channel_args_copy_and_add( + new_args_from_connector != NULL ? new_args_from_connector : args->args, + &new_security_connector_arg, 1); + GRPC_SECURITY_CONNECTOR_UNREF(exec_ctx, &subchannel_security_connector->base, + "lb_channel_create"); + if (new_args_from_connector != NULL) { + grpc_channel_args_destroy(exec_ctx, new_args_from_connector); + } + grpc_subchannel_args *final_sc_args = gpr_malloc(sizeof(*final_sc_args)); + memcpy(final_sc_args, args, sizeof(*args)); + final_sc_args->args = new_args; + return final_sc_args; +} + static grpc_subchannel *client_channel_factory_create_subchannel( grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory, const grpc_subchannel_args *args) { + grpc_subchannel_args *subchannel_args = + get_secure_naming_subchannel_args(exec_ctx, args); + if (subchannel_args == NULL) { + gpr_log( + GPR_ERROR, + "Failed to create subchannel arguments during subchannel creation."); + return NULL; + } grpc_connector *connector = grpc_chttp2_connector_create(); - grpc_subchannel *s = grpc_subchannel_create(exec_ctx, connector, args); + grpc_subchannel *s = + grpc_subchannel_create(exec_ctx, connector, subchannel_args); grpc_connector_unref(exec_ctx, connector); + grpc_channel_args_destroy(exec_ctx, + (grpc_channel_args *)subchannel_args->args); + gpr_free(subchannel_args); return s; } @@ -91,10 +198,10 @@ static const grpc_client_channel_factory_vtable client_channel_factory_vtable = static grpc_client_channel_factory client_channel_factory = { &client_channel_factory_vtable}; -/* Create a secure client channel: - Asynchronously: - resolve target - - connect to it (trying alternatives as presented) - - perform handshakes */ +// Create a secure client channel: +// Asynchronously: - resolve target +// - connect to it (trying alternatives as presented) +// - perform handshakes grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds, const char *target, const grpc_channel_args *args, @@ -103,47 +210,25 @@ grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds, GRPC_API_TRACE( "grpc_secure_channel_create(creds=%p, target=%s, args=%p, " "reserved=%p)", - 4, (creds, target, args, reserved)); + 4, ((void *)creds, target, (void *)args, (void *)reserved)); GPR_ASSERT(reserved == NULL); - // Make sure security connector does not already exist in args. - if (grpc_find_security_connector_in_args(args) != NULL) { - gpr_log(GPR_ERROR, "Cannot set security context in channel args."); + grpc_channel *channel = NULL; + if (creds != NULL) { + // Add channel args containing the client channel factory and channel + // credentials. + grpc_arg args_to_add[] = { + grpc_client_channel_factory_create_channel_arg(&client_channel_factory), + grpc_channel_credentials_to_arg(creds)}; + grpc_channel_args *new_args = grpc_channel_args_copy_and_add( + args, args_to_add, GPR_ARRAY_SIZE(args_to_add)); + // Create channel. + channel = client_channel_factory_create_channel( + &exec_ctx, &client_channel_factory, target, + GRPC_CLIENT_CHANNEL_TYPE_REGULAR, new_args); + // Clean up. + grpc_channel_args_destroy(&exec_ctx, new_args); grpc_exec_ctx_finish(&exec_ctx); - return grpc_lame_client_channel_create( - target, GRPC_STATUS_INTERNAL, - "Security connector exists in channel args."); - } - // Create security connector and construct new channel args. - grpc_channel_security_connector *security_connector; - grpc_channel_args *new_args_from_connector; - if (grpc_channel_credentials_create_security_connector( - &exec_ctx, creds, target, args, &security_connector, - &new_args_from_connector) != GRPC_SECURITY_OK) { - grpc_exec_ctx_finish(&exec_ctx); - return grpc_lame_client_channel_create( - target, GRPC_STATUS_INTERNAL, "Failed to create security connector."); - } - // Add channel args containing the client channel factory and security - // connector. - grpc_arg args_to_add[2]; - args_to_add[0] = - grpc_client_channel_factory_create_channel_arg(&client_channel_factory); - args_to_add[1] = grpc_security_connector_to_arg(&security_connector->base); - grpc_channel_args *new_args = grpc_channel_args_copy_and_add( - new_args_from_connector != NULL ? new_args_from_connector : args, - args_to_add, GPR_ARRAY_SIZE(args_to_add)); - if (new_args_from_connector != NULL) { - grpc_channel_args_destroy(&exec_ctx, new_args_from_connector); } - // Create channel. - grpc_channel *channel = client_channel_factory_create_channel( - &exec_ctx, &client_channel_factory, target, - GRPC_CLIENT_CHANNEL_TYPE_REGULAR, new_args); - // Clean up. - GRPC_SECURITY_CONNECTOR_UNREF(&exec_ctx, &security_connector->base, - "secure_client_channel_factory_create_channel"); - grpc_channel_args_destroy(&exec_ctx, new_args); - grpc_exec_ctx_finish(&exec_ctx); return channel != NULL ? channel : grpc_lame_client_channel_create( target, GRPC_STATUS_INTERNAL, diff --git a/src/core/lib/iomgr/sockaddr_utils.c b/src/core/lib/iomgr/sockaddr_utils.c index 44bc2f968be..ffa62cb53c0 100644 --- a/src/core/lib/iomgr/sockaddr_utils.c +++ b/src/core/lib/iomgr/sockaddr_utils.c @@ -190,31 +190,37 @@ int grpc_sockaddr_to_string(char **out, } char *grpc_sockaddr_to_uri(const grpc_resolved_address *resolved_addr) { - char *temp; - char *result; grpc_resolved_address addr_normalized; - const struct sockaddr *addr; - if (grpc_sockaddr_is_v4mapped(resolved_addr, &addr_normalized)) { resolved_addr = &addr_normalized; } + const char *scheme = grpc_sockaddr_get_uri_scheme(resolved_addr); + if (scheme == NULL || strcmp("unix", scheme) == 0) { + return grpc_sockaddr_to_uri_unix_if_possible(resolved_addr); + } + char *path = NULL; + char *uri_str = NULL; + if (grpc_sockaddr_to_string(&path, resolved_addr, + false /* suppress errors */) && + scheme != NULL) { + gpr_asprintf(&uri_str, "%s:%s", scheme, path); + } + gpr_free(path); + return uri_str != NULL ? uri_str : NULL; +} - addr = (const struct sockaddr *)resolved_addr->addr; - +const char *grpc_sockaddr_get_uri_scheme( + const grpc_resolved_address *resolved_addr) { + const struct sockaddr *addr = (const struct sockaddr *)resolved_addr->addr; switch (addr->sa_family) { case AF_INET: - grpc_sockaddr_to_string(&temp, resolved_addr, 0); - gpr_asprintf(&result, "ipv4:%s", temp); - gpr_free(temp); - return result; + return "ipv4"; case AF_INET6: - grpc_sockaddr_to_string(&temp, resolved_addr, 0); - gpr_asprintf(&result, "ipv6:%s", temp); - gpr_free(temp); - return result; - default: - return grpc_sockaddr_to_uri_unix_if_possible(resolved_addr); + return "ipv6"; + case AF_UNIX: + return "unix"; } + return NULL; } int grpc_sockaddr_get_port(const grpc_resolved_address *resolved_addr) { diff --git a/src/core/lib/iomgr/sockaddr_utils.h b/src/core/lib/iomgr/sockaddr_utils.h index 5371e360c59..2b22f11b49d 100644 --- a/src/core/lib/iomgr/sockaddr_utils.h +++ b/src/core/lib/iomgr/sockaddr_utils.h @@ -84,6 +84,10 @@ int grpc_sockaddr_set_port(const grpc_resolved_address *addr, int port); int grpc_sockaddr_to_string(char **out, const grpc_resolved_address *addr, int normalize); +/* Returns the URI string corresponding to \a addr */ char *grpc_sockaddr_to_uri(const grpc_resolved_address *addr); +/* Returns the URI scheme corresponding to \a addr */ +const char *grpc_sockaddr_get_uri_scheme(const grpc_resolved_address *addr); + #endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_UTILS_H */ diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c index 9781a22a86d..b24697ce54b 100644 --- a/src/core/lib/security/credentials/credentials.c +++ b/src/core/lib/security/credentials/credentials.c @@ -160,6 +160,53 @@ grpc_channel_credentials_duplicate_without_call_credentials( } } +static void credentials_pointer_arg_destroy(grpc_exec_ctx *exec_ctx, void *p) { + grpc_channel_credentials_unref(exec_ctx, p); +} + +static void *credentials_pointer_arg_copy(void *p) { + return grpc_channel_credentials_ref(p); +} + +static int credentials_pointer_cmp(void *a, void *b) { return GPR_ICMP(a, b); } + +static const grpc_arg_pointer_vtable credentials_pointer_vtable = { + credentials_pointer_arg_copy, credentials_pointer_arg_destroy, + credentials_pointer_cmp}; + +grpc_arg grpc_channel_credentials_to_arg( + grpc_channel_credentials *credentials) { + grpc_arg result; + result.type = GRPC_ARG_POINTER; + result.key = GRPC_ARG_CHANNEL_CREDENTIALS; + result.value.pointer.vtable = &credentials_pointer_vtable; + result.value.pointer.p = credentials; + return result; +} + +grpc_channel_credentials *grpc_channel_credentials_from_arg( + const grpc_arg *arg) { + if (strcmp(arg->key, GRPC_ARG_CHANNEL_CREDENTIALS)) return NULL; + if (arg->type != GRPC_ARG_POINTER) { + gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type, + GRPC_ARG_CHANNEL_CREDENTIALS); + return NULL; + } + return arg->value.pointer.p; +} + +grpc_channel_credentials *grpc_channel_credentials_find_in_args( + const grpc_channel_args *args) { + size_t i; + if (args == NULL) return NULL; + for (i = 0; i < args->num_args; i++) { + grpc_channel_credentials *credentials = + grpc_channel_credentials_from_arg(&args->args[i]); + if (credentials != NULL) return credentials; + } + return NULL; +} + grpc_server_credentials *grpc_server_credentials_ref( grpc_server_credentials *creds) { if (creds == NULL) return NULL; diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h index 3011df6b8a2..510b79552a3 100644 --- a/src/core/lib/security/credentials/credentials.h +++ b/src/core/lib/security/credentials/credentials.h @@ -100,6 +100,8 @@ void grpc_override_well_known_credentials_path_getter( /* --- grpc_channel_credentials. --- */ +#define GRPC_ARG_CHANNEL_CREDENTIALS "grpc.channel_credentials" + typedef struct { void (*destruct)(grpc_exec_ctx *exec_ctx, grpc_channel_credentials *c); @@ -140,6 +142,17 @@ grpc_channel_credentials * grpc_channel_credentials_duplicate_without_call_credentials( grpc_channel_credentials *creds); +/* Util to encapsulate the channel credentials in a channel arg. */ +grpc_arg grpc_channel_credentials_to_arg(grpc_channel_credentials *credentials); + +/* Util to get the channel credentials from a channel arg. */ +grpc_channel_credentials *grpc_channel_credentials_from_arg( + const grpc_arg *arg); + +/* Util to find the channel credentials from channel args. */ +grpc_channel_credentials *grpc_channel_credentials_find_in_args( + const grpc_channel_args *args); + /* --- grpc_credentials_md. --- */ typedef struct { diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c index a8679d097d0..a0629f76ce1 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.c +++ b/src/core/lib/security/credentials/fake/fake_credentials.c @@ -35,13 +35,13 @@ #include -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/iomgr/executor.h" - #include #include #include +#include "src/core/lib/iomgr/executor.h" +#include "src/core/lib/support/string.h" + /* -- Fake transport security credentials. -- */ static grpc_security_status fake_transport_security_create_security_connector( @@ -49,7 +49,7 @@ static grpc_security_status fake_transport_security_create_security_connector( grpc_call_credentials *call_creds, const char *target, const grpc_channel_args *args, grpc_channel_security_connector **sc, grpc_channel_args **new_args) { - *sc = grpc_fake_channel_security_connector_create(call_creds); + *sc = grpc_fake_channel_security_connector_create(call_creds, target, args); return GRPC_SECURITY_OK; } diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h index 9cf38084a3d..0fe98417c6c 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.h +++ b/src/core/lib/security/credentials/fake/fake_credentials.h @@ -38,6 +38,21 @@ /* -- Fake transport security credentials. -- */ +/* Used to verify the target names given to the fake transport security + * connector. + * + * Its syntax by example: + * For LB channels: + * "backend_target_1,backend_target_2,...;lb_target_1,lb_target_2,..." + * For regular channels: + * "backend_taget_1,backend_target_2,..." + * + * That is to say, LB channels have a heading list of LB targets separated from + * the list of backend targets by a semicolon. For non-LB channels, only the + * latter is present. */ +#define GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS \ + "grpc.test_only.fake_security.expected_target" + /* Creates a fake transport security credentials object for testing. */ grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void); diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c index cf056e80080..b9bbe1b3043 100644 --- a/src/core/lib/security/transport/client_auth_filter.c +++ b/src/core/lib/security/transport/client_auth_filter.c @@ -335,7 +335,7 @@ static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_channel_element_args *args) { grpc_security_connector *sc = - grpc_find_security_connector_in_args(args->channel_args); + grpc_security_connector_find_in_args(args->channel_args); grpc_auth_context *auth_context = grpc_find_auth_context_in_args(args->channel_args); diff --git a/src/core/lib/security/transport/lb_targets_info.c b/src/core/lib/security/transport/lb_targets_info.c new file mode 100644 index 00000000000..e73483c0399 --- /dev/null +++ b/src/core/lib/security/transport/lb_targets_info.c @@ -0,0 +1,70 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include + +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/security/transport/lb_targets_info.h" + +/* Channel arg key for the mapping of LB server addresses to their names for + * secure naming purposes. */ +#define GRPC_ARG_LB_SECURE_NAMING_MAP "grpc.lb_secure_naming_map" + +static void *targets_info_copy(void *p) { return grpc_slice_hash_table_ref(p); } +static void targets_info_destroy(grpc_exec_ctx *exec_ctx, void *p) { + grpc_slice_hash_table_unref(exec_ctx, p); +} +static int targets_info_cmp(void *a, void *b) { return GPR_ICMP(a, b); } +static const grpc_arg_pointer_vtable server_to_balancer_names_vtable = { + targets_info_copy, targets_info_destroy, targets_info_cmp}; + +grpc_arg grpc_lb_targets_info_create_channel_arg( + grpc_slice_hash_table *targets_info) { + grpc_arg arg; + arg.type = GRPC_ARG_POINTER; + arg.key = GRPC_ARG_LB_SECURE_NAMING_MAP; + arg.value.pointer.p = targets_info; + arg.value.pointer.vtable = &server_to_balancer_names_vtable; + return arg; +} + +grpc_slice_hash_table *grpc_lb_targets_info_find_in_args( + const grpc_channel_args *args) { + const grpc_arg *targets_info_arg = + grpc_channel_args_find(args, GRPC_ARG_LB_SECURE_NAMING_MAP); + if (targets_info_arg != NULL) { + GPR_ASSERT(targets_info_arg->type == GRPC_ARG_POINTER); + return targets_info_arg->value.pointer.p; + } + return NULL; +} diff --git a/src/core/lib/security/transport/lb_targets_info.h b/src/core/lib/security/transport/lb_targets_info.h new file mode 100644 index 00000000000..5e6cacc1979 --- /dev/null +++ b/src/core/lib/security/transport/lb_targets_info.h @@ -0,0 +1,47 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_LB_TARGETS_INFO_H +#define GRPC_CORE_LIB_SECURITY_TRANSPORT_LB_TARGETS_INFO_H + +#include "src/core/lib/slice/slice_hash_table.h" + +/** Return a channel argument containing \a targets_info. */ +grpc_arg grpc_lb_targets_info_create_channel_arg( + grpc_slice_hash_table *targets_info); + +/** Return the instance of targets info in \a args or NULL */ +grpc_slice_hash_table *grpc_lb_targets_info_find_in_args( + const grpc_channel_args *args); + +#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_LB_TARGETS_INFO_H */ diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index b09127811b0..aeb04e33a31 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -43,10 +43,13 @@ #include #include "src/core/ext/transport/chttp2/alpn/alpn.h" +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/handshaker.h" #include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/context/security_context.h" #include "src/core/lib/security/credentials/credentials.h" +#include "src/core/lib/security/credentials/fake/fake_credentials.h" +#include "src/core/lib/security/transport/lb_targets_info.h" #include "src/core/lib/security/transport/secure_endpoint.h" #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/lib/support/env.h" @@ -205,23 +208,23 @@ static const grpc_arg_pointer_vtable connector_pointer_vtable = { grpc_arg grpc_security_connector_to_arg(grpc_security_connector *sc) { grpc_arg result; result.type = GRPC_ARG_POINTER; - result.key = GRPC_SECURITY_CONNECTOR_ARG; + result.key = GRPC_ARG_SECURITY_CONNECTOR; result.value.pointer.vtable = &connector_pointer_vtable; result.value.pointer.p = sc; return result; } grpc_security_connector *grpc_security_connector_from_arg(const grpc_arg *arg) { - if (strcmp(arg->key, GRPC_SECURITY_CONNECTOR_ARG)) return NULL; + if (strcmp(arg->key, GRPC_ARG_SECURITY_CONNECTOR)) return NULL; if (arg->type != GRPC_ARG_POINTER) { gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type, - GRPC_SECURITY_CONNECTOR_ARG); + GRPC_ARG_SECURITY_CONNECTOR); return NULL; } return arg->value.pointer.p; } -grpc_security_connector *grpc_find_security_connector_in_args( +grpc_security_connector *grpc_security_connector_find_in_args( const grpc_channel_args *args) { size_t i; if (args == NULL) return NULL; @@ -235,11 +238,21 @@ grpc_security_connector *grpc_find_security_connector_in_args( /* -- Fake implementation. -- */ +typedef struct { + grpc_channel_security_connector base; + char *target; + char *expected_targets; + bool is_lb_channel; +} grpc_fake_channel_security_connector; + static void fake_channel_destroy(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc) { - grpc_channel_security_connector *c = (grpc_channel_security_connector *)sc; - grpc_call_credentials_unref(exec_ctx, c->request_metadata_creds); - gpr_free(sc); + grpc_fake_channel_security_connector *c = + (grpc_fake_channel_security_connector *)sc; + grpc_call_credentials_unref(exec_ctx, c->base.request_metadata_creds); + gpr_free(c->target); + gpr_free(c->expected_targets); + gpr_free(c); } static void fake_server_destroy(grpc_exec_ctx *exec_ctx, @@ -247,6 +260,68 @@ static void fake_server_destroy(grpc_exec_ctx *exec_ctx, gpr_free(sc); } +static bool fake_check_target(const char *target_type, const char *target, + const char *set_str) { + GPR_ASSERT(target_type != NULL); + GPR_ASSERT(target != NULL); + char **set = NULL; + size_t set_size = 0; + gpr_string_split(set_str, ",", &set, &set_size); + bool found = false; + for (size_t i = 0; i < set_size; ++i) { + if (set[i] != NULL && strcmp(target, set[i]) == 0) found = true; + } + for (size_t i = 0; i < set_size; ++i) { + gpr_free(set[i]); + } + gpr_free(set); + return found; +} + +static void fake_secure_name_check(const char *target, + const char *expected_targets, + bool is_lb_channel) { + if (expected_targets == NULL) return; + char **lbs_and_backends = NULL; + size_t lbs_and_backends_size = 0; + bool success = false; + gpr_string_split(expected_targets, ";", &lbs_and_backends, + &lbs_and_backends_size); + if (lbs_and_backends_size > 2 || lbs_and_backends_size == 0) { + gpr_log(GPR_ERROR, "Invalid expected targets arg value: '%s'", + expected_targets); + goto done; + } + if (is_lb_channel) { + if (lbs_and_backends_size != 2) { + gpr_log(GPR_ERROR, + "Invalid expected targets arg value: '%s'. Expectations for LB " + "channels must be of the form 'be1,be2,be3,...;lb1,lb2,...", + expected_targets); + goto done; + } + if (!fake_check_target("LB", target, lbs_and_backends[1])) { + gpr_log(GPR_ERROR, "LB target '%s' not found in expected set '%s'", + target, lbs_and_backends[1]); + goto done; + } + success = true; + } else { + if (!fake_check_target("Backend", target, lbs_and_backends[0])) { + gpr_log(GPR_ERROR, "Backend target '%s' not found in expected set '%s'", + target, lbs_and_backends[0]); + goto done; + } + success = true; + } +done: + for (size_t i = 0; i < lbs_and_backends_size; ++i) { + gpr_free(lbs_and_backends[i]); + } + gpr_free(lbs_and_backends); + if (!success) abort(); +} + static void fake_check_peer(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc, tsi_peer peer, grpc_auth_context **auth_context, @@ -277,12 +352,28 @@ static void fake_check_peer(grpc_exec_ctx *exec_ctx, grpc_auth_context_add_cstring_property( *auth_context, GRPC_TRANSPORT_SECURITY_TYPE_PROPERTY_NAME, GRPC_FAKE_TRANSPORT_SECURITY_TYPE); - end: grpc_closure_sched(exec_ctx, on_peer_checked, error); tsi_peer_destruct(&peer); } +static void fake_channel_check_peer(grpc_exec_ctx *exec_ctx, + grpc_security_connector *sc, tsi_peer peer, + grpc_auth_context **auth_context, + grpc_closure *on_peer_checked) { + fake_check_peer(exec_ctx, sc, peer, auth_context, on_peer_checked); + grpc_fake_channel_security_connector *c = + (grpc_fake_channel_security_connector *)sc; + fake_secure_name_check(c->target, c->expected_targets, c->is_lb_channel); +} + +static void fake_server_check_peer(grpc_exec_ctx *exec_ctx, + grpc_security_connector *sc, tsi_peer peer, + grpc_auth_context **auth_context, + grpc_closure *on_peer_checked) { + fake_check_peer(exec_ctx, sc, peer, auth_context, on_peer_checked); +} + static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, const char *host, @@ -313,22 +404,32 @@ static void fake_server_add_handshakers(grpc_exec_ctx *exec_ctx, } static grpc_security_connector_vtable fake_channel_vtable = { - fake_channel_destroy, fake_check_peer}; + fake_channel_destroy, fake_channel_check_peer}; -static grpc_security_connector_vtable fake_server_vtable = {fake_server_destroy, - fake_check_peer}; +static grpc_security_connector_vtable fake_server_vtable = { + fake_server_destroy, fake_server_check_peer}; grpc_channel_security_connector *grpc_fake_channel_security_connector_create( - grpc_call_credentials *request_metadata_creds) { - grpc_channel_security_connector *c = gpr_malloc(sizeof(*c)); + grpc_call_credentials *request_metadata_creds, const char *target, + const grpc_channel_args *args) { + grpc_fake_channel_security_connector *c = gpr_malloc(sizeof(*c)); memset(c, 0, sizeof(*c)); - gpr_ref_init(&c->base.refcount, 1); - c->base.url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME; - c->base.vtable = &fake_channel_vtable; - c->request_metadata_creds = grpc_call_credentials_ref(request_metadata_creds); - c->check_call_host = fake_channel_check_call_host; - c->add_handshakers = fake_channel_add_handshakers; - return c; + gpr_ref_init(&c->base.base.refcount, 1); + c->base.base.url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME; + c->base.base.vtable = &fake_channel_vtable; + c->base.request_metadata_creds = + grpc_call_credentials_ref(request_metadata_creds); + c->base.check_call_host = fake_channel_check_call_host; + c->base.add_handshakers = fake_channel_add_handshakers; + c->target = gpr_strdup(target); + const grpc_arg *expected_target_arg = + grpc_channel_args_find(args, GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS); + if (expected_target_arg != NULL) { + GPR_ASSERT(expected_target_arg->type == GRPC_ARG_STRING); + c->expected_targets = gpr_strdup(expected_target_arg->value.string); + } + c->is_lb_channel = (grpc_lb_targets_info_find_in_args(args) != NULL); + return &c->base; } grpc_server_security_connector *grpc_fake_server_security_connector_create( diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h index eba4e6d1d7b..3df2fecd393 100644 --- a/src/core/lib/security/transport/security_connector.h +++ b/src/core/lib/security/transport/security_connector.h @@ -57,7 +57,7 @@ typedef enum { GRPC_SECURITY_OK = 0, GRPC_SECURITY_ERROR } grpc_security_status; typedef struct grpc_security_connector grpc_security_connector; -#define GRPC_SECURITY_CONNECTOR_ARG "grpc.security_connector" +#define GRPC_ARG_SECURITY_CONNECTOR "grpc.security_connector" typedef struct { void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc); @@ -115,7 +115,7 @@ grpc_arg grpc_security_connector_to_arg(grpc_security_connector *sc); grpc_security_connector *grpc_security_connector_from_arg(const grpc_arg *arg); /* Util to find the connector from channel args. */ -grpc_security_connector *grpc_find_security_connector_in_args( +grpc_security_connector *grpc_security_connector_find_in_args( const grpc_channel_args *args); /* --- channel_security_connector object. --- @@ -175,7 +175,8 @@ void grpc_server_security_connector_add_handshakers( /* For TESTING ONLY! Creates a fake connector that emulates real channel security. */ grpc_channel_security_connector *grpc_fake_channel_security_connector_create( - grpc_call_credentials *request_metadata_creds); + grpc_call_credentials *request_metadata_creds, const char *target, + const grpc_channel_args *args); /* For TESTING ONLY! Creates a fake connector that emulates real server security. */ diff --git a/src/core/lib/security/transport/security_handshaker.c b/src/core/lib/security/transport/security_handshaker.c index bb8a3bf6cdf..5d57543ac58 100644 --- a/src/core/lib/security/transport/security_handshaker.c +++ b/src/core/lib/security/transport/security_handshaker.c @@ -451,7 +451,7 @@ static void client_handshaker_factory_add_handshakers( grpc_exec_ctx *exec_ctx, grpc_handshaker_factory *handshaker_factory, const grpc_channel_args *args, grpc_handshake_manager *handshake_mgr) { grpc_channel_security_connector *security_connector = - (grpc_channel_security_connector *)grpc_find_security_connector_in_args( + (grpc_channel_security_connector *)grpc_security_connector_find_in_args( args); grpc_channel_security_connector_add_handshakers(exec_ctx, security_connector, handshake_mgr); @@ -461,7 +461,7 @@ static void server_handshaker_factory_add_handshakers( grpc_exec_ctx *exec_ctx, grpc_handshaker_factory *hf, const grpc_channel_args *args, grpc_handshake_manager *handshake_mgr) { grpc_server_security_connector *security_connector = - (grpc_server_security_connector *)grpc_find_security_connector_in_args( + (grpc_server_security_connector *)grpc_security_connector_find_in_args( args); grpc_server_security_connector_add_handshakers(exec_ctx, security_connector, handshake_mgr); diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c index a44407d3bb4..46b9a8f922c 100644 --- a/src/core/lib/surface/init_secure.c +++ b/src/core/lib/surface/init_secure.c @@ -56,7 +56,7 @@ static bool maybe_prepend_client_auth_filter( grpc_channel_stack_builder_get_channel_arguments(builder); if (args) { for (size_t i = 0; i < args->num_args; i++) { - if (0 == strcmp(GRPC_SECURITY_CONNECTOR_ARG, args->args[i].key)) { + if (0 == strcmp(GRPC_ARG_SECURITY_CONNECTOR, args->args[i].key)) { return grpc_channel_stack_builder_prepend_filter( builder, &grpc_client_auth_filter, NULL, NULL); } diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 6bca3ed1a58..a9f20e6d2a8 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -231,6 +231,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/security/credentials/plugin/plugin_credentials.c', 'src/core/lib/security/credentials/ssl/ssl_credentials.c', 'src/core/lib/security/transport/client_auth_filter.c', + 'src/core/lib/security/transport/lb_targets_info.c', 'src/core/lib/security/transport/secure_endpoint.c', 'src/core/lib/security/transport/security_connector.c', 'src/core/lib/security/transport/security_handshaker.c', @@ -271,6 +272,7 @@ CORE_SOURCE_FILES = [ 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c', 'src/core/ext/lb_policy/grpclb/grpclb.c', + 'src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', diff --git a/test/core/end2end/fake_resolver.c b/test/core/end2end/fake_resolver.c index 8e711c6b419..4f05f69f01f 100644 --- a/test/core/end2end/fake_resolver.c +++ b/test/core/end2end/fake_resolver.c @@ -154,12 +154,34 @@ static grpc_resolver* fake_resolver_create(grpc_exec_ctx* exec_ctx, grpc_uri_get_query_arg(args->uri, "lb_enabled"); const bool lb_enabled = lb_enabled_qpart != NULL && strcmp("0", lb_enabled_qpart) != 0; + + // Get the balancer's names. + const char* balancer_names = + grpc_uri_get_query_arg(args->uri, "balancer_names"); + grpc_slice_buffer balancer_names_parts; + grpc_slice_buffer_init(&balancer_names_parts); + if (balancer_names != NULL) { + const grpc_slice balancer_names_slice = + grpc_slice_from_copied_string(balancer_names); + grpc_slice_split(balancer_names_slice, ",", &balancer_names_parts); + grpc_slice_unref(balancer_names_slice); + } + // Construct addresses. grpc_slice path_slice = grpc_slice_new(args->uri->path, strlen(args->uri->path), do_nothing); grpc_slice_buffer path_parts; grpc_slice_buffer_init(&path_parts); grpc_slice_split(path_slice, ",", &path_parts); + if (balancer_names_parts.count > 0 && + path_parts.count != balancer_names_parts.count) { + gpr_log(GPR_ERROR, + "Balancer names present but mismatched with number of addresses: " + "%lu balancer names != %lu addresses", + (unsigned long)balancer_names_parts.count, + (unsigned long)path_parts.count); + return NULL; + } grpc_lb_addresses* addresses = grpc_lb_addresses_create(path_parts.count, NULL /* user_data_vtable */); bool errors_found = false; @@ -171,10 +193,15 @@ static grpc_resolver* fake_resolver_create(grpc_exec_ctx* exec_ctx, errors_found = true; } gpr_free(part_str); - addresses->addresses[i].is_balancer = lb_enabled; if (errors_found) break; + addresses->addresses[i].is_balancer = lb_enabled; + addresses->addresses[i].balancer_name = + balancer_names_parts.count > 0 + ? grpc_dump_slice(balancer_names_parts.slices[i], GPR_DUMP_ASCII) + : NULL; } grpc_slice_buffer_destroy_internal(exec_ctx, &path_parts); + grpc_slice_buffer_destroy_internal(exec_ctx, &balancer_names_parts); grpc_slice_unref(path_slice); if (errors_found) { grpc_lb_addresses_destroy(exec_ctx, addresses); diff --git a/test/core/surface/secure_channel_create_test.c b/test/core/surface/secure_channel_create_test.c index 280806707e2..567f8ae16e0 100644 --- a/test/core/surface/secure_channel_create_test.c +++ b/test/core/surface/secure_channel_create_test.c @@ -62,7 +62,7 @@ void test_security_connector_already_in_arg(void) { grpc_arg arg; arg.type = GRPC_ARG_POINTER; arg.value.pointer.p = NULL; - arg.key = GRPC_SECURITY_CONNECTOR_ARG; + arg.key = GRPC_ARG_SECURITY_CONNECTOR; grpc_channel_args args; args.num_args = 1; args.args = &arg; diff --git a/test/cpp/grpclb/grpclb_test.cc b/test/cpp/grpclb/grpclb_test.cc index 4b8a434c789..89ed9249adc 100644 --- a/test/cpp/grpclb/grpclb_test.cc +++ b/test/cpp/grpclb/grpclb_test.cc @@ -52,8 +52,10 @@ #include extern "C" { #include "src/core/ext/client_channel/client_channel.h" +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/iomgr/sockaddr.h" +#include "src/core/lib/security/credentials/fake/fake_credentials.h" #include "src/core/lib/support/string.h" #include "src/core/lib/support/tmpfile.h" #include "src/core/lib/surface/channel.h" @@ -110,6 +112,7 @@ typedef struct server_fixture { grpc_call *server_call; grpc_completion_queue *cq; char *servers_hostport; + const char *balancer_name; int port; const char *lb_token_prefix; gpr_thd_id tid; @@ -201,10 +204,12 @@ static void start_lb_server(server_fixture *sf, int *ports, size_t nports, &request_metadata_recv, sf->cq, sf->cq, tag(200)); GPR_ASSERT(GRPC_CALL_OK == error); - gpr_log(GPR_INFO, "LB Server[%s] up", sf->servers_hostport); + gpr_log(GPR_INFO, "LB Server[%s](%s) up", sf->servers_hostport, + sf->balancer_name); CQ_EXPECT_COMPLETION(cqv, tag(200), 1); cq_verify(cqv); - gpr_log(GPR_INFO, "LB Server[%s] after tag 200", sf->servers_hostport); + gpr_log(GPR_INFO, "LB Server[%s](%s) after tag 200", sf->servers_hostport, + sf->balancer_name); // make sure we've received the initial metadata from the grpclb request. GPR_ASSERT(request_metadata_recv.count > 0); @@ -221,7 +226,8 @@ static void start_lb_server(server_fixture *sf, int *ports, size_t nports, GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(202), 1); cq_verify(cqv); - gpr_log(GPR_INFO, "LB Server[%s] after RECV_MSG", sf->servers_hostport); + gpr_log(GPR_INFO, "LB Server[%s](%s) after RECV_MSG", sf->servers_hostport, + sf->balancer_name); // validate initial request. grpc_byte_buffer_reader bbr; @@ -250,7 +256,8 @@ static void start_lb_server(server_fixture *sf, int *ports, size_t nports, op++; error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(201), NULL); GPR_ASSERT(GRPC_CALL_OK == error); - gpr_log(GPR_INFO, "LB Server[%s] after tag 201", sf->servers_hostport); + gpr_log(GPR_INFO, "LB Server[%s](%s) after tag 201", sf->servers_hostport, + sf->balancer_name); for (int i = 0; i < 2; i++) { if (i == 0) { @@ -276,13 +283,14 @@ static void start_lb_server(server_fixture *sf, int *ports, size_t nports, GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(203), 1); cq_verify(cqv); - gpr_log(GPR_INFO, "LB Server[%s] after SEND_MESSAGE, iter %d", - sf->servers_hostport, i); + gpr_log(GPR_INFO, "LB Server[%s](%s) after SEND_MESSAGE, iter %d", + sf->servers_hostport, sf->balancer_name, i); grpc_byte_buffer_destroy(response_payload); grpc_slice_unref(response_payload_slice); } - gpr_log(GPR_INFO, "LB Server[%s] shutting down", sf->servers_hostport); + gpr_log(GPR_INFO, "LB Server[%s](%s) shutting down", sf->servers_hostport, + sf->balancer_name); op = ops; op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; @@ -299,8 +307,8 @@ static void start_lb_server(server_fixture *sf, int *ports, size_t nports, CQ_EXPECT_COMPLETION(cqv, tag(201), 1); CQ_EXPECT_COMPLETION(cqv, tag(204), 1); cq_verify(cqv); - gpr_log(GPR_INFO, "LB Server[%s] after tag 204. All done. LB server out", - sf->servers_hostport); + gpr_log(GPR_INFO, "LB Server[%s](%s) after tag 204. All done. LB server out", + sf->servers_hostport, sf->balancer_name); grpc_call_destroy(s); @@ -561,10 +569,38 @@ static void perform_request(client_fixture *cf) { gpr_free(peer); } -static void setup_client(const char *server_hostport, client_fixture *cf) { +#define BALANCERS_NAME "lb.name" +static void setup_client(const server_fixture *lb_server, + const server_fixture *backends, client_fixture *cf) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + char *lb_uri; + // The grpclb LB policy will be automatically selected by virtue of + // the fact that the returned addresses are balancer addresses. + gpr_asprintf(&lb_uri, "test:///%s?lb_enabled=1&balancer_names=%s", + lb_server->servers_hostport, lb_server->balancer_name); + + grpc_arg expected_target_arg; + expected_target_arg.type = GRPC_ARG_STRING; + expected_target_arg.key = + const_cast(GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS); + + char *expected_target_names = NULL; + const char *backends_name = lb_server->servers_hostport; + gpr_asprintf(&expected_target_names, "%s;%s", backends_name, BALANCERS_NAME); + + expected_target_arg.value.string = const_cast(expected_target_names); + grpc_channel_args *args = + grpc_channel_args_copy_and_add(NULL, &expected_target_arg, 1); + gpr_free(expected_target_names); + cf->cq = grpc_completion_queue_create(NULL); - cf->server_uri = gpr_strdup(server_hostport); - cf->client = grpc_insecure_channel_create(cf->server_uri, NULL, NULL); + cf->server_uri = lb_uri; + grpc_channel_credentials *fake_creds = + grpc_fake_transport_security_credentials_create(); + cf->client = + grpc_secure_channel_create(fake_creds, cf->server_uri, args, NULL); + grpc_channel_credentials_unref(&exec_ctx, fake_creds); + grpc_channel_args_destroy(&exec_ctx, args); } static void teardown_client(client_fixture *cf) { @@ -591,10 +627,14 @@ static void setup_server(const char *host, server_fixture *sf) { gpr_join_host_port(&sf->servers_hostport, host, sf->port); } + grpc_server_credentials *server_creds = + grpc_fake_transport_security_server_credentials_create(); + sf->server = grpc_server_create(NULL, NULL); grpc_server_register_completion_queue(sf->server, sf->cq, NULL); - GPR_ASSERT((assigned_port = grpc_server_add_insecure_http2_port( - sf->server, sf->servers_hostport)) > 0); + GPR_ASSERT((assigned_port = grpc_server_add_secure_http2_port( + sf->server, sf->servers_hostport, server_creds)) > 0); + grpc_server_credentials_release(server_creds); GPR_ASSERT(sf->port == assigned_port); grpc_server_start(sf->server); } @@ -656,17 +696,10 @@ static test_fixture setup_test_fixture(int lb_server_update_delay_ms) { } tf.lb_server.lb_token_prefix = LB_TOKEN_PREFIX; + tf.lb_server.balancer_name = BALANCERS_NAME; setup_server("127.0.0.1", &tf.lb_server); gpr_thd_new(&tf.lb_server.tid, fork_lb_server, &tf.lb_server, &options); - - char *server_uri; - // The grpclb LB policy will be automatically selected by virtue of - // the fact that the returned addresses are balancer addresses. - gpr_asprintf(&server_uri, "test:///%s?lb_enabled=1", - tf.lb_server.servers_hostport); - setup_client(server_uri, &tf.client); - gpr_free(server_uri); - + setup_client(&tf.lb_server, tf.lb_backends, &tf.client); return tf; } @@ -711,8 +744,9 @@ TEST(GrpclbTest, Updates) { // batch 1. All subsequent picks will come from the second half of the // backends, those coming in the LB update. tf_result = grpc::test_update(800); - GPR_ASSERT(tf_result.lb_backends[0].num_calls_serviced == 1); - GPR_ASSERT(tf_result.lb_backends[1].num_calls_serviced == 0); + GPR_ASSERT(tf_result.lb_backends[0].num_calls_serviced + + tf_result.lb_backends[1].num_calls_serviced == + 1); GPR_ASSERT(tf_result.lb_backends[2].num_calls_serviced + tf_result.lb_backends[3].num_calls_serviced > 0); @@ -728,8 +762,9 @@ TEST(GrpclbTest, Updates) { // update. In any case, the total number of serviced calls must again be equal // to four across all the backends. tf_result = grpc::test_update(2500); - GPR_ASSERT(tf_result.lb_backends[0].num_calls_serviced >= 1); - GPR_ASSERT(tf_result.lb_backends[1].num_calls_serviced == 1); + GPR_ASSERT(tf_result.lb_backends[0].num_calls_serviced + + tf_result.lb_backends[1].num_calls_serviced >= + 2); GPR_ASSERT(tf_result.lb_backends[2].num_calls_serviced + tf_result.lb_backends[3].num_calls_serviced > 0); diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 285b7dd3215..499390eb5b9 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -927,6 +927,8 @@ src/core/ext/client_channel/uri_parser.c \ src/core/ext/client_channel/uri_parser.h \ src/core/ext/lb_policy/grpclb/grpclb.c \ src/core/ext/lb_policy/grpclb/grpclb.h \ +src/core/ext/lb_policy/grpclb/grpclb_channel.h \ +src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.h \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ @@ -1191,6 +1193,8 @@ src/core/lib/security/credentials/ssl/ssl_credentials.c \ src/core/lib/security/credentials/ssl/ssl_credentials.h \ src/core/lib/security/transport/auth_filters.h \ src/core/lib/security/transport/client_auth_filter.c \ +src/core/lib/security/transport/lb_targets_info.c \ +src/core/lib/security/transport/lb_targets_info.h \ src/core/lib/security/transport/secure_endpoint.c \ src/core/lib/security/transport/secure_endpoint.h \ src/core/lib/security/transport/security_connector.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index def360afc85..6a8c29169bd 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -5197,7 +5197,7 @@ "census", "gpr", "grpc_base", - "grpc_lb_policy_grpclb", + "grpc_lb_policy_grpclb_secure", "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", "grpc_load_reporting", @@ -7514,6 +7514,7 @@ ], "headers": [ "src/core/ext/lb_policy/grpclb/grpclb.h", + "src/core/ext/lb_policy/grpclb/grpclb_channel.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" ], @@ -7523,6 +7524,37 @@ "src": [ "src/core/ext/lb_policy/grpclb/grpclb.c", "src/core/ext/lb_policy/grpclb/grpclb.h", + "src/core/ext/lb_policy/grpclb/grpclb_channel.c", + "src/core/ext/lb_policy/grpclb/grpclb_channel.h", + "src/core/ext/lb_policy/grpclb/load_balancer_api.c", + "src/core/ext/lb_policy/grpclb/load_balancer_api.h", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ + "gpr", + "grpc_base", + "grpc_client_channel", + "nanopb" + ], + "headers": [ + "src/core/ext/lb_policy/grpclb/grpclb.h", + "src/core/ext/lb_policy/grpclb/grpclb_channel.h", + "src/core/ext/lb_policy/grpclb/load_balancer_api.h", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + ], + "is_filegroup": true, + "language": "c", + "name": "grpc_lb_policy_grpclb_secure", + "src": [ + "src/core/ext/lb_policy/grpclb/grpclb.c", + "src/core/ext/lb_policy/grpclb/grpclb.h", + "src/core/ext/lb_policy/grpclb/grpclb_channel.h", + "src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", @@ -7638,6 +7670,7 @@ "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/transport/auth_filters.h", + "src/core/lib/security/transport/lb_targets_info.h", "src/core/lib/security/transport/secure_endpoint.h", "src/core/lib/security/transport/security_connector.h", "src/core/lib/security/transport/security_handshaker.h", @@ -7679,6 +7712,8 @@ "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/transport/auth_filters.h", "src/core/lib/security/transport/client_auth_filter.c", + "src/core/lib/security/transport/lb_targets_info.c", + "src/core/lib/security/transport/lb_targets_info.h", "src/core/lib/security/transport/secure_endpoint.c", "src/core/lib/security/transport/secure_endpoint.h", "src/core/lib/security/transport/security_connector.c", diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index c7aa771e5ff..4d010ac6072 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -437,6 +437,7 @@ + @@ -469,6 +470,7 @@ + @@ -804,6 +806,8 @@ + + @@ -884,6 +888,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 00aa8b27818..191fc18f868 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -472,6 +472,9 @@ src\core\lib\security\transport + + src\core\lib\security\transport + src\core\lib\security\transport @@ -592,6 +595,9 @@ src\core\ext\lb_policy\grpclb + + src\core\ext\lb_policy\grpclb + src\core\ext\lb_policy\grpclb @@ -1178,6 +1184,9 @@ src\core\lib\security\transport + + src\core\lib\security\transport + src\core\lib\security\transport @@ -1274,6 +1283,9 @@ src\core\ext\lb_policy\grpclb + + src\core\ext\lb_policy\grpclb + src\core\ext\lb_policy\grpclb diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index f2c9d1ae03f..6d120cc0fa1 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -437,6 +437,7 @@ + @@ -802,6 +803,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index d3117bd272d..c06af3fd86e 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -520,6 +520,9 @@ src\core\ext\lb_policy\grpclb + + src\core\ext\lb_policy\grpclb + src\core\ext\lb_policy\grpclb @@ -1121,6 +1124,9 @@ src\core\ext\lb_policy\grpclb + + src\core\ext\lb_policy\grpclb + src\core\ext\lb_policy\grpclb From 2b43481ff3d99339f2fdb8e1e681c0717bed2259 Mon Sep 17 00:00:00 2001 From: thinkerou Date: Wed, 8 Feb 2017 13:02:48 +0800 Subject: [PATCH 059/167] fix undefined name --- src/python/grpcio_tests/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index af0ffe34750..40f9b834546 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -28,7 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Provides distutils command classes for the gRPC Python setup process.""" -import distutils +from distutils import errors as _errors import glob import os import os.path @@ -186,7 +186,7 @@ class RunInterop(test.test): def finalize_options(self): if self.client and self.server: - raise DistutilsOptionError( + raise _errors.DistutilsOptionError( 'you may only specify one of client or server') def run(self): From c18f3ed03873a90a0f986f12a84e742c89fc84b0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Feb 2017 09:28:55 -0800 Subject: [PATCH 060/167] Reduce concurrency --- tools/run_tests/run_microbenchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 852b83967a9..851c506a2e8 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -94,7 +94,7 @@ for bm_name in sys.argv[1:]: '--source', '%s.trace' % fnize(line), '--fmt', 'simple', '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None)) cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)])) - if len(benchmarks) >= 2 * multiprocessing.cpu_count(): + if len(benchmarks) >= multiprocessing.cpu_count(): jobset.run(benchmarks, maxjobs=multiprocessing.cpu_count()/2, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) From 265775fc20496ea18d49c42d39fe727b65f78bc4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Feb 2017 11:40:14 -0800 Subject: [PATCH 061/167] Handle zero-length filenames --- src/core/lib/profiling/basic_timers.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/profiling/basic_timers.c b/src/core/lib/profiling/basic_timers.c index c4f27728227..1fbc9d7023d 100644 --- a/src/core/lib/profiling/basic_timers.c +++ b/src/core/lib/profiling/basic_timers.c @@ -90,7 +90,8 @@ static int g_writing_enabled = 1; static const char *output_filename() { if (output_filename_or_null == NULL) { output_filename_or_null = gpr_getenv("LATENCY_TRACE"); - if (output_filename_or_null == NULL) { + if (output_filename_or_null == NULL || + strlen(output_filename_or_null) == 0) { output_filename_or_null = "latency_trace.txt"; } } From f74d1727917747a0cef75fa9b849aa77cfa091c4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Feb 2017 11:38:09 -0800 Subject: [PATCH 062/167] Further limit concurrency --- tools/run_tests/run_microbenchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 851c506a2e8..7856affd8c4 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -94,7 +94,7 @@ for bm_name in sys.argv[1:]: '--source', '%s.trace' % fnize(line), '--fmt', 'simple', '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None)) cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)])) - if len(benchmarks) >= multiprocessing.cpu_count(): + if len(benchmarks) >= min(4, multiprocessing.cpu_count()): jobset.run(benchmarks, maxjobs=multiprocessing.cpu_count()/2, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) From 360c0d50654716966e2ca6e96110ffd15aabfee0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Feb 2017 13:36:44 -0800 Subject: [PATCH 063/167] Add comments, fix bug --- tools/profiling/latency_profile/run_latency_profile.sh | 2 +- tools/run_tests/run_microbenchmark.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/profiling/latency_profile/run_latency_profile.sh b/tools/profiling/latency_profile/run_latency_profile.sh index 7ebe308e0a0..715f525d2ff 100755 --- a/tools/profiling/latency_profile/run_latency_profile.sh +++ b/tools/profiling/latency_profile/run_latency_profile.sh @@ -44,4 +44,4 @@ else PYTHON=python2.7 fi -$PYTHON tools/profiling/microbenchmark/bm.py bm_fullstack +$PYTHON tools/run_tests/run_microbenchmark.py bm_fullstack diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 7856affd8c4..a5794dc9eee 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -94,7 +94,13 @@ for bm_name in sys.argv[1:]: '--source', '%s.trace' % fnize(line), '--fmt', 'simple', '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None)) cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)])) + # periodically flush out the list of jobs: profile_analysis jobs at least + # consume upwards of five gigabytes of ram in some cases, and so analysing + # hundreds of them at once is impractical -- but we want at least some + # concurrency or the work takes too long if len(benchmarks) >= min(4, multiprocessing.cpu_count()): + # run up to half the cpu count: each benchmark can use up to two cores + # (one for the microbenchmark, one for the data flush) jobset.run(benchmarks, maxjobs=multiprocessing.cpu_count()/2, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) @@ -102,6 +108,7 @@ for bm_name in sys.argv[1:]: benchmarks = [] profile_analysis = [] cleanup = [] + # run the remaining benchmarks that weren't flushed if len(benchmarks): jobset.run(benchmarks, maxjobs=multiprocessing.cpu_count()/2, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) @@ -131,4 +138,4 @@ for bm_name in sys.argv[1:]: index_html += "\n\n" with open('reports/index.html', 'w') as f: - w.write(index_html) + f.write(index_html) From 2ef0d54ffcddbd1dadd6cc546ce9df6c5c0bfb0f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Feb 2017 13:53:18 -0800 Subject: [PATCH 064/167] Handle one core --- src/core/lib/profiling/basic_timers.c | 1 + tools/run_tests/run_microbenchmark.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/lib/profiling/basic_timers.c b/src/core/lib/profiling/basic_timers.c index 1fbc9d7023d..1f1987fb8e7 100644 --- a/src/core/lib/profiling/basic_timers.c +++ b/src/core/lib/profiling/basic_timers.c @@ -43,6 +43,7 @@ #include #include #include +#include #include "src/core/lib/support/env.h" diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index a5794dc9eee..096d7d78ab2 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -101,7 +101,7 @@ for bm_name in sys.argv[1:]: if len(benchmarks) >= min(4, multiprocessing.cpu_count()): # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) - jobset.run(benchmarks, maxjobs=multiprocessing.cpu_count()/2, + jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2), add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) @@ -110,7 +110,7 @@ for bm_name in sys.argv[1:]: cleanup = [] # run the remaining benchmarks that weren't flushed if len(benchmarks): - jobset.run(benchmarks, maxjobs=multiprocessing.cpu_count()/2, + jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2), add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) From aa64ddf10962b811e0767b71775d8983ec89cab5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Feb 2017 14:20:08 -0800 Subject: [PATCH 065/167] Allow uploading results to BigQuery --- .../latency_profile/run_latency_profile.sh | 2 +- tools/profiling/microbenchmarks/bm2bq.py | 162 ++++++++++++++++++ tools/run_tests/run_microbenchmark.py | 59 ++++++- 3 files changed, 216 insertions(+), 7 deletions(-) create mode 100755 tools/profiling/microbenchmarks/bm2bq.py diff --git a/tools/profiling/latency_profile/run_latency_profile.sh b/tools/profiling/latency_profile/run_latency_profile.sh index 715f525d2ff..e9baee09574 100755 --- a/tools/profiling/latency_profile/run_latency_profile.sh +++ b/tools/profiling/latency_profile/run_latency_profile.sh @@ -44,4 +44,4 @@ else PYTHON=python2.7 fi -$PYTHON tools/run_tests/run_microbenchmark.py bm_fullstack +$PYTHON tools/run_tests/run_microbenchmark.py --collect summary perf latency --bigquery_upload diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py new file mode 100755 index 00000000000..124dbdfec52 --- /dev/null +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python2.7 +# +# Convert google-benchmark json output to something that can be uploaded to +# BigQuery +# +# +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import sys +import json +import csv +import os + +columns = [ + ('jenkins_build', 'integer'), + ('jenkins_job', 'string'), + ('date', 'timestamp'), + ('cpu_scaling_enabled', 'boolean'), + ('num_cpus', 'integer'), + ('mhz_per_cpu', 'integer'), + ('library_build_type', 'string'), + ('name', 'string'), + ('fixture', 'string'), + ('client_mutator', 'string'), + ('server_mutator', 'string'), + ('request_size', 'integer'), + ('response_size', 'integer'), + ('request_count', 'integer'), + ('iterations', 'integer'), + ('time_unit', 'string'), + ('real_time', 'integer'), + ('cpu_time', 'integer'), + ('bytes_per_second', 'float'), + ('allocs_per_iteration', 'float'), + ('locks_per_iteration', 'float'), + ('writes_per_iteration', 'float'), +] + +if sys.argv[1] == '--schema': + print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns) + sys.exit(0) + +with open(sys.argv[1]) as f: + js = json.loads(f.read()) + +writer = csv.DictWriter(sys.stdout, [c for c,t in columns]) + +bm_specs = { + 'BM_UnaryPingPong': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size', 'response_size'], + }, + 'BM_PumpStreamClientToServer': { + 'tpl': ['fixture'], + 'dyn': ['request_size'], + }, + 'BM_PumpStreamServerToClient': { + 'tpl': ['fixture'], + 'dyn': ['request_size'], + }, + 'BM_StreamingPingPong': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size', 'request_count'], + }, + 'BM_StreamingPingPongMsgs': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size'], + } +} + +def numericalize(s): + if not s: return '' + if s[-1] == 'k': + return int(s[:-1]) * 1024 + if s[-1] == 'M': + return int(s[:-1]) * 1024 * 1024 + if 0 <= (ord(s[-1]) - ord('0')) <= 9: + return int(s) + assert 'not a number: %s' % s + +def parse_name(name): + rest = name + out = {} + tpl_args = [] + dyn_args = [] + if '<' in rest: + tpl_bit = rest[rest.find('<') + 1 : rest.rfind('>')] + arg = '' + nesting = 0 + for c in tpl_bit: + if c == '<': + nesting += 1 + arg += c + elif c == '>': + nesting -= 1 + arg += c + elif c == ',': + if nesting == 0: + tpl_args.append(arg.strip()) + arg = '' + else: + arg += c + else: + arg += c + tpl_args.append(arg.strip()) + rest = rest[:rest.find('<')] + rest[rest.rfind('>') + 1:] + if '/' in rest: + s = rest.split('/') + rest = s[0] + dyn_args = s[1:] + name = rest + assert name in bm_specs + assert len(dyn_args) == len(bm_specs[name]['dyn']) + assert len(tpl_args) == len(bm_specs[name]['tpl']) + out['name'] = name + out.update(dict((k, numericalize(v)) for k, v in zip(bm_specs[name]['dyn'], dyn_args))) + out.update(dict(zip(bm_specs[name]['tpl'], tpl_args))) + return out + +for bm in js['benchmarks']: + context = js['context'] + labels_list = [s.split(':') for s in bm.get('label', '').split(' ')] + for el in labels_list: + el[0] = el[0].replace('/iter', '_per_iteration') + labels = dict(labels_list) + row = { + 'jenkins_build': os.environ.get('BUILD_NUMBER', ''), + 'jenkins_job': os.environ.get('JOB_NAME', ''), + } + row.update(context) + row.update(bm) + row.update(parse_name(row['name'])) + row.update(labels) + del row['label'] + writer.writerow(row) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 096d7d78ab2..2084599e352 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -32,6 +32,7 @@ import multiprocessing import os import subprocess import sys +import argparse import python_utils.jobset as jobset import python_utils.start_port_server as start_port_server @@ -72,12 +73,16 @@ def link(txt, tgt): global index_html index_html += "

%s

\n" % (tgt, txt) -benchmarks = [] -profile_analysis = [] -cleanup = [] +def text(txt): + global index_html + index_html += "

%s

" % txt + +def collect_latency(bm_name, args): + """generate latency profiles""" + benchmarks = [] + profile_analysis = [] + cleanup = [] -for bm_name in sys.argv[1:]: - # generate latency profiles heading('Latency Profiles: %s' % bm_name) subprocess.check_call( ['make', bm_name, @@ -115,7 +120,8 @@ for bm_name in sys.argv[1:]: jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) - # generate flamegraphs +def collect_perf(bm_name, args): + """generate flamegraphs""" heading('Flamegraphs: %s' % bm_name) subprocess.check_call( ['make', bm_name, @@ -136,6 +142,47 @@ for bm_name in sys.argv[1:]: f.write(subprocess.check_output([ '%s/flamegraph.pl' % flamegraph_dir, '/tmp/bm.folded'])) +def collect_summary(bm_name, args): + heading('Summary: %s' % bm_name) + subprocess.check_call( + ['make', bm_name, + 'CONFIG=counters', '-j', '%d' % multiprocessing.cpu_count()]) + text(subprocess.check_output(['bins/counters/%s' % bm_name, + '--benchmark_out=out.json', + '--benchmark_out_format=json'])) + if args.bigquery_upload: + with open('/tmp/out.csv', 'w') as f: + f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py out.json'])) + subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', 'out.csv']) + +collectors = { + 'latency': collect_latency, + 'perf': collect_perf, + 'summary': collect_summary, +} + +argp = argparse.ArgumentParser(description='Collect data from microbenchmarks') +argp.add_argument('-c', '--collect', + choices=sorted(collectors.keys()), + nargs='+', + default=sorted(collectors.keys()), + help='Which collectors should be run against each benchmark') +argp.add_argument('-b', '--benchmarks', + default=['bm_fullstack'], + nargs='+', + type=str, + help='Which microbenchmarks should be run') +argp.add_argument('--bigquery_upload', + default=False, + action='store_const', + const=True, + help='Upload results from summary collection to bigquery') +args = argp.parse_args() + +for bm_name in args.benchmarks: + for collect in args.collect: + collectors[collect](bm_name, args) + index_html += "\n\n" with open('reports/index.html', 'w') as f: f.write(index_html) From 5f4264f9a6b891fd032784ef9811c53d4a70bcdd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Feb 2017 14:53:04 -0800 Subject: [PATCH 066/167] Signal that authority can be compressed in hpack: saves 1 allocation/rpc --- src/core/lib/surface/channel.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index 429dbad7c77..d6acd392c1a 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -128,7 +128,8 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target, } channel->default_authority = grpc_mdelem_from_slices( exec_ctx, GRPC_MDSTR_AUTHORITY, - grpc_slice_from_copied_string(args->args[i].value.string)); + grpc_slice_intern( + grpc_slice_from_static_string(args->args[i].value.string))); } } else if (0 == strcmp(args->args[i].key, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG)) { @@ -144,7 +145,8 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target, } else { channel->default_authority = grpc_mdelem_from_slices( exec_ctx, GRPC_MDSTR_AUTHORITY, - grpc_slice_from_copied_string(args->args[i].value.string)); + grpc_slice_intern( + grpc_slice_from_static_string(args->args[i].value.string))); } } } else if (0 == strcmp(args->args[i].key, From 8f1b31530c0e6ae01072afb1682ee15ccc191bee Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Feb 2017 15:06:20 -0800 Subject: [PATCH 067/167] Use atomics for memory counters Avoids two mutex acquisitions per allocation in bm_fullstack (where we also count memory allocations) --- test/core/util/memory_counters.c | 40 ++++++++++++++------------------ test/core/util/memory_counters.h | 10 ++++---- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/test/core/util/memory_counters.c b/test/core/util/memory_counters.c index bebe94e5826..0a782b34708 100644 --- a/test/core/util/memory_counters.c +++ b/test/core/util/memory_counters.c @@ -39,7 +39,6 @@ #include "test/core/util/memory_counters.h" -static gpr_mu g_memory_mutex; static struct grpc_memory_counters g_memory_counters; static gpr_allocation_functions g_old_allocs; @@ -50,12 +49,10 @@ static void guard_free(void *vptr); static void *guard_malloc(size_t size) { size_t *ptr; if (!size) return NULL; - gpr_mu_lock(&g_memory_mutex); - g_memory_counters.total_size_absolute += size; - g_memory_counters.total_size_relative += size; - g_memory_counters.total_allocs_absolute++; - g_memory_counters.total_allocs_relative++; - gpr_mu_unlock(&g_memory_mutex); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_absolute, size); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, size); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_absolute, 1); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_relative, 1); ptr = g_old_allocs.malloc_fn(size + sizeof(size)); *ptr++ = size; return ptr; @@ -71,12 +68,10 @@ static void *guard_realloc(void *vptr, size_t size) { return NULL; } --ptr; - gpr_mu_lock(&g_memory_mutex); - g_memory_counters.total_size_absolute += size; - g_memory_counters.total_size_relative -= *ptr; - g_memory_counters.total_size_relative += size; - g_memory_counters.total_allocs_absolute++; - gpr_mu_unlock(&g_memory_mutex); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_absolute, size); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, -*ptr); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, size); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_absolute, 1); ptr = g_old_allocs.realloc_fn(ptr, size + sizeof(size)); *ptr++ = size; return ptr; @@ -86,10 +81,8 @@ static void guard_free(void *vptr) { size_t *ptr = vptr; if (!vptr) return; --ptr; - gpr_mu_lock(&g_memory_mutex); - g_memory_counters.total_size_relative -= *ptr; - g_memory_counters.total_allocs_relative--; - gpr_mu_unlock(&g_memory_mutex); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, -*ptr); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_relative, -1); g_old_allocs.free_fn(ptr); } @@ -98,20 +91,23 @@ struct gpr_allocation_functions g_guard_allocs = {guard_malloc, guard_realloc, void grpc_memory_counters_init() { memset(&g_memory_counters, 0, sizeof(g_memory_counters)); - gpr_mu_init(&g_memory_mutex); g_old_allocs = gpr_get_allocation_functions(); gpr_set_allocation_functions(g_guard_allocs); } void grpc_memory_counters_destroy() { gpr_set_allocation_functions(g_old_allocs); - gpr_mu_destroy(&g_memory_mutex); } struct grpc_memory_counters grpc_memory_counters_snapshot() { struct grpc_memory_counters counters; - gpr_mu_lock(&g_memory_mutex); - counters = g_memory_counters; - gpr_mu_unlock(&g_memory_mutex); + counters.total_size_relative = + gpr_atm_no_barrier_load(&g_memory_counters.total_size_relative); + counters.total_size_absolute = + gpr_atm_no_barrier_load(&g_memory_counters.total_size_absolute); + counters.total_allocs_relative = + gpr_atm_no_barrier_load(&g_memory_counters.total_allocs_relative); + counters.total_allocs_absolute = + gpr_atm_no_barrier_load(&g_memory_counters.total_allocs_absolute); return counters; } diff --git a/test/core/util/memory_counters.h b/test/core/util/memory_counters.h index b9b2b3adda5..51487c73ce7 100644 --- a/test/core/util/memory_counters.h +++ b/test/core/util/memory_counters.h @@ -34,13 +34,13 @@ #ifndef GRPC_TEST_CORE_UTIL_MEMORY_COUNTERS_H #define GRPC_TEST_CORE_UTIL_MEMORY_COUNTERS_H -#include +#include struct grpc_memory_counters { - size_t total_size_relative; - size_t total_size_absolute; - size_t total_allocs_relative; - size_t total_allocs_absolute; + gpr_atm total_size_relative; + gpr_atm total_size_absolute; + gpr_atm total_allocs_relative; + gpr_atm total_allocs_absolute; }; void grpc_memory_counters_init(); From 5fcc5af0946fa0eb2b03146bb48afa91ceabe73c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Feb 2017 15:22:05 -0800 Subject: [PATCH 068/167] Should only do host-->:authority if there is no :authority --- src/core/lib/channel/http_server_filter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c index 3f992977c08..b475dd6b86d 100644 --- a/src/core/lib/channel/http_server_filter.c +++ b/src/core/lib/channel/http_server_filter.c @@ -198,10 +198,10 @@ static grpc_error *server_filter_incoming_metadata(grpc_exec_ctx *exec_ctx, GRPC_ERROR_STR_KEY, ":path")); } - if (b->idx.named.host != NULL) { + if (b->idx.named.host != NULL && b->idx.named.authority == NULL) { add_error( error_name, &error, - grpc_metadata_batch_substitute( + grpc_metadata_batch_add_head( exec_ctx, b, b->idx.named.host, grpc_mdelem_from_slices( exec_ctx, GRPC_MDSTR_AUTHORITY, From fc4f72a977e65fa3b061b87952350f9b00ec6932 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Feb 2017 15:57:03 -0800 Subject: [PATCH 069/167] Fix uploader --- tools/run_tests/run_microbenchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 2084599e352..edc9e6a8001 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -152,7 +152,7 @@ def collect_summary(bm_name, args): '--benchmark_out_format=json'])) if args.bigquery_upload: with open('/tmp/out.csv', 'w') as f: - f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py out.json'])) + f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.json'])) subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', 'out.csv']) collectors = { From d1307fe61101a0ad3a224bed2e10bbf4cd8173f6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Feb 2017 18:02:21 -0800 Subject: [PATCH 070/167] Add \n --- tools/run_tests/run_microbenchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index edc9e6a8001..fe901056c59 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -75,7 +75,7 @@ def link(txt, tgt): def text(txt): global index_html - index_html += "

%s

" % txt + index_html += "

%s

\n" % txt def collect_latency(bm_name, args): """generate latency profiles""" From 9c7c4b91e54a9ad4314e689d69b07b4fc45fc56f Mon Sep 17 00:00:00 2001 From: Piotr Pietraszkiewicz Date: Thu, 9 Feb 2017 11:10:26 +0100 Subject: [PATCH 071/167] Cosmetics (removed one newline) --- templates/CMakeLists.txt.template | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 52a060a8808..1c2ef0a9c16 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -454,7 +454,6 @@ endif() endif() - % for src in lib.src: % if proto_re.match(src): protobuf_generate_grpc_cpp( From 7aff583720087c8dcdb8e0d319f28d09f4da3e6d Mon Sep 17 00:00:00 2001 From: Piotr Pietraszkiewicz Date: Thu, 9 Feb 2017 11:12:33 +0100 Subject: [PATCH 072/167] Regenerate CMakeListst.txt --- CMakeLists.txt | 341 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 341 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index d52e199842c..2ae92eaeda5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -696,6 +696,17 @@ add_library(gpr src/core/lib/support/wrap_memcpy.c ) +if(WIN32 AND MSVC) + set_target_properties(gpr PROPERTIES COMPILE_PDB_NAME "gpr" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/gpr.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(gpr PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -774,6 +785,17 @@ add_library(gpr_test_util test/core/util/test_config.c ) +if(WIN32 AND MSVC) + set_target_properties(gpr_test_util PROPERTIES COMPILE_PDB_NAME "gpr_test_util" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/gpr_test_util.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(gpr_test_util PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -1020,6 +1042,17 @@ add_library(grpc src/core/plugin_registry/grpc_plugin_registry.c ) +if(WIN32 AND MSVC) + set_target_properties(grpc PROPERTIES COMPILE_PDB_NAME "grpc" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -1286,6 +1319,17 @@ add_library(grpc_cronet src/core/plugin_registry/grpc_cronet_plugin_registry.c ) +if(WIN32 AND MSVC) + set_target_properties(grpc_cronet PROPERTIES COMPILE_PDB_NAME "grpc_cronet" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc_cronet.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc_cronet PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -1497,6 +1541,17 @@ add_library(grpc_test_util src/core/lib/transport/transport_op_string.c ) +if(WIN32 AND MSVC) + set_target_properties(grpc_test_util PROPERTIES COMPILE_PDB_NAME "grpc_test_util" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc_test_util.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc_test_util PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -1575,6 +1630,17 @@ add_library(grpc_test_util_unsecure test/core/util/slice_splitter.c ) +if(WIN32 AND MSVC) + set_target_properties(grpc_test_util_unsecure PROPERTIES COMPILE_PDB_NAME "grpc_test_util_unsecure" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc_test_util_unsecure.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc_test_util_unsecure PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -1796,6 +1862,17 @@ add_library(grpc_unsecure src/core/plugin_registry/grpc_unsecure_plugin_registry.c ) +if(WIN32 AND MSVC) + set_target_properties(grpc_unsecure PROPERTIES COMPILE_PDB_NAME "grpc_unsecure" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc_unsecure.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc_unsecure PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -1867,6 +1944,17 @@ add_library(reconnect_server test/core/util/reconnect_server.c ) +if(WIN32 AND MSVC) + set_target_properties(reconnect_server PROPERTIES COMPILE_PDB_NAME "reconnect_server" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/reconnect_server.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(reconnect_server PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -1896,6 +1984,17 @@ add_library(test_tcp_server test/core/util/test_tcp_server.c ) +if(WIN32 AND MSVC) + set_target_properties(test_tcp_server PROPERTIES COMPILE_PDB_NAME "test_tcp_server" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/test_tcp_server.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(test_tcp_server PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -1959,6 +2058,17 @@ add_library(grpc++ src/cpp/codegen/codegen_init.cc ) +if(WIN32 AND MSVC) + set_target_properties(grpc++ PROPERTIES COMPILE_PDB_NAME "grpc++" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc++ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -2309,6 +2419,17 @@ add_library(grpc++_cronet third_party/nanopb/pb_encode.c ) +if(WIN32 AND MSVC) + set_target_properties(grpc++_cronet PROPERTIES COMPILE_PDB_NAME "grpc++_cronet" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++_cronet.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc++_cronet PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -2460,6 +2581,17 @@ add_library(grpc++_proto_reflection_desc_db ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h ) +if(WIN32 AND MSVC) + set_target_properties(grpc++_proto_reflection_desc_db PROPERTIES COMPILE_PDB_NAME "grpc++_proto_reflection_desc_db" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++_proto_reflection_desc_db.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + protobuf_generate_grpc_cpp( src/proto/grpc/reflection/v1alpha/reflection.proto ) @@ -2505,6 +2637,17 @@ add_library(grpc++_reflection ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h ) +if(WIN32 AND MSVC) + set_target_properties(grpc++_reflection PROPERTIES COMPILE_PDB_NAME "grpc++_reflection" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++_reflection.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + protobuf_generate_grpc_cpp( src/proto/grpc/reflection/v1alpha/reflection.proto ) @@ -2552,6 +2695,17 @@ add_library(grpc++_test src/cpp/test/server_context_test_spouse.cc ) +if(WIN32 AND MSVC) + set_target_properties(grpc++_test PROPERTIES COMPILE_PDB_NAME "grpc++_test" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++_test.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc++_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -2581,6 +2735,17 @@ add_library(grpc++_test_config test/cpp/util/test_config_cc.cc ) +if(WIN32 AND MSVC) + set_target_properties(grpc++_test_config PROPERTIES COMPILE_PDB_NAME "grpc++_test_config" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++_test_config.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc++_test_config PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -2627,6 +2792,17 @@ add_library(grpc++_test_util src/cpp/codegen/codegen_init.cc ) +if(WIN32 AND MSVC) + set_target_properties(grpc++_test_util PROPERTIES COMPILE_PDB_NAME "grpc++_test_util" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++_test_util.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + protobuf_generate_grpc_cpp( src/proto/grpc/testing/echo_messages.proto ) @@ -2757,6 +2933,17 @@ add_library(grpc++_unsecure src/cpp/codegen/codegen_init.cc ) +if(WIN32 AND MSVC) + set_target_properties(grpc++_unsecure PROPERTIES COMPILE_PDB_NAME "grpc++_unsecure" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++_unsecure.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc++_unsecure PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -2901,6 +3088,17 @@ add_library(grpc_cli_libs ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h ) +if(WIN32 AND MSVC) + set_target_properties(grpc_cli_libs PROPERTIES COMPILE_PDB_NAME "grpc_cli_libs" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc_cli_libs.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + protobuf_generate_grpc_cpp( src/proto/grpc/reflection/v1alpha/reflection.proto ) @@ -2948,6 +3146,17 @@ add_library(grpc_plugin_support src/compiler/ruby_generator.cc ) +if(WIN32 AND MSVC) + set_target_properties(grpc_plugin_support PROPERTIES COMPILE_PDB_NAME "grpc_plugin_support" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc_plugin_support.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc_plugin_support PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -3004,6 +3213,17 @@ add_library(http2_client_main test/cpp/interop/http2_client.cc ) +if(WIN32 AND MSVC) + set_target_properties(http2_client_main PROPERTIES COMPILE_PDB_NAME "http2_client_main" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/http2_client_main.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + protobuf_generate_grpc_cpp( src/proto/grpc/testing/empty.proto ) @@ -3050,6 +3270,17 @@ add_library(interop_client_helper test/cpp/interop/client_helper.cc ) +if(WIN32 AND MSVC) + set_target_properties(interop_client_helper PROPERTIES COMPILE_PDB_NAME "interop_client_helper" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/interop_client_helper.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + protobuf_generate_grpc_cpp( src/proto/grpc/testing/messages.proto ) @@ -3099,6 +3330,17 @@ add_library(interop_client_main test/cpp/interop/interop_client.cc ) +if(WIN32 AND MSVC) + set_target_properties(interop_client_main PROPERTIES COMPILE_PDB_NAME "interop_client_main" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/interop_client_main.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + protobuf_generate_grpc_cpp( src/proto/grpc/testing/empty.proto ) @@ -3144,6 +3386,17 @@ add_library(interop_server_helper test/cpp/interop/server_helper.cc ) +if(WIN32 AND MSVC) + set_target_properties(interop_server_helper PROPERTIES COMPILE_PDB_NAME "interop_server_helper" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/interop_server_helper.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(interop_server_helper PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -3189,6 +3442,17 @@ add_library(interop_server_lib test/cpp/interop/interop_server.cc ) +if(WIN32 AND MSVC) + set_target_properties(interop_server_lib PROPERTIES COMPILE_PDB_NAME "interop_server_lib" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/interop_server_lib.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + protobuf_generate_grpc_cpp( src/proto/grpc/testing/empty.proto ) @@ -3234,6 +3498,17 @@ add_library(interop_server_main test/cpp/interop/interop_server_bootstrap.cc ) +if(WIN32 AND MSVC) + set_target_properties(interop_server_main PROPERTIES COMPILE_PDB_NAME "interop_server_main" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/interop_server_main.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(interop_server_main PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -3292,6 +3567,17 @@ add_library(qps test/cpp/util/benchmark_config.cc ) +if(WIN32 AND MSVC) + set_target_properties(qps PROPERTIES COMPILE_PDB_NAME "qps" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/qps.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + protobuf_generate_grpc_cpp( src/proto/grpc/testing/messages.proto ) @@ -3337,6 +3623,17 @@ add_library(grpc_csharp_ext SHARED src/csharp/ext/grpc_csharp_ext.c ) +if(WIN32 AND MSVC) + set_target_properties(grpc_csharp_ext PROPERTIES COMPILE_PDB_NAME "grpc_csharp_ext" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc_csharp_ext.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(grpc_csharp_ext PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -3371,6 +3668,17 @@ add_library(bad_client_test test/core/bad_client/bad_client.c ) +if(WIN32 AND MSVC) + set_target_properties(bad_client_test PROPERTIES COMPILE_PDB_NAME "bad_client_test" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/bad_client_test.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(bad_client_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -3399,6 +3707,17 @@ add_library(bad_ssl_test_server test/core/bad_ssl/server_common.c ) +if(WIN32 AND MSVC) + set_target_properties(bad_ssl_test_server PROPERTIES COMPILE_PDB_NAME "bad_ssl_test_server" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/bad_ssl_test_server.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(bad_ssl_test_server PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -3477,6 +3796,17 @@ add_library(end2end_tests test/core/end2end/tests/write_buffering_at_end.c ) +if(WIN32 AND MSVC) + set_target_properties(end2end_tests PROPERTIES COMPILE_PDB_NAME "end2end_tests" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/end2end_tests.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(end2end_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} @@ -3555,6 +3885,17 @@ add_library(end2end_nosec_tests test/core/end2end/tests/write_buffering_at_end.c ) +if(WIN32 AND MSVC) + set_target_properties(end2end_nosec_tests PROPERTIES COMPILE_PDB_NAME "end2end_nosec_tests" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/end2end_nosec_tests.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + target_include_directories(end2end_nosec_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} From a5fac1f8a00b0ba6ca784baa4783ab947579693b Mon Sep 17 00:00:00 2001 From: Piotr Pietraszkiewicz Date: Thu, 9 Feb 2017 11:15:44 +0100 Subject: [PATCH 073/167] Regenerate CMakeLists.txt --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index d52e199842c..30ed8168c5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2072,6 +2072,8 @@ foreach(_hdr include/grpc/impl/codegen/sync_generic.h include/grpc/impl/codegen/sync_posix.h include/grpc/impl/codegen/sync_windows.h + include/grpc++/impl/codegen/proto_utils.h + include/grpc++/impl/codegen/config_protobuf.h ) string(REPLACE "include/" "" _path ${_hdr}) get_filename_component(_path ${_path} PATH) From 44cc814bcafb470049f26de8e0a45ccc2c1a8c78 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 08:00:31 -0800 Subject: [PATCH 074/167] Add lots of casts --- test/core/util/memory_counters.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/test/core/util/memory_counters.c b/test/core/util/memory_counters.c index 0a782b34708..fd6770bd49a 100644 --- a/test/core/util/memory_counters.c +++ b/test/core/util/memory_counters.c @@ -49,10 +49,14 @@ static void guard_free(void *vptr); static void *guard_malloc(size_t size) { size_t *ptr; if (!size) return NULL; - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_absolute, size); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, size); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_absolute, 1); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_relative, 1); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_absolute, + (gpr_atm)size); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, + (gpr_atm)size); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_absolute, + (gpr_atm)1); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_relative, + (gpr_atm)1); ptr = g_old_allocs.malloc_fn(size + sizeof(size)); *ptr++ = size; return ptr; @@ -68,10 +72,14 @@ static void *guard_realloc(void *vptr, size_t size) { return NULL; } --ptr; - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_absolute, size); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, -*ptr); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, size); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_absolute, 1); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_absolute, + (gpr_atm)size); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, + -(gpr_atm)*ptr); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, + (gpr_atm)size); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_absolute, + (gpr_atm)1); ptr = g_old_allocs.realloc_fn(ptr, size + sizeof(size)); *ptr++ = size; return ptr; @@ -81,8 +89,10 @@ static void guard_free(void *vptr) { size_t *ptr = vptr; if (!vptr) return; --ptr; - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, -*ptr); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_relative, -1); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, + -(gpr_atm)*ptr); + gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_relative, + -(gpr_atm)1); g_old_allocs.free_fn(ptr); } From 7f8eb7feed0f8c2d826921675a5e1297bb037643 Mon Sep 17 00:00:00 2001 From: "David G. Quintas" Date: Thu, 9 Feb 2017 09:07:40 -0800 Subject: [PATCH 075/167] Revert "Port #9488 and part of #9463" --- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler.podspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 1a3b775c609..3429e2b29bd 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.1.0' + s.dependency '!ProtoCompiler', '3.0.2' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.1' s.osx.deployment_target = '10.9' diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index dc4d8e964e6..b55f6c93c62 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.1.0' + v = '3.0.2' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC From 387885954b3f1337e2b6af3d0ab7dc7ce6a53718 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 9 Feb 2017 09:28:53 -0800 Subject: [PATCH 076/167] Don't use proxy when target is a Unix domain socket. --- src/core/ext/client_channel/http_proxy.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/core/ext/client_channel/http_proxy.c b/src/core/ext/client_channel/http_proxy.c index 7daa0714951..bbe4ff550c6 100644 --- a/src/core/ext/client_channel/http_proxy.c +++ b/src/core/ext/client_channel/http_proxy.c @@ -87,6 +87,12 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, if (uri != NULL) grpc_uri_destroy(uri); return false; } + if (strcmp(uri->scheme, "unix") == 0) { + gpr_log(GPR_INFO, "not using proxy for Unix domain socket '%s'", + server_uri); + grpc_uri_destroy(uri); + return false; + } grpc_arg new_arg; new_arg.key = GRPC_ARG_HTTP_CONNECT_SERVER; new_arg.type = GRPC_ARG_STRING; From 76296f93760c6d4b5b04aa8bdd4a95a4dae12acb Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 27 Jan 2017 17:20:20 -0800 Subject: [PATCH 077/167] Advance protoc version to 3.1.0 --- src/objective-c/!ProtoCompiler.podspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index b55f6c93c62..dc4d8e964e6 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.0.2' + v = '3.1.0' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC From e5ca175aeb1e8c47c1a588a2ffa9dd884362edae Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Sat, 28 Jan 2017 20:34:05 -0800 Subject: [PATCH 078/167] Advance protoc dependency version as well --- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 3429e2b29bd..1a3b775c609 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.0.2' + s.dependency '!ProtoCompiler', '3.1.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.1' s.osx.deployment_target = '10.9' From 0fccdb83273e3d876dfb5c03f548b9347f13dd83 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 9 Feb 2017 11:21:57 -0800 Subject: [PATCH 079/167] Advance protoc podspec template version to 3.1.0 as well --- .../src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template index 80420af7db2..3a10cfab3cc 100644 --- a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template +++ b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template @@ -103,7 +103,7 @@ s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.0.2' + s.dependency '!ProtoCompiler', '3.1.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.1' s.osx.deployment_target = '10.9' From d1c2866968b63e82ebeb47607d460ea869ca0ecc Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Mon, 6 Feb 2017 16:37:45 -0800 Subject: [PATCH 080/167] remove fatal null check in c# protobuf QPS server --- src/csharp/Grpc.IntegrationTesting/ServerRunners.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs index 8689d188aed..4591b72cbaa 100644 --- a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs +++ b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs @@ -79,8 +79,7 @@ namespace Grpc.IntegrationTesting ServerServiceDefinition service = null; if (config.ServerType == ServerType.AsyncServer) { - GrpcPreconditions.CheckArgument(config.PayloadConfig == null, - "ServerConfig.PayloadConfig shouldn't be set for BenchmarkService based server."); + Logger.Warning("ServerConfig.PayloadConfig is not supported for C# ServerType.AsyncServer. Ignoring the value"); service = BenchmarkService.BindService(new BenchmarkServiceImpl()); } else if (config.ServerType == ServerType.AsyncGenericServer) From f797c651609c4a99b13ad8b0c3bf2b82eb189b15 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Wed, 8 Feb 2017 14:29:30 -0800 Subject: [PATCH 081/167] make the change in scenario config rather than c# --- src/csharp/Grpc.IntegrationTesting/ServerRunners.cs | 3 ++- src/proto/grpc/testing/control.proto | 5 ++++- tools/run_tests/performance/scenario_config.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs index 4591b72cbaa..8689d188aed 100644 --- a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs +++ b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs @@ -79,7 +79,8 @@ namespace Grpc.IntegrationTesting ServerServiceDefinition service = null; if (config.ServerType == ServerType.AsyncServer) { - Logger.Warning("ServerConfig.PayloadConfig is not supported for C# ServerType.AsyncServer. Ignoring the value"); + GrpcPreconditions.CheckArgument(config.PayloadConfig == null, + "ServerConfig.PayloadConfig shouldn't be set for BenchmarkService based server."); service = BenchmarkService.BindService(new BenchmarkServiceImpl()); } else if (config.ServerType == ServerType.AsyncGenericServer) diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto index 529313cfb1e..8f0d25c2c96 100644 --- a/src/proto/grpc/testing/control.proto +++ b/src/proto/grpc/testing/control.proto @@ -139,7 +139,10 @@ message ServerConfig { int32 async_server_threads = 7; // Specify the number of cores to limit server to, if desired int32 core_limit = 8; - // payload config, used in generic server + // payload config, used in generic server. + // Note this must NOT be used in proto (non-generic) servers. For proto servers, + // 'response sizes' must be configured from the 'response_size' field of the + // 'SimpleRequest' objects in RPC requests. PayloadConfig payload_config = 9; // Specify the cores we should run the server on, if desired diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 6d138186ced..865125fcd78 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -143,8 +143,9 @@ def _ping_pong_scenario(name, rpc_type, if use_generic_payload: if server_type != 'ASYNC_GENERIC_SERVER': raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.') + scenario['server_config']['payload_config'] = _payload_type(use_generic_payload, req_size, resp_size) + scenario['client_config']['payload_config'] = _payload_type(use_generic_payload, req_size, resp_size) - scenario['server_config']['payload_config'] = _payload_type(use_generic_payload, req_size, resp_size) if unconstrained_client: outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[unconstrained_client] From e0aed71d85ce807e860918d45a38778a83270b63 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Wed, 8 Feb 2017 17:30:55 -0800 Subject: [PATCH 082/167] re-run generate projects --- tools/run_tests/generated/tests.json | 112 +++++++++++++-------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 9206d209b54..c193caa26f0 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -38487,7 +38487,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38514,7 +38514,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38541,7 +38541,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38566,7 +38566,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38591,7 +38591,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38616,7 +38616,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38641,7 +38641,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38666,7 +38666,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38691,7 +38691,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38716,7 +38716,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38741,7 +38741,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38766,7 +38766,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38791,7 +38791,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38816,7 +38816,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38916,7 +38916,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38943,7 +38943,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38970,7 +38970,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -38995,7 +38995,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39020,7 +39020,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39045,7 +39045,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39070,7 +39070,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39095,7 +39095,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39120,7 +39120,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39145,7 +39145,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39170,7 +39170,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39195,7 +39195,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39220,7 +39220,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39245,7 +39245,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39378,7 +39378,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 10, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 10, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39416,7 +39416,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 10, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 10, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39454,7 +39454,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39490,7 +39490,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39526,7 +39526,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39562,7 +39562,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39598,7 +39598,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39634,7 +39634,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39670,7 +39670,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39706,7 +39706,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39742,7 +39742,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39778,7 +39778,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39814,7 +39814,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39850,7 +39850,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -39994,7 +39994,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 10, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 10, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40032,7 +40032,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 10, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_servers\": 1, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 10, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40070,7 +40070,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40106,7 +40106,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40142,7 +40142,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40178,7 +40178,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40214,7 +40214,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40250,7 +40250,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40286,7 +40286,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40322,7 +40322,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40358,7 +40358,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40394,7 +40394,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40430,7 +40430,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -40466,7 +40466,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ From befafe64f9a010920c492956ee8df1257a6d4e77 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 11:30:54 -0800 Subject: [PATCH 083/167] Initial pass to remove mu_lock from client_channel: trickier cases remain --- src/core/ext/client_channel/client_channel.c | 291 ++++++++++--------- 1 file changed, 150 insertions(+), 141 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 06038bb5ba9..2595acd8c42 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -51,6 +51,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/connected_channel.h" #include "src/core/lib/channel/deadline_filter.h" +#include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/profiling/timers.h" @@ -160,8 +161,8 @@ typedef struct client_channel_channel_data { /** client channel factory */ grpc_client_channel_factory *client_channel_factory; - /** mutex protecting all variables below in this data structure */ - gpr_mu mu; + /** combiner protecting all variables below in this data structure */ + grpc_combiner *combiner; /** currently active load balancer */ char *lb_policy_name; grpc_lb_policy *lb_policy; @@ -218,8 +219,8 @@ static void set_channel_connectivity_state_locked(grpc_exec_ctx *exec_ctx, } static void on_lb_policy_state_changed_locked(grpc_exec_ctx *exec_ctx, - lb_policy_connectivity_watcher *w, - grpc_error *error) { + void *arg, grpc_error *error) { + lb_policy_connectivity_watcher *w = arg; grpc_connectivity_state publish_state = w->state; /* check if the notification is for a stale policy */ if (w->lb_policy != w->chand->lb_policy) return; @@ -235,15 +236,6 @@ static void on_lb_policy_state_changed_locked(grpc_exec_ctx *exec_ctx, if (w->state != GRPC_CHANNEL_SHUTDOWN) { watch_lb_policy(exec_ctx, w->chand, w->lb_policy, w->state); } -} - -static void on_lb_policy_state_changed(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - lb_policy_connectivity_watcher *w = arg; - - gpr_mu_lock(&w->chand->mu); - on_lb_policy_state_changed_locked(exec_ctx, w, error); - gpr_mu_unlock(&w->chand->mu); GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack, "watch_lb_policy"); gpr_free(w); @@ -256,16 +248,16 @@ static void watch_lb_policy(grpc_exec_ctx *exec_ctx, channel_data *chand, GRPC_CHANNEL_STACK_REF(chand->owning_stack, "watch_lb_policy"); w->chand = chand; - grpc_closure_init(&w->on_changed, on_lb_policy_state_changed, w, - grpc_schedule_on_exec_ctx); + grpc_closure_init(&w->on_changed, on_lb_policy_state_changed_locked, w, + grpc_combiner_scheduler(chand->combiner, false)); w->state = current_state; w->lb_policy = lb_policy; grpc_lb_policy_notify_on_state_change(exec_ctx, lb_policy, &w->state, &w->on_changed); } -static void on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { +static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, + void *arg, grpc_error *error) { channel_data *chand = arg; char *lb_policy_name = NULL; grpc_lb_policy *lb_policy = NULL; @@ -353,7 +345,6 @@ static void on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg, chand->interested_parties); } - gpr_mu_lock(&chand->mu); if (lb_policy_name != NULL) { gpr_free(chand->lb_policy_name); chand->lb_policy_name = lb_policy_name; @@ -391,7 +382,6 @@ static void on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg, GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver"); grpc_resolver_next(exec_ctx, chand->resolver, &chand->resolver_result, &chand->on_resolver_result_changed); - gpr_mu_unlock(&chand->mu); } else { if (chand->resolver != NULL) { grpc_resolver_shutdown(exec_ctx, chand->resolver); @@ -404,7 +394,6 @@ static void on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg, GRPC_ERROR_CREATE_REFERENCING("Got config after disconnection", refs, GPR_ARRAY_SIZE(refs)), "resolver_gone"); - gpr_mu_unlock(&chand->mu); } if (exit_idle) { @@ -426,20 +415,12 @@ static void on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg, GRPC_ERROR_UNREF(state_error); } -static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, - grpc_transport_op *op) { +static void cc_start_transport_op_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error_ignored) { + grpc_transport_op *op = arg; + grpc_channel_element *elem = op->transport_private.args[0]; channel_data *chand = elem->channel_data; - grpc_closure_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE); - - GPR_ASSERT(op->set_accept_stream == false); - if (op->bind_pollset != NULL) { - grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, - op->bind_pollset); - } - - gpr_mu_lock(&chand->mu); if (op->on_connectivity_state_change != NULL) { grpc_connectivity_state_notify_on_state_change( exec_ctx, &chand->state_tracker, op->connectivity_state, @@ -482,7 +463,28 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, } GRPC_ERROR_UNREF(op->disconnect_with_error); } - gpr_mu_unlock(&chand->mu); +} + +static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, + grpc_channel_element *elem, + grpc_transport_op *op) { + channel_data *chand = elem->channel_data; + + grpc_closure_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE); + + GPR_ASSERT(op->set_accept_stream == false); + if (op->bind_pollset != NULL) { + grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, + op->bind_pollset); + } + + op->transport_private.args[0] = elem; + grpc_closure_sched( + exec_ctx, + grpc_closure_init(&op->transport_private.closure, + cc_start_transport_op_locked, op, + grpc_combiner_scheduler(chand->combiner, false)), + GRPC_ERROR_NONE); } static void cc_get_channel_info(grpc_exec_ctx *exec_ctx, @@ -512,11 +514,11 @@ static grpc_error *cc_init_channel_elem(grpc_exec_ctx *exec_ctx, GPR_ASSERT(args->is_last); GPR_ASSERT(elem->filter == &grpc_client_channel_filter); // Initialize data members. - gpr_mu_init(&chand->mu); + chand->combiner = grpc_combiner_create(NULL); chand->owning_stack = args->channel_stack; grpc_closure_init(&chand->on_resolver_result_changed, - on_resolver_result_changed, chand, - grpc_schedule_on_exec_ctx); + on_resolver_result_changed_locked, chand, + grpc_combiner_scheduler(chand->combiner, false)); chand->interested_parties = grpc_pollset_set_create(); grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE, "client_channel"); @@ -572,7 +574,7 @@ static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx, } grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker); grpc_pollset_set_destroy(chand->interested_parties); - gpr_mu_destroy(&chand->mu); + grpc_combiner_destroy(exec_ctx, chand->combiner); } /************************************************************************* @@ -615,8 +617,6 @@ typedef struct client_channel_call_data { grpc_subchannel_call */ gpr_atm subchannel_call; - gpr_mu mu; - subchannel_creation_phase creation_phase; grpc_connected_subchannel *connected_subchannel; grpc_polling_entity *pollent; @@ -701,12 +701,11 @@ static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, call_data *calld) { GRPC_ERROR_NONE); } -static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { +static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { grpc_call_element *elem = arg; call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; - gpr_mu_lock(&calld->mu); GPR_ASSERT(calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL); grpc_polling_entity_del_from_pollset_set(exec_ctx, calld->pollent, @@ -742,7 +741,6 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, (gpr_atm)(uintptr_t)subchannel_call); retry_waiting_locked(exec_ctx, calld); } - gpr_mu_unlock(&calld->mu); GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel"); } @@ -768,37 +766,35 @@ typedef struct { /** Return true if subchannel is available immediately (in which case on_ready should not be called), or false otherwise (in which case on_ready should be called when the subchannel is available). */ -static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_metadata_batch *initial_metadata, - uint32_t initial_metadata_flags, - grpc_connected_subchannel **connected_subchannel, - grpc_closure *on_ready, grpc_error *error); - -static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { +static bool pick_subchannel_locked( + grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_metadata_batch *initial_metadata, uint32_t initial_metadata_flags, + grpc_connected_subchannel **connected_subchannel, grpc_closure *on_ready, + grpc_error *error); + +static void continue_picking_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { continue_picking_args *cpa = arg; if (cpa->connected_subchannel == NULL) { /* cancelled, do nothing */ } else if (error != GRPC_ERROR_NONE) { grpc_closure_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_REF(error)); } else { - call_data *calld = cpa->elem->call_data; - gpr_mu_lock(&calld->mu); - if (pick_subchannel(exec_ctx, cpa->elem, cpa->initial_metadata, - cpa->initial_metadata_flags, cpa->connected_subchannel, - cpa->on_ready, GRPC_ERROR_NONE)) { + if (pick_subchannel_locked(exec_ctx, cpa->elem, cpa->initial_metadata, + cpa->initial_metadata_flags, + cpa->connected_subchannel, cpa->on_ready, + GRPC_ERROR_NONE)) { grpc_closure_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_NONE); } - gpr_mu_unlock(&calld->mu); } gpr_free(cpa); } -static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_metadata_batch *initial_metadata, - uint32_t initial_metadata_flags, - grpc_connected_subchannel **connected_subchannel, - grpc_closure *on_ready, grpc_error *error) { +static bool pick_subchannel_locked( + grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_metadata_batch *initial_metadata, uint32_t initial_metadata_flags, + grpc_connected_subchannel **connected_subchannel, grpc_closure *on_ready, + grpc_error *error) { GPR_TIMER_BEGIN("pick_subchannel", 0); channel_data *chand = elem->channel_data; @@ -808,7 +804,6 @@ static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, GPR_ASSERT(connected_subchannel); - gpr_mu_lock(&chand->mu); if (initial_metadata == NULL) { if (chand->lb_policy != NULL) { grpc_lb_policy_cancel_pick(exec_ctx, chand->lb_policy, @@ -824,7 +819,6 @@ static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, GRPC_ERROR_CREATE_REFERENCING("Pick cancelled", &error, 1)); } } - gpr_mu_unlock(&chand->mu); GPR_TIMER_END("pick_subchannel", 0); GRPC_ERROR_UNREF(error); return true; @@ -833,7 +827,6 @@ static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, if (chand->lb_policy != NULL) { grpc_lb_policy *lb_policy = chand->lb_policy; GRPC_LB_POLICY_REF(lb_policy, "pick_subchannel"); - gpr_mu_unlock(&chand->mu); // If the application explicitly set wait_for_ready, use that. // Otherwise, if the service config specified a value for this // method, use that. @@ -872,59 +865,37 @@ static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, cpa->connected_subchannel = connected_subchannel; cpa->on_ready = on_ready; cpa->elem = elem; - grpc_closure_init(&cpa->closure, continue_picking, cpa, - grpc_schedule_on_exec_ctx); + grpc_closure_init(&cpa->closure, continue_picking_locked, cpa, + grpc_combiner_scheduler(chand->combiner, true)); grpc_closure_list_append(&chand->waiting_for_config_closures, &cpa->closure, GRPC_ERROR_NONE); } else { grpc_closure_sched(exec_ctx, on_ready, GRPC_ERROR_CREATE("Disconnected")); } - gpr_mu_unlock(&chand->mu); GPR_TIMER_END("pick_subchannel", 0); return false; } -// The logic here is fairly complicated, due to (a) the fact that we -// need to handle the case where we receive the send op before the -// initial metadata op, and (b) the need for efficiency, especially in -// the streaming case. -// TODO(ctiller): Explain this more thoroughly. -static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op) { +static void cc_start_transport_stream_op_locked(grpc_exec_ctx *exec_ctx, + void *arg, + grpc_error *error_ignored) { + grpc_transport_stream_op *op = arg; + grpc_call_element *elem = op->transport_private.args[0]; call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; - GRPC_CALL_LOG_OP(GPR_INFO, elem, op); - grpc_deadline_state_client_start_transport_stream_op(exec_ctx, elem, op); - /* try to (atomically) get the call */ - grpc_subchannel_call *call = GET_CALL(calld); - GPR_TIMER_BEGIN("cc_start_transport_stream_op", 0); - if (call == CANCELLED_CALL) { - grpc_transport_stream_op_finish_with_failure( - exec_ctx, op, GRPC_ERROR_REF(calld->cancel_error)); - GPR_TIMER_END("cc_start_transport_stream_op", 0); - return; - } - if (call != NULL) { - grpc_subchannel_call_process_op(exec_ctx, call, op); - GPR_TIMER_END("cc_start_transport_stream_op", 0); - return; - } - /* we failed; lock and figure out what to do */ - gpr_mu_lock(&calld->mu); + grpc_subchannel_call *call; + retry: /* need to recheck that another thread hasn't set the call */ call = GET_CALL(calld); if (call == CANCELLED_CALL) { - gpr_mu_unlock(&calld->mu); grpc_transport_stream_op_finish_with_failure( exec_ctx, op, GRPC_ERROR_REF(calld->cancel_error)); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; } if (call != NULL) { - gpr_mu_unlock(&calld->mu); grpc_subchannel_call_process_op(exec_ctx, call, op); GPR_TIMER_END("cc_start_transport_stream_op", 0); return; @@ -946,11 +917,11 @@ retry: fail_locked(exec_ctx, calld, GRPC_ERROR_REF(op->cancel_error)); break; case GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL: - pick_subchannel(exec_ctx, elem, NULL, 0, &calld->connected_subchannel, - NULL, GRPC_ERROR_REF(op->cancel_error)); + pick_subchannel_locked(exec_ctx, elem, NULL, 0, + &calld->connected_subchannel, NULL, + GRPC_ERROR_REF(op->cancel_error)); break; } - gpr_mu_unlock(&calld->mu); grpc_transport_stream_op_finish_with_failure( exec_ctx, op, GRPC_ERROR_REF(op->cancel_error)); GPR_TIMER_END("cc_start_transport_stream_op", 0); @@ -962,16 +933,16 @@ retry: calld->connected_subchannel == NULL && op->send_initial_metadata != NULL) { calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL; - grpc_closure_init(&calld->next_step, subchannel_ready, elem, - grpc_schedule_on_exec_ctx); + grpc_closure_init(&calld->next_step, subchannel_ready_locked, elem, + grpc_combiner_scheduler(chand->combiner, true)); GRPC_CALL_STACK_REF(calld->owning_call, "pick_subchannel"); /* If a subchannel is not available immediately, the polling entity from call_data should be provided to channel_data's interested_parties, so that IO of the lb_policy and resolver could be done under it. */ - if (pick_subchannel(exec_ctx, elem, op->send_initial_metadata, - op->send_initial_metadata_flags, - &calld->connected_subchannel, &calld->next_step, - GRPC_ERROR_NONE)) { + if (pick_subchannel_locked(exec_ctx, elem, op->send_initial_metadata, + op->send_initial_metadata_flags, + &calld->connected_subchannel, &calld->next_step, + GRPC_ERROR_NONE)) { calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel"); } else { @@ -998,27 +969,60 @@ retry: } /* nothing to be done but wait */ add_waiting_locked(calld, op); - gpr_mu_unlock(&calld->mu); GPR_TIMER_END("cc_start_transport_stream_op", 0); } +// The logic here is fairly complicated, due to (a) the fact that we +// need to handle the case where we receive the send op before the +// initial metadata op, and (b) the need for efficiency, especially in +// the streaming case. +// TODO(ctiller): Explain this more thoroughly. +static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem, + grpc_transport_stream_op *op) { + call_data *calld = elem->call_data; + channel_data *chand = elem->channel_data; + GRPC_CALL_LOG_OP(GPR_INFO, elem, op); + grpc_deadline_state_client_start_transport_stream_op(exec_ctx, elem, op); + /* try to (atomically) get the call */ + grpc_subchannel_call *call = GET_CALL(calld); + GPR_TIMER_BEGIN("cc_start_transport_stream_op", 0); + if (call == CANCELLED_CALL) { + grpc_transport_stream_op_finish_with_failure( + exec_ctx, op, GRPC_ERROR_REF(calld->cancel_error)); + GPR_TIMER_END("cc_start_transport_stream_op", 0); + return; + } + if (call != NULL) { + grpc_subchannel_call_process_op(exec_ctx, call, op); + GPR_TIMER_END("cc_start_transport_stream_op", 0); + return; + } + /* we failed; lock and figure out what to do */ + op->transport_private.args[0] = elem; + grpc_closure_sched( + exec_ctx, + grpc_closure_init(&op->transport_private.closure, + cc_start_transport_stream_op_locked, op, + grpc_combiner_scheduler(chand->combiner, false)), + GRPC_ERROR_NONE); +} + // Gets data from the service config. Invoked when the resolver returns // its initial result. -static void read_service_config(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { +static void read_service_config_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { grpc_call_element *elem = arg; channel_data *chand = elem->channel_data; call_data *calld = elem->call_data; // If this is an error, there's no point in looking at the service config. if (error == GRPC_ERROR_NONE) { // Get the method config table from channel data. - gpr_mu_lock(&chand->mu); grpc_slice_hash_table *method_params_table = NULL; if (chand->method_params_table != NULL) { method_params_table = grpc_slice_hash_table_ref(chand->method_params_table); } - gpr_mu_unlock(&chand->mu); // If the method config table was present, use it. if (method_params_table != NULL) { const method_parameters *method_params = grpc_method_config_table_get( @@ -1028,7 +1032,6 @@ static void read_service_config(grpc_exec_ctx *exec_ctx, void *arg, gpr_time_cmp(method_params->timeout, gpr_time_0(GPR_TIMESPAN)) != 0; if (have_method_timeout || method_params->wait_for_ready != WAIT_FOR_READY_UNSET) { - gpr_mu_lock(&calld->mu); if (have_method_timeout) { const gpr_timespec per_method_deadline = gpr_time_add(calld->call_start_time, method_params->timeout); @@ -1042,7 +1045,6 @@ static void read_service_config(grpc_exec_ctx *exec_ctx, void *arg, calld->wait_for_ready_from_service_config = method_params->wait_for_ready; } - gpr_mu_unlock(&calld->mu); } } grpc_slice_hash_table_unref(exec_ctx, method_params_table); @@ -1051,43 +1053,25 @@ static void read_service_config(grpc_exec_ctx *exec_ctx, void *arg, GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "read_service_config"); } -/* Constructor for call_data */ -static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_call_element_args *args) { +static void initial_read_service_config_locked(grpc_exec_ctx *exec_ctx, + void *arg, + grpc_error *error_ignored) { + grpc_call_element *elem = arg; channel_data *chand = elem->channel_data; call_data *calld = elem->call_data; - // Initialize data members. - grpc_deadline_state_init(exec_ctx, elem, args->call_stack); - calld->path = grpc_slice_ref_internal(args->path); - calld->call_start_time = args->start_time; - calld->deadline = gpr_convert_clock_type(args->deadline, GPR_CLOCK_MONOTONIC); - calld->wait_for_ready_from_service_config = WAIT_FOR_READY_UNSET; - calld->cancel_error = GRPC_ERROR_NONE; - gpr_atm_rel_store(&calld->subchannel_call, 0); - gpr_mu_init(&calld->mu); - calld->connected_subchannel = NULL; - calld->waiting_ops = NULL; - calld->waiting_ops_count = 0; - calld->waiting_ops_capacity = 0; - calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; - calld->owning_call = args->call_stack; - calld->pollent = NULL; // If the resolver has already returned results, then we can access // the service config parameters immediately. Otherwise, we need to // defer that work until the resolver returns an initial result. // TODO(roth): This code is almost but not quite identical to the code // in read_service_config() above. It would be nice to find a way to // combine them, to avoid having to maintain it twice. - gpr_mu_lock(&chand->mu); if (chand->lb_policy != NULL) { // We already have a resolver result, so check for service config. if (chand->method_params_table != NULL) { grpc_slice_hash_table *method_params_table = grpc_slice_hash_table_ref(chand->method_params_table); - gpr_mu_unlock(&chand->mu); method_parameters *method_params = grpc_method_config_table_get( - exec_ctx, method_params_table, args->path); + exec_ctx, method_params_table, calld->path); if (method_params != NULL) { if (gpr_time_cmp(method_params->timeout, gpr_time_0(GPR_CLOCK_MONOTONIC)) != 0) { @@ -1101,24 +1085,50 @@ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, } } grpc_slice_hash_table_unref(exec_ctx, method_params_table); - } else { - gpr_mu_unlock(&chand->mu); } } else { // We don't yet have a resolver result, so register a callback to // get the service config data once the resolver returns. // Take a reference to the call stack to be owned by the callback. GRPC_CALL_STACK_REF(calld->owning_call, "read_service_config"); - grpc_closure_init(&calld->read_service_config, read_service_config, elem, - grpc_schedule_on_exec_ctx); + grpc_closure_init(&calld->read_service_config, read_service_config_locked, + elem, grpc_combiner_scheduler(chand->combiner, false)); grpc_closure_list_append(&chand->waiting_for_config_closures, &calld->read_service_config, GRPC_ERROR_NONE); - gpr_mu_unlock(&chand->mu); } // Start the deadline timer with the current deadline value. If we // do not yet have service config data, then the timer may be reset // later. grpc_deadline_state_start(exec_ctx, elem, calld->deadline); +} + +/* Constructor for call_data */ +static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem, + grpc_call_element_args *args) { + channel_data *chand = elem->channel_data; + call_data *calld = elem->call_data; + // Initialize data members. + grpc_deadline_state_init(exec_ctx, elem, args->call_stack); + calld->path = grpc_slice_ref_internal(args->path); + calld->call_start_time = args->start_time; + calld->deadline = gpr_convert_clock_type(args->deadline, GPR_CLOCK_MONOTONIC); + calld->wait_for_ready_from_service_config = WAIT_FOR_READY_UNSET; + calld->cancel_error = GRPC_ERROR_NONE; + gpr_atm_rel_store(&calld->subchannel_call, 0); + calld->connected_subchannel = NULL; + calld->waiting_ops = NULL; + calld->waiting_ops_count = 0; + calld->waiting_ops_capacity = 0; + calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; + calld->owning_call = args->call_stack; + calld->pollent = NULL; + grpc_closure_sched( + exec_ctx, + grpc_closure_init(&calld->read_service_config, + initial_read_service_config_locked, elem, + grpc_combiner_scheduler(chand->combiner, false)), + GRPC_ERROR_NONE); return GRPC_ERROR_NONE; } @@ -1136,7 +1146,6 @@ static void cc_destroy_call_elem(grpc_exec_ctx *exec_ctx, GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, call, "client_channel_destroy_call"); } GPR_ASSERT(calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING); - gpr_mu_destroy(&calld->mu); GPR_ASSERT(calld->waiting_ops_count == 0); if (calld->connected_subchannel != NULL) { GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, calld->connected_subchannel, From 613dafa60ce3a22a5d7f1351b8054a3090b9deb1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 12:00:43 -0800 Subject: [PATCH 084/167] Convert connectivity_state, channel info into a combiner-compatible form --- src/core/ext/client_channel/client_channel.c | 100 +++++++++++------- src/core/ext/client_channel/subchannel.c | 2 +- src/core/ext/lb_policy/grpclb/grpclb.c | 9 +- .../ext/lb_policy/pick_first/pick_first.c | 2 +- .../ext/lb_policy/round_robin/round_robin.c | 2 +- src/core/lib/transport/connectivity_state.c | 42 +++++--- src/core/lib/transport/connectivity_state.h | 20 +++- test/core/transport/connectivity_state_test.c | 3 +- 8 files changed, 117 insertions(+), 63 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 2595acd8c42..58504de87d2 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -164,10 +164,7 @@ typedef struct client_channel_channel_data { /** combiner protecting all variables below in this data structure */ grpc_combiner *combiner; /** currently active load balancer */ - char *lb_policy_name; grpc_lb_policy *lb_policy; - /** service config in JSON form */ - char *service_config_json; /** maps method names to method_parameters structs */ grpc_slice_hash_table *method_params_table; /** incoming resolver result - set by resolver.next() */ @@ -184,6 +181,13 @@ typedef struct client_channel_channel_data { grpc_channel_stack *owning_stack; /** interested parties (owned) */ grpc_pollset_set *interested_parties; + + /* the following properties are guarded by a mutex since API's require them + to be instantaniously available */ + gpr_mu info_mu; + char *info_lb_policy_name; + /** service config in JSON form */ + char *info_service_config_json; } channel_data; /** We create one watcher for each new lb_policy that is returned from a @@ -345,16 +349,18 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, chand->interested_parties); } + gpr_mu_lock(&chand->info_mu); if (lb_policy_name != NULL) { - gpr_free(chand->lb_policy_name); - chand->lb_policy_name = lb_policy_name; + gpr_free(chand->info_lb_policy_name); + chand->info_lb_policy_name = lb_policy_name; } old_lb_policy = chand->lb_policy; chand->lb_policy = lb_policy; if (service_config_json != NULL) { - gpr_free(chand->service_config_json); - chand->service_config_json = service_config_json; + gpr_free(chand->info_service_config_json); + chand->info_service_config_json = service_config_json; } + gpr_mu_unlock(&chand->info_mu); if (chand->method_params_table != NULL) { grpc_slice_hash_table_unref(exec_ctx, chand->method_params_table); } @@ -491,18 +497,19 @@ static void cc_get_channel_info(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, const grpc_channel_info *info) { channel_data *chand = elem->channel_data; - gpr_mu_lock(&chand->mu); + gpr_mu_lock(&chand->info_mu); if (info->lb_policy_name != NULL) { - *info->lb_policy_name = chand->lb_policy_name == NULL + *info->lb_policy_name = chand->info_lb_policy_name == NULL ? NULL - : gpr_strdup(chand->lb_policy_name); + : gpr_strdup(chand->info_lb_policy_name); } if (info->service_config_json != NULL) { - *info->service_config_json = chand->service_config_json == NULL - ? NULL - : gpr_strdup(chand->service_config_json); + *info->service_config_json = + chand->info_service_config_json == NULL + ? NULL + : gpr_strdup(chand->info_service_config_json); } - gpr_mu_unlock(&chand->mu); + gpr_mu_unlock(&chand->info_mu); } /* Constructor for channel_data */ @@ -567,8 +574,8 @@ static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx, chand->interested_parties); GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel"); } - gpr_free(chand->lb_policy_name); - gpr_free(chand->service_config_json); + gpr_free(chand->info_lb_policy_name); + gpr_free(chand->info_service_config_json); if (chand->method_params_table != NULL) { grpc_slice_hash_table_unref(exec_ctx, chand->method_params_table); } @@ -1181,26 +1188,34 @@ const grpc_channel_filter grpc_client_channel_filter = { "client-channel", }; +static void try_to_connect_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error_ignored) { + channel_data *chand = arg; + if (chand->lb_policy != NULL) { + grpc_lb_policy_exit_idle(exec_ctx, chand->lb_policy); + } else { + chand->exit_idle_when_lb_policy_arrives = true; + if (!chand->started_resolving && chand->resolver != NULL) { + GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver"); + chand->started_resolving = true; + grpc_resolver_next(exec_ctx, chand->resolver, &chand->resolver_result, + &chand->on_resolver_result_changed); + } + } +} + grpc_connectivity_state grpc_client_channel_check_connectivity_state( grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, int try_to_connect) { channel_data *chand = elem->channel_data; grpc_connectivity_state out; - gpr_mu_lock(&chand->mu); - out = grpc_connectivity_state_check(&chand->state_tracker, NULL); + out = grpc_connectivity_state_check(&chand->state_tracker); if (out == GRPC_CHANNEL_IDLE && try_to_connect) { - if (chand->lb_policy != NULL) { - grpc_lb_policy_exit_idle(exec_ctx, chand->lb_policy); - } else { - chand->exit_idle_when_lb_policy_arrives = true; - if (!chand->started_resolving && chand->resolver != NULL) { - GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver"); - chand->started_resolving = true; - grpc_resolver_next(exec_ctx, chand->resolver, &chand->resolver_result, - &chand->on_resolver_result_changed); - } - } + grpc_closure_sched( + exec_ctx, + grpc_closure_create(try_to_connect_locked, chand, + grpc_combiner_scheduler(chand->combiner, false)), + GRPC_ERROR_NONE); } - gpr_mu_unlock(&chand->mu); return out; } @@ -1208,6 +1223,7 @@ typedef struct { channel_data *chand; grpc_pollset *pollset; grpc_closure *on_complete; + grpc_connectivity_state *state; grpc_closure my_closure; } external_connectivity_watcher; @@ -1220,7 +1236,17 @@ static void on_external_watch_complete(grpc_exec_ctx *exec_ctx, void *arg, GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack, "external_connectivity_watcher"); gpr_free(w); - follow_up->cb(exec_ctx, follow_up->cb_arg, error); + grpc_closure_run(exec_ctx, follow_up, GRPC_ERROR_REF(error)); +} + +static void cc_watch_connectivity_state_locked(grpc_exec_ctx *exec_ctx, + void *arg, + grpc_error *error_ignored) { + external_connectivity_watcher *w = arg; + grpc_closure_init(&w->my_closure, on_external_watch_complete, w, + grpc_schedule_on_exec_ctx); + grpc_connectivity_state_notify_on_state_change( + exec_ctx, &w->chand->state_tracker, w->state, &w->my_closure); } void grpc_client_channel_watch_connectivity_state( @@ -1231,13 +1257,13 @@ void grpc_client_channel_watch_connectivity_state( w->chand = chand; w->pollset = pollset; w->on_complete = on_complete; + w->state = state; grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, pollset); - grpc_closure_init(&w->my_closure, on_external_watch_complete, w, - grpc_schedule_on_exec_ctx); GRPC_CHANNEL_STACK_REF(w->chand->owning_stack, "external_connectivity_watcher"); - gpr_mu_lock(&chand->mu); - grpc_connectivity_state_notify_on_state_change( - exec_ctx, &chand->state_tracker, state, &w->my_closure); - gpr_mu_unlock(&chand->mu); + grpc_closure_sched( + exec_ctx, + grpc_closure_init(&w->my_closure, cc_watch_connectivity_state_locked, w, + grpc_combiner_scheduler(chand->combiner, true)), + GRPC_ERROR_NONE); } diff --git a/src/core/ext/client_channel/subchannel.c b/src/core/ext/client_channel/subchannel.c index aa036e883ba..c37134fd5e7 100644 --- a/src/core/ext/client_channel/subchannel.c +++ b/src/core/ext/client_channel/subchannel.c @@ -419,7 +419,7 @@ grpc_connectivity_state grpc_subchannel_check_connectivity(grpc_subchannel *c, grpc_error **error) { grpc_connectivity_state state; gpr_mu_lock(&c->mu); - state = grpc_connectivity_state_check(&c->state_tracker, error); + state = grpc_connectivity_state_get(&c->state_tracker, error); gpr_mu_unlock(&c->mu); return state; } diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index ab62e5ed6a1..8a2af483283 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -492,9 +492,8 @@ static grpc_lb_addresses *process_serverlist_locked( static bool update_lb_connectivity_status_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, grpc_connectivity_state new_rr_state, grpc_error *new_rr_state_error) { - grpc_error *curr_state_error; - const grpc_connectivity_state curr_glb_state = grpc_connectivity_state_check( - &glb_policy->state_tracker, &curr_state_error); + const grpc_connectivity_state curr_glb_state = + grpc_connectivity_state_check(&glb_policy->state_tracker); /* The new connectivity status is a function of the previous one and the new * input coming from the status of the RR policy. @@ -1098,8 +1097,8 @@ static grpc_connectivity_state glb_check_connectivity( glb_lb_policy *glb_policy = (glb_lb_policy *)pol; grpc_connectivity_state st; gpr_mu_lock(&glb_policy->mu); - st = grpc_connectivity_state_check(&glb_policy->state_tracker, - connectivity_error); + st = grpc_connectivity_state_get(&glb_policy->state_tracker, + connectivity_error); gpr_mu_unlock(&glb_policy->mu); return st; } diff --git a/src/core/ext/lb_policy/pick_first/pick_first.c b/src/core/ext/lb_policy/pick_first/pick_first.c index 9f2aa461bed..1b965183f6b 100644 --- a/src/core/ext/lb_policy/pick_first/pick_first.c +++ b/src/core/ext/lb_policy/pick_first/pick_first.c @@ -398,7 +398,7 @@ static grpc_connectivity_state pf_check_connectivity(grpc_exec_ctx *exec_ctx, pick_first_lb_policy *p = (pick_first_lb_policy *)pol; grpc_connectivity_state st; gpr_mu_lock(&p->mu); - st = grpc_connectivity_state_check(&p->state_tracker, error); + st = grpc_connectivity_state_get(&p->state_tracker, error); gpr_mu_unlock(&p->mu); return st; } diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index 3e060d189ab..63e3d033adf 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -655,7 +655,7 @@ static grpc_connectivity_state rr_check_connectivity(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p = (round_robin_lb_policy *)pol; grpc_connectivity_state st; gpr_mu_lock(&p->mu); - st = grpc_connectivity_state_check(&p->state_tracker, error); + st = grpc_connectivity_state_get(&p->state_tracker, error); gpr_mu_unlock(&p->mu); return st; } diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c index 8fc5bf3e9a6..e884567240d 100644 --- a/src/core/lib/transport/connectivity_state.c +++ b/src/core/lib/transport/connectivity_state.c @@ -62,7 +62,7 @@ const char *grpc_connectivity_state_name(grpc_connectivity_state state) { void grpc_connectivity_state_init(grpc_connectivity_state_tracker *tracker, grpc_connectivity_state init_state, const char *name) { - tracker->current_state = init_state; + gpr_atm_no_barrier_store(&tracker->current_state_atm, init_state); tracker->current_error = GRPC_ERROR_NONE; tracker->watchers = NULL; tracker->name = gpr_strdup(name); @@ -89,15 +89,29 @@ void grpc_connectivity_state_destroy(grpc_exec_ctx *exec_ctx, } grpc_connectivity_state grpc_connectivity_state_check( + grpc_connectivity_state_tracker *tracker) { + grpc_connectivity_state cur = + (grpc_connectivity_state)gpr_atm_no_barrier_load( + &tracker->current_state_atm); + if (grpc_connectivity_state_trace) { + gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name, + grpc_connectivity_state_name(cur)); + } + return cur; +} + +grpc_connectivity_state grpc_connectivity_state_get( grpc_connectivity_state_tracker *tracker, grpc_error **error) { + grpc_connectivity_state cur =(grpc_connectivity_state) + gpr_atm_no_barrier_load(&tracker->current_state_atm); if (grpc_connectivity_state_trace) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name, - grpc_connectivity_state_name(tracker->current_state)); + grpc_connectivity_state_name(cur)); } if (error != NULL) { *error = GRPC_ERROR_REF(tracker->current_error); } - return tracker->current_state; + return cur; } bool grpc_connectivity_state_has_watchers( @@ -108,6 +122,8 @@ bool grpc_connectivity_state_has_watchers( bool grpc_connectivity_state_notify_on_state_change( grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker, grpc_connectivity_state *current, grpc_closure *notify) { + grpc_connectivity_state cur =(grpc_connectivity_state) + gpr_atm_no_barrier_load(&tracker->current_state_atm); if (grpc_connectivity_state_trace) { if (current == NULL) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: unsubscribe notify=%p", tracker, @@ -115,7 +131,7 @@ bool grpc_connectivity_state_notify_on_state_change( } else { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: from %s [cur=%s] notify=%p", tracker, tracker->name, grpc_connectivity_state_name(*current), - grpc_connectivity_state_name(tracker->current_state), notify); + grpc_connectivity_state_name(cur), notify); } } if (current == NULL) { @@ -138,8 +154,8 @@ bool grpc_connectivity_state_notify_on_state_change( } return false; } else { - if (tracker->current_state != *current) { - *current = tracker->current_state; + if (cur != *current) { + *current = cur; grpc_closure_sched(exec_ctx, notify, GRPC_ERROR_REF(tracker->current_error)); } else { @@ -149,7 +165,7 @@ bool grpc_connectivity_state_notify_on_state_change( w->next = tracker->watchers; tracker->watchers = w; } - return tracker->current_state == GRPC_CHANNEL_IDLE; + return cur == GRPC_CHANNEL_IDLE; } } @@ -157,11 +173,13 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker, grpc_connectivity_state state, grpc_error *error, const char *reason) { + grpc_connectivity_state cur =(grpc_connectivity_state) + gpr_atm_no_barrier_load(&tracker->current_state_atm); grpc_connectivity_state_watcher *w; if (grpc_connectivity_state_trace) { const char *error_string = grpc_error_string(error); gpr_log(GPR_DEBUG, "SET: %p %s: %s --> %s [%s] error=%p %s", tracker, - tracker->name, grpc_connectivity_state_name(tracker->current_state), + tracker->name, grpc_connectivity_state_name(cur), grpc_connectivity_state_name(state), reason, error, error_string); } switch (state) { @@ -178,13 +196,13 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, } GRPC_ERROR_UNREF(tracker->current_error); tracker->current_error = error; - if (tracker->current_state == state) { + if (cur == state) { return; } - GPR_ASSERT(tracker->current_state != GRPC_CHANNEL_SHUTDOWN); - tracker->current_state = state; + GPR_ASSERT(cur != GRPC_CHANNEL_SHUTDOWN); + gpr_atm_no_barrier_store(&tracker->current_state_atm, state); while ((w = tracker->watchers) != NULL) { - *w->current = tracker->current_state; + *w->current = state; tracker->watchers = w->next; if (grpc_connectivity_state_trace) { gpr_log(GPR_DEBUG, "NOTIFY: %p %s: %p", tracker, tracker->name, diff --git a/src/core/lib/transport/connectivity_state.h b/src/core/lib/transport/connectivity_state.h index 769c675b797..c9604c34dda 100644 --- a/src/core/lib/transport/connectivity_state.h +++ b/src/core/lib/transport/connectivity_state.h @@ -47,8 +47,8 @@ typedef struct grpc_connectivity_state_watcher { } grpc_connectivity_state_watcher; typedef struct { - /** current connectivity state */ - grpc_connectivity_state current_state; + /** current grpc_connectivity_state */ + gpr_atm current_state_atm; /** error associated with state */ grpc_error *current_error; /** all our watchers */ @@ -59,6 +59,7 @@ typedef struct { extern int grpc_connectivity_state_trace; +/** enum --> string conversion */ const char *grpc_connectivity_state_name(grpc_connectivity_state state); void grpc_connectivity_state_init(grpc_connectivity_state_tracker *tracker, @@ -68,22 +69,31 @@ void grpc_connectivity_state_destroy(grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker); /** Set connectivity state; not thread safe; access must be serialized with an - * external lock */ + * external lock */ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker, grpc_connectivity_state state, grpc_error *associated_error, const char *reason); +/** Return true if this connectivity state has watchers. + Access must be serialized with an external lock. */ bool grpc_connectivity_state_has_watchers( grpc_connectivity_state_tracker *tracker); +/** Return the last seen connectivity state. No need to synchronize access. */ grpc_connectivity_state grpc_connectivity_state_check( - grpc_connectivity_state_tracker *tracker, grpc_error **current_error); + grpc_connectivity_state_tracker *tracker); + +/** Return the last seen connectivity state, and the associated error. + Access must be serialized with an external lock. */ +grpc_connectivity_state grpc_connectivity_state_get( + grpc_connectivity_state_tracker *tracker, grpc_error **error); /** Return 1 if the channel should start connecting, 0 otherwise. If current==NULL cancel notify if it is already queued (success==0 in that - case) */ + case). + Access must be serialized with an external lock. */ bool grpc_connectivity_state_notify_on_state_change( grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker, grpc_connectivity_state *current, grpc_closure *notify); diff --git a/test/core/transport/connectivity_state_test.c b/test/core/transport/connectivity_state_test.c index 3520ef0a80d..8314a5f6190 100644 --- a/test/core/transport/connectivity_state_test.c +++ b/test/core/transport/connectivity_state_test.c @@ -77,8 +77,9 @@ static void test_check(void) { grpc_error *error; gpr_log(GPR_DEBUG, "test_check"); grpc_connectivity_state_init(&tracker, GRPC_CHANNEL_IDLE, "xxx"); - GPR_ASSERT(grpc_connectivity_state_check(&tracker, &error) == + GPR_ASSERT(grpc_connectivity_state_get(&tracker, &error) == GRPC_CHANNEL_IDLE); + GPR_ASSERT(grpc_connectivity_state_check(&tracker) == GRPC_CHANNEL_IDLE); GPR_ASSERT(error == GRPC_ERROR_NONE); grpc_connectivity_state_destroy(&exec_ctx, &tracker); grpc_exec_ctx_finish(&exec_ctx); From d85477515230c5161659175cbc60b684109aedbf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 12:02:39 -0800 Subject: [PATCH 085/167] Init mutex --- src/core/ext/client_channel/client_channel.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 58504de87d2..a7dd967a51e 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -522,6 +522,7 @@ static grpc_error *cc_init_channel_elem(grpc_exec_ctx *exec_ctx, GPR_ASSERT(elem->filter == &grpc_client_channel_filter); // Initialize data members. chand->combiner = grpc_combiner_create(NULL); + gpr_mu_init(&chand->info_mu); chand->owning_stack = args->channel_stack; grpc_closure_init(&chand->on_resolver_result_changed, on_resolver_result_changed_locked, chand, @@ -582,6 +583,7 @@ static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx, grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker); grpc_pollset_set_destroy(chand->interested_parties); grpc_combiner_destroy(exec_ctx, chand->combiner); + gpr_mu_destroy(&chand->info_mu); } /************************************************************************* From d2e5cfc5bbe844ece808a843314458b798368748 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 13:02:20 -0800 Subject: [PATCH 086/167] Start fixing refcounting --- src/core/ext/client_channel/client_channel.c | 48 ++++++++------------ 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index a7dd967a51e..282913431f3 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -469,6 +469,9 @@ static void cc_start_transport_op_locked(grpc_exec_ctx *exec_ctx, void *arg, } GRPC_ERROR_UNREF(op->disconnect_with_error); } + GRPC_CHANNEL_STACK_UNREF(exec_ctx, chand->owning_stack, "start_transport_op"); + + grpc_closure_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE); } static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, @@ -476,8 +479,6 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport_op *op) { channel_data *chand = elem->channel_data; - grpc_closure_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE); - GPR_ASSERT(op->set_accept_stream == false); if (op->bind_pollset != NULL) { grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, @@ -485,6 +486,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, } op->transport_private.args[0] = elem; + GRPC_CHANNEL_STACK_REF(chand->owning_stack, "start_transport_op"); grpc_closure_sched( exec_ctx, grpc_closure_init(&op->transport_private.closure, @@ -670,44 +672,24 @@ static void fail_locked(grpc_exec_ctx *exec_ctx, call_data *calld, GRPC_ERROR_UNREF(error); } -typedef struct { - grpc_transport_stream_op **ops; - size_t nops; - grpc_subchannel_call *call; -} retry_ops_args; - -static void retry_ops(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) { - retry_ops_args *a = args; - size_t i; - for (i = 0; i < a->nops; i++) { - grpc_subchannel_call_process_op(exec_ctx, a->call, a->ops[i]); - } - GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, a->call, "retry_ops"); - gpr_free(a->ops); - gpr_free(a); -} - static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, call_data *calld) { if (calld->waiting_ops_count == 0) { return; } - retry_ops_args *a = gpr_malloc(sizeof(*a)); - a->ops = calld->waiting_ops; - a->nops = calld->waiting_ops_count; - a->call = GET_CALL(calld); - if (a->call == CANCELLED_CALL) { - gpr_free(a); + grpc_subchannel_call *call = GET_CALL(calld); + grpc_transport_stream_op **ops = calld->waiting_ops; + size_t nops = calld->waiting_ops_count; + if (call == CANCELLED_CALL) { fail_locked(exec_ctx, calld, GRPC_ERROR_CANCELLED); return; } calld->waiting_ops = NULL; calld->waiting_ops_count = 0; calld->waiting_ops_capacity = 0; - GRPC_SUBCHANNEL_CALL_REF(a->call, "retry_ops"); - grpc_closure_sched( - exec_ctx, grpc_closure_create(retry_ops, a, grpc_schedule_on_exec_ctx), - GRPC_ERROR_NONE); + for (size_t i = 0; i < nops; i++) { + grpc_subchannel_call_process_op(exec_ctx, call, ops[i]); + } } static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, @@ -978,6 +960,8 @@ retry: } /* nothing to be done but wait */ add_waiting_locked(calld, op); + GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, + "start_transport_stream_op"); GPR_TIMER_END("cc_start_transport_stream_op", 0); } @@ -1008,6 +992,7 @@ static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, return; } /* we failed; lock and figure out what to do */ + GRPC_CALL_STACK_REF(calld->owning_call, "start_transport_stream_op"); op->transport_private.args[0] = elem; grpc_closure_sched( exec_ctx, @@ -1109,6 +1094,8 @@ static void initial_read_service_config_locked(grpc_exec_ctx *exec_ctx, // do not yet have service config data, then the timer may be reset // later. grpc_deadline_state_start(exec_ctx, elem, calld->deadline); + GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, + "initial_read_service_config"); } /* Constructor for call_data */ @@ -1132,6 +1119,7 @@ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; calld->owning_call = args->call_stack; calld->pollent = NULL; + GRPC_CALL_STACK_REF(calld->owning_call, "initial_read_service_config"); grpc_closure_sched( exec_ctx, grpc_closure_init(&calld->read_service_config, @@ -1204,6 +1192,7 @@ static void try_to_connect_locked(grpc_exec_ctx *exec_ctx, void *arg, &chand->on_resolver_result_changed); } } + GRPC_CHANNEL_STACK_UNREF(exec_ctx, chand->owning_stack, "try_to_connect"); } grpc_connectivity_state grpc_client_channel_check_connectivity_state( @@ -1212,6 +1201,7 @@ grpc_connectivity_state grpc_client_channel_check_connectivity_state( grpc_connectivity_state out; out = grpc_connectivity_state_check(&chand->state_tracker); if (out == GRPC_CHANNEL_IDLE && try_to_connect) { + GRPC_CHANNEL_STACK_REF(chand->owning_stack, "try_to_connect"); grpc_closure_sched( exec_ctx, grpc_closure_create(try_to_connect_locked, chand, From 9efea88d33132964b3eadcbdc43bd2bba38517ad Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 13:06:52 -0800 Subject: [PATCH 087/167] Fix waiting_ops leak --- src/core/ext/client_channel/client_channel.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 282913431f3..4bde9ec8054 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -690,6 +690,7 @@ static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, call_data *calld) { for (size_t i = 0; i < nops; i++) { grpc_subchannel_call_process_op(exec_ctx, call, ops[i]); } + gpr_free(ops); } static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, From 2b56dcc49ee44a9c30cb8397c0d0fdf0d42a5a65 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 13:13:05 -0800 Subject: [PATCH 088/167] Fix leak in client_channel --- src/core/ext/client_channel/client_channel.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 4bde9ec8054..e62fdf3adb5 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -884,13 +884,11 @@ retry: if (call == CANCELLED_CALL) { grpc_transport_stream_op_finish_with_failure( exec_ctx, op, GRPC_ERROR_REF(calld->cancel_error)); - GPR_TIMER_END("cc_start_transport_stream_op", 0); - return; + goto done; } if (call != NULL) { grpc_subchannel_call_process_op(exec_ctx, call, op); - GPR_TIMER_END("cc_start_transport_stream_op", 0); - return; + goto done; } /* if this is a cancellation, then we can raise our cancelled flag */ if (op->cancel_error != GRPC_ERROR_NONE) { @@ -916,8 +914,7 @@ retry: } grpc_transport_stream_op_finish_with_failure( exec_ctx, op, GRPC_ERROR_REF(op->cancel_error)); - GPR_TIMER_END("cc_start_transport_stream_op", 0); - return; + goto done; } } /* if we don't have a subchannel, try to get one */ @@ -961,6 +958,7 @@ retry: } /* nothing to be done but wait */ add_waiting_locked(calld, op); +done: GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "start_transport_stream_op"); GPR_TIMER_END("cc_start_transport_stream_op", 0); From c5de83531ee88919da57cc37dbc8a2725c7b94ca Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 14:08:05 -0800 Subject: [PATCH 089/167] Fix leaks --- src/core/ext/client_channel/client_channel.c | 26 ++++++++++---------- src/core/lib/transport/connectivity_state.c | 15 ++++++----- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index e62fdf3adb5..907d81866dc 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -226,19 +226,19 @@ static void on_lb_policy_state_changed_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { lb_policy_connectivity_watcher *w = arg; grpc_connectivity_state publish_state = w->state; - /* check if the notification is for a stale policy */ - if (w->lb_policy != w->chand->lb_policy) return; - - if (publish_state == GRPC_CHANNEL_SHUTDOWN && w->chand->resolver != NULL) { - publish_state = GRPC_CHANNEL_TRANSIENT_FAILURE; - grpc_resolver_channel_saw_error(exec_ctx, w->chand->resolver); - GRPC_LB_POLICY_UNREF(exec_ctx, w->chand->lb_policy, "channel"); - w->chand->lb_policy = NULL; - } - set_channel_connectivity_state_locked(exec_ctx, w->chand, publish_state, - GRPC_ERROR_REF(error), "lb_changed"); - if (w->state != GRPC_CHANNEL_SHUTDOWN) { - watch_lb_policy(exec_ctx, w->chand, w->lb_policy, w->state); + /* check if the notification is for the latest policy */ + if (w->lb_policy == w->chand->lb_policy) { + if (publish_state == GRPC_CHANNEL_SHUTDOWN && w->chand->resolver != NULL) { + publish_state = GRPC_CHANNEL_TRANSIENT_FAILURE; + grpc_resolver_channel_saw_error(exec_ctx, w->chand->resolver); + GRPC_LB_POLICY_UNREF(exec_ctx, w->chand->lb_policy, "channel"); + w->chand->lb_policy = NULL; + } + set_channel_connectivity_state_locked(exec_ctx, w->chand, publish_state, + GRPC_ERROR_REF(error), "lb_changed"); + if (w->state != GRPC_CHANNEL_SHUTDOWN) { + watch_lb_policy(exec_ctx, w->chand, w->lb_policy, w->state); + } } GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack, "watch_lb_policy"); diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c index e884567240d..afe1f6164d7 100644 --- a/src/core/lib/transport/connectivity_state.c +++ b/src/core/lib/transport/connectivity_state.c @@ -102,8 +102,9 @@ grpc_connectivity_state grpc_connectivity_state_check( grpc_connectivity_state grpc_connectivity_state_get( grpc_connectivity_state_tracker *tracker, grpc_error **error) { - grpc_connectivity_state cur =(grpc_connectivity_state) - gpr_atm_no_barrier_load(&tracker->current_state_atm); + grpc_connectivity_state cur = + (grpc_connectivity_state)gpr_atm_no_barrier_load( + &tracker->current_state_atm); if (grpc_connectivity_state_trace) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name, grpc_connectivity_state_name(cur)); @@ -122,8 +123,9 @@ bool grpc_connectivity_state_has_watchers( bool grpc_connectivity_state_notify_on_state_change( grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker, grpc_connectivity_state *current, grpc_closure *notify) { - grpc_connectivity_state cur =(grpc_connectivity_state) - gpr_atm_no_barrier_load(&tracker->current_state_atm); + grpc_connectivity_state cur = + (grpc_connectivity_state)gpr_atm_no_barrier_load( + &tracker->current_state_atm); if (grpc_connectivity_state_trace) { if (current == NULL) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: unsubscribe notify=%p", tracker, @@ -173,8 +175,9 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker, grpc_connectivity_state state, grpc_error *error, const char *reason) { - grpc_connectivity_state cur =(grpc_connectivity_state) - gpr_atm_no_barrier_load(&tracker->current_state_atm); + grpc_connectivity_state cur = + (grpc_connectivity_state)gpr_atm_no_barrier_load( + &tracker->current_state_atm); grpc_connectivity_state_watcher *w; if (grpc_connectivity_state_trace) { const char *error_string = grpc_error_string(error); From 5634ef6e4a86d284d973c400f49b6370ad6035eb Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 14:25:32 -0800 Subject: [PATCH 090/167] Make combiners refcounted, to facilitate sharing --- .../transport/chttp2/transport/chttp2_transport.c | 2 +- src/core/lib/iomgr/combiner.c | 12 +++++++++++- src/core/lib/iomgr/combiner.h | 5 +++-- src/core/lib/iomgr/resource_quota.c | 2 +- test/core/iomgr/combiner_test.c | 8 ++++---- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 8a9eaa8b6a2..d576f85dd25 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -168,7 +168,7 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream_map_destroy(&t->stream_map); grpc_connectivity_state_destroy(exec_ctx, &t->channel_callback.state_tracker); - grpc_combiner_destroy(exec_ctx, t->combiner); + grpc_combiner_unref(exec_ctx, t->combiner); cancel_pings(exec_ctx, t, GRPC_ERROR_CREATE("Transport destroyed")); diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c index ba6c7087a92..93d8c0bd70a 100644 --- a/src/core/lib/iomgr/combiner.c +++ b/src/core/lib/iomgr/combiner.c @@ -72,6 +72,7 @@ struct grpc_combiner { bool final_list_covered_by_poller; grpc_closure_list final_list; grpc_closure offload; + gpr_refcount refs; }; static void combiner_exec_uncovered(grpc_exec_ctx *exec_ctx, @@ -126,6 +127,7 @@ static bool is_covered_by_poller(grpc_combiner *lock) { grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue) { grpc_combiner *lock = gpr_malloc(sizeof(*lock)); + gpr_ref_init(&lock->refs, 1); lock->next_combiner_on_this_exec_ctx = NULL; lock->time_to_execute_final_list = false; lock->optional_workqueue = optional_workqueue; @@ -152,7 +154,7 @@ static void really_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { gpr_free(lock); } -void grpc_combiner_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { +static void start_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { gpr_atm old_state = gpr_atm_full_fetch_add(&lock->state, -STATE_UNORPHANED); GRPC_COMBINER_TRACE(gpr_log( GPR_DEBUG, "C:%p really_destroy old_state=%" PRIdPTR, lock, old_state)); @@ -161,6 +163,14 @@ void grpc_combiner_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { } } +void grpc_combiner_unref(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { + if (gpr_unref(&lock->refs)) { + start_destroy(exec_ctx, lock); + } +} + +void grpc_combiner_ref(grpc_combiner *lock) { gpr_ref(&lock->refs); } + static void push_last_on_exec_ctx(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { lock->next_combiner_on_this_exec_ctx = NULL; diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h index 81dff85d402..352e87d050a 100644 --- a/src/core/lib/iomgr/combiner.h +++ b/src/core/lib/iomgr/combiner.h @@ -48,8 +48,9 @@ // Initialize the lock, with an optional workqueue to shift load to when // necessary grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue); -// Destroy the lock -void grpc_combiner_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock); +// Ref/unref the lock, for when we're sharing the lock ownership +void grpc_combiner_ref(grpc_combiner *lock); +void grpc_combiner_unref(grpc_exec_ctx *exec_ctx, grpc_combiner *lock); // Fetch a scheduler to schedule closures against grpc_closure_scheduler *grpc_combiner_scheduler(grpc_combiner *lock, bool covered_by_poller); diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 2cc979467f6..3c701164ad6 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -599,7 +599,7 @@ grpc_resource_quota *grpc_resource_quota_create(const char *name) { void grpc_resource_quota_unref_internal(grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota) { if (gpr_unref(&resource_quota->refs)) { - grpc_combiner_destroy(exec_ctx, resource_quota->combiner); + grpc_combiner_unref(exec_ctx, resource_quota->combiner); gpr_free(resource_quota->name); gpr_free(resource_quota); } diff --git a/test/core/iomgr/combiner_test.c b/test/core/iomgr/combiner_test.c index 4c9275a6732..08cd2c1e26a 100644 --- a/test/core/iomgr/combiner_test.c +++ b/test/core/iomgr/combiner_test.c @@ -44,7 +44,7 @@ static void test_no_op(void) { gpr_log(GPR_DEBUG, "test_no_op"); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_combiner_destroy(&exec_ctx, grpc_combiner_create(NULL)); + grpc_combiner_unref(&exec_ctx, grpc_combiner_create(NULL)); grpc_exec_ctx_finish(&exec_ctx); } @@ -65,7 +65,7 @@ static void test_execute_one(void) { GRPC_ERROR_NONE); grpc_exec_ctx_flush(&exec_ctx); GPR_ASSERT(done); - grpc_combiner_destroy(&exec_ctx, lock); + grpc_combiner_unref(&exec_ctx, lock); grpc_exec_ctx_finish(&exec_ctx); } @@ -125,7 +125,7 @@ static void test_execute_many(void) { gpr_thd_join(thds[i]); } grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_combiner_destroy(&exec_ctx, lock); + grpc_combiner_unref(&exec_ctx, lock); grpc_exec_ctx_finish(&exec_ctx); } @@ -153,7 +153,7 @@ static void test_execute_finally(void) { GRPC_ERROR_NONE); grpc_exec_ctx_flush(&exec_ctx); GPR_ASSERT(got_in_finally); - grpc_combiner_destroy(&exec_ctx, lock); + grpc_combiner_unref(&exec_ctx, lock); grpc_exec_ctx_finish(&exec_ctx); } From 51319c7b6608060b33e66b4a1d7d3d0dae69e2d9 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 7 Feb 2017 10:19:02 -0800 Subject: [PATCH 091/167] Add Cronet remote interop test to Jenkins --- src/objective-c/tests/InteropTests.m | 3 +++ src/objective-c/tests/run_tests.sh | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m index c3935ce1e09..5584246ad94 100644 --- a/src/objective-c/tests/InteropTests.m +++ b/src/objective-c/tests/InteropTests.m @@ -169,6 +169,8 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } +#ifndef GRPC_COMPILE_WITH_CRONET +// TODO (mxyan): Fix this test - (void)testResponsesOverMaxSizeFailWithActionableMessage { XCTAssertNotNil(self.class.host); __weak XCTestExpectation *expectation = [self expectationWithDescription:@"ResponseOverMaxSize"]; @@ -189,6 +191,7 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } +#endif - (void)testResponsesOver4MBAreAcceptedIfOptedIn { XCTAssertNotNil(self.class.host); diff --git a/src/objective-c/tests/run_tests.sh b/src/objective-c/tests/run_tests.sh index 81c28853205..677459e0822 100755 --- a/src/objective-c/tests/run_tests.sh +++ b/src/objective-c/tests/run_tests.sh @@ -61,3 +61,11 @@ xcodebuild \ test \ | egrep "$XCODEBUILD_FILTER" \ | egrep -v "(GPBDictionary|GPBArray)" - + +xcodebuild \ + -workspace Tests.xcworkspace \ + -scheme InteropTestsRemoteWithCronet \ + -destination name="iPhone 6" \ + test \ + | egrep "$XCODEBUILD_FILTER" \ + | egrep -v "(GPBDictionary|GPBArray)" - From 3845e559816025477e59030b62ad864d5c093499 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 15:19:28 -0800 Subject: [PATCH 092/167] Add debug macros --- .../chttp2/transport/chttp2_transport.c | 2 +- src/core/lib/iomgr/combiner.c | 20 ++++++++++++++++-- src/core/lib/iomgr/combiner.h | 21 +++++++++++++++++-- src/core/lib/iomgr/resource_quota.c | 2 +- test/core/iomgr/combiner_test.c | 8 +++---- 5 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index d576f85dd25..3ee5e976f84 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -168,7 +168,7 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream_map_destroy(&t->stream_map); grpc_connectivity_state_destroy(exec_ctx, &t->channel_callback.state_tracker); - grpc_combiner_unref(exec_ctx, t->combiner); + GRPC_COMBINER_UNREF(exec_ctx, t->combiner, "chttp2_transport"); cancel_pings(exec_ctx, t, GRPC_ERROR_CREATE("Transport destroyed")); diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c index 93d8c0bd70a..fa9966c3a69 100644 --- a/src/core/lib/iomgr/combiner.c +++ b/src/core/lib/iomgr/combiner.c @@ -163,13 +163,29 @@ static void start_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { } } -void grpc_combiner_unref(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { +#ifdef GRPC_COMBINER_REFCOUNT_DEBUG +#define GRPC_COMBINER_DEBUG_SPAM(op, delta) \ + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, \ + "combiner[%p] %s %" PRIdPTR " --> %" PRIdPTR " %s", lock, (op), \ + gpr_atm_no_barrier_load(&lock->refs.count), \ + gpr_atm_no_barrier_load(&lock->refs.count) + (delta), reason); +#else +#define GRPC_COMBINER_DEBUG_SPAM(op, delta) +#endif + +void grpc_combiner_unref(grpc_exec_ctx *exec_ctx, + grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS) { + GRPC_COMBINER_DEBUG_SPAM("UNREF", -1); if (gpr_unref(&lock->refs)) { start_destroy(exec_ctx, lock); } } -void grpc_combiner_ref(grpc_combiner *lock) { gpr_ref(&lock->refs); } +grpc_combiner *grpc_combiner_ref(grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS) { + GRPC_COMBINER_DEBUG_SPAM(" REF", 1); + gpr_ref(&lock->refs); + return lock; +} static void push_last_on_exec_ctx(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h index 352e87d050a..065689e38f7 100644 --- a/src/core/lib/iomgr/combiner.h +++ b/src/core/lib/iomgr/combiner.h @@ -48,9 +48,26 @@ // Initialize the lock, with an optional workqueue to shift load to when // necessary grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue); + +#define GRPC_COMBINER_REFCOUNT_DEBUG +#ifdef GRPC_COMBINER_REFCOUNT_DEBUG +#define GRPC_COMBINER_DEBUG_ARGS \ + , const char *file, int line, const char *reason +#define GRPC_COMBINER_REF(combiner, reason) \ + grpc_combiner_ref((combiner), __FILE__, __LINE__, (reason)) +#define GRPC_COMBINER_UNREF(exec_ctx, combiner, reason) \ + grpc_combiner_unref((exec_ctx), (combiner), __FILE__, __LINE__, (reason)) +#else +#define GRPC_COMBINER_DEBUG_ARGS +#define GRPC_COMBINER_REF(combiner) grpc_combiner_ref((combiner)) +#define GRPC_COMBINER_UNREF(exec_ctx, combiner) \ + grpc_combiner_unref((exec_ctx), (combiner)) +#endif + // Ref/unref the lock, for when we're sharing the lock ownership -void grpc_combiner_ref(grpc_combiner *lock); -void grpc_combiner_unref(grpc_exec_ctx *exec_ctx, grpc_combiner *lock); +grpc_combiner *grpc_combiner_ref(grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS); +void grpc_combiner_unref(grpc_exec_ctx *exec_ctx, + grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS); // Fetch a scheduler to schedule closures against grpc_closure_scheduler *grpc_combiner_scheduler(grpc_combiner *lock, bool covered_by_poller); diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 3c701164ad6..511ffdcdf13 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -599,7 +599,7 @@ grpc_resource_quota *grpc_resource_quota_create(const char *name) { void grpc_resource_quota_unref_internal(grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota) { if (gpr_unref(&resource_quota->refs)) { - grpc_combiner_unref(exec_ctx, resource_quota->combiner); + GRPC_COMBINER_UNREF(exec_ctx, resource_quota->combiner, "resource_quota"); gpr_free(resource_quota->name); gpr_free(resource_quota); } diff --git a/test/core/iomgr/combiner_test.c b/test/core/iomgr/combiner_test.c index 08cd2c1e26a..bc4d2af8ac4 100644 --- a/test/core/iomgr/combiner_test.c +++ b/test/core/iomgr/combiner_test.c @@ -44,7 +44,7 @@ static void test_no_op(void) { gpr_log(GPR_DEBUG, "test_no_op"); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_combiner_unref(&exec_ctx, grpc_combiner_create(NULL)); + GRPC_COMBINER_UNREF(&exec_ctx, grpc_combiner_create(NULL), "test_no_op"); grpc_exec_ctx_finish(&exec_ctx); } @@ -65,7 +65,7 @@ static void test_execute_one(void) { GRPC_ERROR_NONE); grpc_exec_ctx_flush(&exec_ctx); GPR_ASSERT(done); - grpc_combiner_unref(&exec_ctx, lock); + GRPC_COMBINER_UNREF(&exec_ctx, lock, "test_execute_one"); grpc_exec_ctx_finish(&exec_ctx); } @@ -125,7 +125,7 @@ static void test_execute_many(void) { gpr_thd_join(thds[i]); } grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_combiner_unref(&exec_ctx, lock); + GRPC_COMBINER_UNREF(&exec_ctx, lock, "test_execute_many"); grpc_exec_ctx_finish(&exec_ctx); } @@ -153,7 +153,7 @@ static void test_execute_finally(void) { GRPC_ERROR_NONE); grpc_exec_ctx_flush(&exec_ctx); GPR_ASSERT(got_in_finally); - grpc_combiner_unref(&exec_ctx, lock); + GRPC_COMBINER_UNREF(&exec_ctx, lock, "test_execute_finally"); grpc_exec_ctx_finish(&exec_ctx); } From f6723896f0370ed1e04d8fff7e50750a574c0093 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 15:21:35 -0800 Subject: [PATCH 093/167] Default debug off --- src/core/lib/iomgr/combiner.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h index 065689e38f7..f1fa941d643 100644 --- a/src/core/lib/iomgr/combiner.h +++ b/src/core/lib/iomgr/combiner.h @@ -49,7 +49,7 @@ // necessary grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue); -#define GRPC_COMBINER_REFCOUNT_DEBUG +//#define GRPC_COMBINER_REFCOUNT_DEBUG #ifdef GRPC_COMBINER_REFCOUNT_DEBUG #define GRPC_COMBINER_DEBUG_ARGS \ , const char *file, int line, const char *reason From b1b2854d3a2beffa8e33d7847bf6072636a38f46 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 15:23:12 -0800 Subject: [PATCH 094/167] Fix non-debug --- src/core/lib/iomgr/combiner.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h index f1fa941d643..c7aa0866174 100644 --- a/src/core/lib/iomgr/combiner.h +++ b/src/core/lib/iomgr/combiner.h @@ -59,8 +59,8 @@ grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue); grpc_combiner_unref((exec_ctx), (combiner), __FILE__, __LINE__, (reason)) #else #define GRPC_COMBINER_DEBUG_ARGS -#define GRPC_COMBINER_REF(combiner) grpc_combiner_ref((combiner)) -#define GRPC_COMBINER_UNREF(exec_ctx, combiner) \ +#define GRPC_COMBINER_REF(combiner, reason) grpc_combiner_ref((combiner)) +#define GRPC_COMBINER_UNREF(exec_ctx, combiner, reason) \ grpc_combiner_unref((exec_ctx), (combiner)) #endif From af1c45197de24b522e6e8fef67caff3965d2a6b6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 16:38:23 -0800 Subject: [PATCH 095/167] Add comment --- src/core/lib/iomgr/combiner.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h index c7aa0866174..75dcb0b70a4 100644 --- a/src/core/lib/iomgr/combiner.h +++ b/src/core/lib/iomgr/combiner.h @@ -65,6 +65,7 @@ grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue); #endif // Ref/unref the lock, for when we're sharing the lock ownership +// Prefer to use the macros above grpc_combiner *grpc_combiner_ref(grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS); void grpc_combiner_unref(grpc_exec_ctx *exec_ctx, grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS); From e7f0b8579bab3401c4314095657a935b3d641759 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 8 Feb 2017 19:06:10 -0800 Subject: [PATCH 096/167] use grpc_csharp_ext built by cmake --- src/csharp/Grpc.Core.Tests/project.json | 8 ++++---- src/csharp/Grpc.Core/NativeDeps.Windows.targets | 2 +- .../Grpc.Examples.MathClient/project.json | 8 ++++---- .../Grpc.Examples.MathServer/project.json | 8 ++++---- src/csharp/Grpc.Examples.Tests/project.json | 8 ++++---- src/csharp/Grpc.HealthCheck.Tests/project.json | 8 ++++---- .../Grpc.IntegrationTesting.Client/project.json | 8 ++++---- .../project.json | 8 ++++---- .../Grpc.IntegrationTesting.Server/project.json | 8 ++++---- .../project.json | 8 ++++---- src/csharp/Grpc.IntegrationTesting/project.json | 8 ++++---- src/csharp/Grpc.Reflection.Tests/project.json | 8 ++++---- templates/src/csharp/build_options.include | 8 ++++---- .../helper_scripts/pre_build_csharp.bat | 13 ++++++++++++- tools/run_tests/run_tests.py | 17 +++++++++-------- 15 files changed, 70 insertions(+), 58 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/project.json b/src/csharp/Grpc.Core.Tests/project.json index 509084a71a6..045207a413d 100644 --- a/src/csharp/Grpc.Core.Tests/project.json +++ b/src/csharp/Grpc.Core.Tests/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.Core/NativeDeps.Windows.targets b/src/csharp/Grpc.Core/NativeDeps.Windows.targets index 93db0935bc1..623f58b95b0 100644 --- a/src/csharp/Grpc.Core/NativeDeps.Windows.targets +++ b/src/csharp/Grpc.Core/NativeDeps.Windows.targets @@ -1,7 +1,7 @@  - + PreserveNewest grpc_csharp_ext.x86.dll diff --git a/src/csharp/Grpc.Examples.MathClient/project.json b/src/csharp/Grpc.Examples.MathClient/project.json index 9a8880b5d42..81c17151aa5 100644 --- a/src/csharp/Grpc.Examples.MathClient/project.json +++ b/src/csharp/Grpc.Examples.MathClient/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.Examples.MathServer/project.json b/src/csharp/Grpc.Examples.MathServer/project.json index 9a8880b5d42..81c17151aa5 100644 --- a/src/csharp/Grpc.Examples.MathServer/project.json +++ b/src/csharp/Grpc.Examples.MathServer/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.Examples.Tests/project.json b/src/csharp/Grpc.Examples.Tests/project.json index 3e130beeac9..e509621a293 100644 --- a/src/csharp/Grpc.Examples.Tests/project.json +++ b/src/csharp/Grpc.Examples.Tests/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.HealthCheck.Tests/project.json b/src/csharp/Grpc.HealthCheck.Tests/project.json index addc782afe8..654454d1cb7 100644 --- a/src/csharp/Grpc.HealthCheck.Tests/project.json +++ b/src/csharp/Grpc.HealthCheck.Tests/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.IntegrationTesting.Client/project.json b/src/csharp/Grpc.IntegrationTesting.Client/project.json index ad81cbc48b3..f90528151b4 100644 --- a/src/csharp/Grpc.IntegrationTesting.Client/project.json +++ b/src/csharp/Grpc.IntegrationTesting.Client/project.json @@ -16,8 +16,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -37,8 +37,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json b/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json index 2c7643b74f0..161e602abca 100644 --- a/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json +++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json @@ -16,8 +16,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -37,8 +37,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.IntegrationTesting.Server/project.json b/src/csharp/Grpc.IntegrationTesting.Server/project.json index ad81cbc48b3..f90528151b4 100644 --- a/src/csharp/Grpc.IntegrationTesting.Server/project.json +++ b/src/csharp/Grpc.IntegrationTesting.Server/project.json @@ -16,8 +16,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -37,8 +37,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/project.json b/src/csharp/Grpc.IntegrationTesting.StressClient/project.json index ad81cbc48b3..f90528151b4 100644 --- a/src/csharp/Grpc.IntegrationTesting.StressClient/project.json +++ b/src/csharp/Grpc.IntegrationTesting.StressClient/project.json @@ -16,8 +16,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -37,8 +37,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.IntegrationTesting/project.json b/src/csharp/Grpc.IntegrationTesting/project.json index e47b5953da0..eba54318a5b 100644 --- a/src/csharp/Grpc.IntegrationTesting/project.json +++ b/src/csharp/Grpc.IntegrationTesting/project.json @@ -16,8 +16,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -37,8 +37,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.Reflection.Tests/project.json b/src/csharp/Grpc.Reflection.Tests/project.json index 61d3b7e47b7..b90834a25eb 100644 --- a/src/csharp/Grpc.Reflection.Tests/project.json +++ b/src/csharp/Grpc.Reflection.Tests/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/templates/src/csharp/build_options.include b/templates/src/csharp/build_options.include index 9a32b7c6f45..db4cc198039 100644 --- a/templates/src/csharp/build_options.include +++ b/templates/src/csharp/build_options.include @@ -21,8 +21,8 @@ "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", % endif - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -44,8 +44,8 @@ "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", % endif - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/tools/run_tests/helper_scripts/pre_build_csharp.bat b/tools/run_tests/helper_scripts/pre_build_csharp.bat index 139955d4dae..f37f63b5840 100644 --- a/tools/run_tests/helper_scripts/pre_build_csharp.bat +++ b/tools/run_tests/helper_scripts/pre_build_csharp.bat @@ -31,9 +31,21 @@ setlocal +set ARCHITECTURE=%1 + @rem enter repo root cd /d %~dp0\..\..\.. +mkdir cmake +cd cmake +mkdir build +cd build +mkdir %ARCHITECTURE% +cd %ARCHITECTURE% +@rem TODO(jtattermusch): Stop hardcoding path to yasm once Jenkins workers can locate yasm correctly +cmake -G "Visual Studio 14 2015" -A %ARCHITECTURE% -DgRPC_BUILD_TESTS=OFF -DCMAKE_ASM_NASM_COMPILER="C:/Program Files (x86)/yasm/yasm.exe" ../../.. || goto :error +cd ..\..\.. + @rem Location of nuget.exe set NUGET=C:\nuget\nuget.exe @@ -42,7 +54,6 @@ if exist %NUGET% ( @rem Restore Grpc packages by packages since Nuget client 3.4.4 doesnt support restore @rem by solution @rem Moving into each directory to let the restores work based on per-project packages.config files - %NUGET% restore vsprojects/grpc_csharp_ext.sln || goto :error cd src/csharp diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 75571aaadbe..7f066328d02 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -722,12 +722,10 @@ class CSharpLanguage(object): self.config = config self.args = args if self.platform == 'windows': - # Explicitly choosing between x86 and x64 arch doesn't work yet + _check_compiler(self.args.compiler, ['coreclr', 'default']) _check_arch(self.args.arch, ['default']) - # CoreCLR use 64bit runtime by default. - arch_option = 'x64' if self.args.compiler == 'coreclr' else self.args.arch - self._make_options = [_windows_toolset_option(self.args.compiler), - _windows_arch_option(arch_option)] + self._cmake_arch_option = 'x64' if self.args.compiler == 'coreclr' else 'Win32' + self._make_options = [] else: _check_compiler(self.args.compiler, ['default', 'coreclr']) if self.platform == 'linux' and self.args.compiler == 'coreclr': @@ -799,7 +797,7 @@ class CSharpLanguage(object): def pre_build_steps(self): if self.platform == 'windows': - return [['tools\\run_tests\\helper_scripts\\pre_build_csharp.bat']] + return [['tools\\run_tests\\helper_scripts\\pre_build_csharp.bat', self._cmake_arch_option]] else: return [['tools/run_tests/helper_scripts/pre_build_csharp.sh']] @@ -830,7 +828,10 @@ class CSharpLanguage(object): return [['tools/run_tests/helper_scripts/post_tests_csharp.sh']] def makefile_name(self): - return 'Makefile' + if self.platform == 'windows': + return 'cmake/build/%s/Makefile' % self._cmake_arch_option + else: + return 'Makefile' def dockerfile_dir(self): return 'tools/dockerfile/test/csharp_%s_%s' % (self._docker_distro, @@ -1292,7 +1293,7 @@ def make_jobspec(cfg, targets, makefile='Makefile'): return [jobset.JobSpec(['cmake', '--build', '.', '--target', '%s' % target, '--config', _MSBUILD_CONFIG[cfg]], - cwd='cmake/build', + cwd=os.path.dirname(makefile), timeout_seconds=None) for target in targets] extra_args = [] # better do parallel compilation From dc5e509b4ebe544bedba5b39990155056ad0c102 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 9 Feb 2017 16:15:12 -0800 Subject: [PATCH 097/167] remove vs2010 references from run_tests.py --- tools/run_tests/run_tests.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 7f066328d02..a9a7ad30129 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1043,8 +1043,6 @@ def _windows_build_bat(compiler): return 'vsprojects\\build_vs2013.bat' elif compiler == 'vs2015': return 'vsprojects\\build_vs2015.bat' - elif compiler == 'vs2010': - return 'vsprojects\\build_vs2010.bat' else: print('Compiler %s not supported.' % compiler) sys.exit(1) @@ -1057,8 +1055,6 @@ def _windows_toolset_option(compiler): return '/p:PlatformToolset=v120' elif compiler == 'vs2015': return '/p:PlatformToolset=v140' - elif compiler == 'vs2010': - return '/p:PlatformToolset=v100' else: print('Compiler %s not supported.' % compiler) sys.exit(1) @@ -1148,7 +1144,7 @@ argp.add_argument('--compiler', choices=['default', 'gcc4.4', 'gcc4.6', 'gcc4.8', 'gcc4.9', 'gcc5.3', 'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7', - 'vs2010', 'vs2013', 'vs2015', + 'vs2013', 'vs2015', 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'node0.12', 'node4', 'node5', 'node6', 'node7', 'electron1.3', From 397d2d96213b13254931fa6b3f73ec7cfa105245 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 9 Feb 2017 16:24:26 -0800 Subject: [PATCH 098/167] compile C# with VS2015 --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index a9a7ad30129..eb4b5c276bc 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -815,7 +815,7 @@ class CSharpLanguage(object): return [['tools/run_tests/helper_scripts/build_csharp_coreclr.sh']] else: if self.platform == 'windows': - return [[_windows_build_bat(self.args.compiler), + return [['vsprojects\\build_vs2015.bat', 'src/csharp/Grpc.sln', '/p:Configuration=%s' % _MSBUILD_CONFIG[self.config.build_config]]] else: From dd900d55b875e3bd3c83530dd17c54356e5067dd Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 9 Feb 2017 16:26:41 -0800 Subject: [PATCH 099/167] cleanup windows_build_bat --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index eb4b5c276bc..8b76193cbe1 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1039,7 +1039,7 @@ def _check_arch_option(arch): def _windows_build_bat(compiler): """Returns name of build.bat for selected compiler.""" # For CoreCLR, fall back to the default compiler for C core - if compiler == 'default' or compiler == 'vs2013' or compiler == 'coreclr': + if compiler == 'default' or compiler == 'vs2013': return 'vsprojects\\build_vs2013.bat' elif compiler == 'vs2015': return 'vsprojects\\build_vs2015.bat' From 6dce2f5d4859a1ae240f441cd383dadc055dd64d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 9 Feb 2017 17:03:03 -0800 Subject: [PATCH 100/167] adjust C# win artifact build --- tools/run_tests/artifacts/artifact_targets.py | 16 +++------------- .../artifacts/build_artifact_csharp.bat | 10 ++++++++-- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 005d99790ad..aba7b8a3054 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -183,24 +183,14 @@ class CSharpExtArtifact: self.labels = ['artifact', 'csharp', platform, arch] def pre_build_jobspecs(self): - if self.platform == 'windows': - return [create_jobspec('prebuild_%s' % self.name, - ['tools\\run_tests\\helper_scripts\\pre_build_c.bat'], - shell=True, - flake_retries=5, - timeout_retries=2)] - else: - return [] + return [] def build_jobspec(self): if self.platform == 'windows': - msbuild_platform = 'Win32' if self.arch == 'x86' else self.arch + cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec(self.name, ['tools\\run_tests\\artifacts\\build_artifact_csharp.bat', - 'vsprojects\\grpc_csharp_ext.sln', - '/p:Configuration=Release', - '/p:PlatformToolset=v120', - '/p:Platform=%s' % msbuild_platform], + cmake_arch_option], shell=True) else: environ = {'CONFIG': 'opt', diff --git a/tools/run_tests/artifacts/build_artifact_csharp.bat b/tools/run_tests/artifacts/build_artifact_csharp.bat index 24c8d485f9f..f84ebc5a359 100644 --- a/tools/run_tests/artifacts/build_artifact_csharp.bat +++ b/tools/run_tests/artifacts/build_artifact_csharp.bat @@ -29,10 +29,16 @@ @rem Builds C# artifacts on Windows -@call vsprojects\build_vs2013.bat %* || goto :error +set ARCHITECTURE=%1 + +@call tools\run_tests\helper_scripts\pre_build_csharp.bat %ARCHITECTURE% || goto :error + +cd cmake\build\%ARCHITECTURE% +cmake --build . --target grpc_csharp_ext --config Release +cd ..\..\.. mkdir artifacts -copy /Y vsprojects\Release\grpc_csharp_ext.dll artifacts || copy /Y vsprojects\x64\Release\grpc_csharp_ext.dll artifacts || goto :error +copy /Y cmake\build\Win32\Release\grpc_csharp_ext.dll artifacts || copy /Y cmake\build\x64\Release\grpc_csharp_ext.dll artifacts || goto :error goto :EOF From 756b13787801adaaf25bf11f9d0b114028fb8f76 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 9 Feb 2017 17:13:03 -0800 Subject: [PATCH 101/167] remove stale buildall.bat script and update readme --- src/csharp/README.md | 18 ++++--------- src/csharp/buildall.bat | 59 ----------------------------------------- 2 files changed, 5 insertions(+), 72 deletions(-) delete mode 100644 src/csharp/buildall.bat diff --git a/src/csharp/README.md b/src/csharp/README.md index 0405ff88a01..8468eb991ec 100644 --- a/src/csharp/README.md +++ b/src/csharp/README.md @@ -37,23 +37,15 @@ BUILD FROM SOURCE You only need to go through these steps if you are planning to develop gRPC C#. If you are a user of gRPC C#, go to Usage section above. -**Windows** +**Windows, Linux or Mac OS X** -- The grpc_csharp_ext native library needs to be built so you can build the gRPC C# solution. Open the - solution `vsprojects/grpc_csharp_ext.sln` in Visual Studio and build it. - -- Open `src\csharp\Grpc.sln` (path is relative to gRPC repository root) - using Visual Studio - -**Linux and Mac OS X** - -- The grpc_csharp_ext native library needs to be built so you can build the gRPC C# solution: - ```sh +- The easiest way to build is using the `run_tests.py` script that will take care of building the `grpc_csharp_ext` native library: + ``` # from the gRPC repository root - $ tools/run_tests/run_tests.py -c dbg -l csharp --build_only + $ python tools/run_tests/run_tests.py -c dbg -l csharp --build_only ``` -- Use MonoDevelop / Xamarin Studio to open the solution Grpc.sln +- Use Visual Studio / MonoDevelop / Xamarin Studio to open the solution Grpc.sln RUNNING TESTS ------------- diff --git a/src/csharp/buildall.bat b/src/csharp/buildall.bat deleted file mode 100644 index 0beb30c1989..00000000000 --- a/src/csharp/buildall.bat +++ /dev/null @@ -1,59 +0,0 @@ -@rem Copyright 2016, Google Inc. -@rem All rights reserved. -@rem -@rem Redistribution and use in source and binary forms, with or without -@rem modification, are permitted provided that the following conditions are -@rem met: -@rem -@rem * Redistributions of source code must retain the above copyright -@rem notice, this list of conditions and the following disclaimer. -@rem * Redistributions in binary form must reproduce the above -@rem copyright notice, this list of conditions and the following disclaimer -@rem in the documentation and/or other materials provided with the -@rem distribution. -@rem * Neither the name of Google Inc. nor the names of its -@rem contributors may be used to endorse or promote products derived from -@rem this software without specific prior written permission. -@rem -@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -@rem Convenience script to build gRPC C# from command line - -setlocal - -@rem enter this directory -cd /d %~dp0 - -@rem Set VS variables (uses Visual Studio 2013) -@call "%VS120COMNTOOLS%\..\..\vc\vcvarsall.bat" x86 - -@rem Fetch all dependencies -nuget restore ..\..\vsprojects\grpc.sln || goto :error -nuget restore ..\..\vsprojects\grpc_csharp_ext.sln || goto :error -nuget restore ..\..\vsprojects\grpc_protoc_plugins.sln || goto :error -nuget restore Grpc.sln || goto :error - -@rem Build the C# native extension -msbuild ..\..\vsprojects\grpc_csharp_ext.sln /p:Configuration=Debug /p:PlatformToolset=v120 || goto :error -msbuild ..\..\vsprojects\grpc_csharp_ext.sln /p:Configuration=Release /p:PlatformToolset=v120 || goto :error - -msbuild Grpc.sln /p:Configuration=Debug || goto :error -msbuild Grpc.sln /p:Configuration=Release || goto :error - -endlocal - -goto :EOF - -:error -echo Failed! -exit /b %errorlevel% From 8886a818656a1ea454bf01b2943ef03418fdc44e Mon Sep 17 00:00:00 2001 From: Dan Born Date: Fri, 8 Jul 2016 16:06:07 -0700 Subject: [PATCH 102/167] Expand wildcard IP addresses on servers. --- include/grpc/impl/codegen/grpc_types.h | 2 + src/core/lib/iomgr/tcp_server_posix.c | 331 ++++++++++++++++++------ test/core/handshake/client_ssl.c | 27 +- test/core/iomgr/tcp_server_posix_test.c | 295 +++++++++++++++------ 4 files changed, 490 insertions(+), 165 deletions(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index bc0b2bd7f86..6661547f6b6 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -222,6 +222,8 @@ typedef struct { /** If non-zero, a pointer to a buffer pool (use grpc_resource_quota_arg_vtable to fetch an appropriate pointer arg vtable) */ #define GRPC_ARG_RESOURCE_QUOTA "grpc.resource_quota" +/** If non-zero, expand wildcard addresses to a list of local addresses. */ +#define GRPC_ARG_EXPAND_WILDCARD_ADDRS "grpc.expand_wildcard_addrs" /** Service config data in JSON form. Not intended for use outside of tests. */ #define GRPC_ARG_SERVICE_CONFIG "grpc.service_config" /** LB policy name. */ diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index e9e7511c9cc..786e6189d72 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -44,6 +44,7 @@ #include #include +#include #include #include #include @@ -115,6 +116,8 @@ struct grpc_tcp_server { bool shutdown; /* use SO_REUSEPORT */ bool so_reuseport; + /* expand wildcard addresses to a list of all local addresses */ + bool expand_wildcard_addrs; /* linked list of server ports */ grpc_tcp_listener *head; @@ -161,6 +164,7 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server)); s->so_reuseport = has_so_reuseport; s->resource_quota = grpc_resource_quota_create(NULL); + s->expand_wildcard_addrs = false; for (size_t i = 0; i < (args == NULL ? 0 : args->num_args); i++) { if (0 == strcmp(GRPC_ARG_ALLOW_REUSEPORT, args->args[i].key)) { if (args->args[i].type == GRPC_ARG_INTEGER) { @@ -183,6 +187,15 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, return GRPC_ERROR_CREATE(GRPC_ARG_RESOURCE_QUOTA " must be a pointer to a buffer pool"); } + } else if (0 == strcmp(GRPC_ARG_EXPAND_WILDCARD_ADDRS, args->args[i].key)) { + if (args->args[i].type == GRPC_ARG_INTEGER) { + s->expand_wildcard_addrs = (args->args[i].value.integer != 0); + } else { + grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota); + gpr_free(s); + return GRPC_ERROR_CREATE(GRPC_ARG_EXPAND_WILDCARD_ADDRS + " must be an integer"); + } } } gpr_ref_init(&s->refs, 1); @@ -504,9 +517,224 @@ static grpc_error *add_socket_to_server(grpc_tcp_server *s, int fd, return err; } -/* Insert count new listeners after listener. Every new listener will have the - same listen address as listener (SO_REUSEPORT must be enabled). Every new - listener is a sibling of listener. */ +/* If successful, add a listener to s for addr, set *dsmode for the socket, and + return the *listener. */ +static grpc_error *add_addr_to_server(grpc_tcp_server *s, + const grpc_resolved_address *addr, + unsigned port_index, unsigned fd_index, + grpc_dualstack_mode *dsmode, + grpc_tcp_listener **listener) { + grpc_resolved_address addr4_copy; + int fd; + grpc_error *err = + grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, dsmode, &fd); + if (err != GRPC_ERROR_NONE) { + return err; + } + if (*dsmode == GRPC_DSMODE_IPV4 && + grpc_sockaddr_is_v4mapped(addr, &addr4_copy)) { + addr = &addr4_copy; + } + return add_socket_to_server(s, fd, addr, port_index, fd_index, listener); +} + +/* Bind to "::" to get a port number not used by any address. */ +static grpc_error *get_unused_port(int *port) { + grpc_resolved_address wild; + grpc_sockaddr_make_wildcard6(0, &wild); + grpc_dualstack_mode dsmode; + int fd; + grpc_error *err = + grpc_create_dualstack_socket(&wild, SOCK_STREAM, 0, &dsmode, &fd); + if (err != GRPC_ERROR_NONE) { + return err; + } + if (dsmode == GRPC_DSMODE_IPV4) { + grpc_sockaddr_make_wildcard4(0, &wild); + } + if (bind(fd, (const struct sockaddr *)wild.addr, (socklen_t)wild.len) != 0) { + err = GRPC_OS_ERROR(errno, "bind"); + close(fd); + return err; + } + if (getsockname(fd, (struct sockaddr *)wild.addr, (socklen_t *)&wild.len) != + 0) { + err = GRPC_OS_ERROR(errno, "getsockname"); + close(fd); + return err; + } + close(fd); + *port = grpc_sockaddr_get_port(&wild); + return *port <= 0 ? GRPC_ERROR_CREATE("Bad port") : GRPC_ERROR_NONE; +} + +/* Return the listener in s with address addr or NULL. */ +static grpc_tcp_listener *find_listener_with_addr(grpc_tcp_server *s, + grpc_resolved_address *addr) { + grpc_tcp_listener *l; + gpr_mu_lock(&s->mu); + for (l = s->head; l != NULL; l = l->next) { + if (l->addr.len != addr->len) { + continue; + } + if (memcmp(l->addr.addr, addr->addr, addr->len) == 0) { + break; + } + } + gpr_mu_unlock(&s->mu); + return l; +} + +/* Get all addresses assigned to network interfaces on the machine and create a + listener for each. requested_port is the port to use for every listener, or 0 + to select one random port that will be used for every listener. Set *out_port + to the port selected. Return GRPC_ERROR_NONE only if all listeners were + added. */ +static grpc_error *add_all_local_addrs_to_server(grpc_tcp_server *s, + unsigned port_index, + int requested_port, + int *out_port) { + struct ifaddrs *ifa = NULL; + struct ifaddrs *ifa_it; + unsigned fd_index = 0; + grpc_tcp_listener *sp = NULL; + grpc_error *err = GRPC_ERROR_NONE; + if (requested_port == 0) { + /* Note: There could be a race where some local addrs can listen on the + selected port and some can't. The sane way to handle this would be to + retry by recreating the whole grpc_tcp_server. Backing out individual + listeners and orphaning the FDs looks like too much trouble. */ + if ((err = get_unused_port(&requested_port)) != GRPC_ERROR_NONE) { + return err; + } else if (requested_port <= 0) { + return GRPC_ERROR_CREATE("Bad get_unused_port()"); + } + gpr_log(GPR_DEBUG, "Picked unused port %d", requested_port); + } + if (getifaddrs(&ifa) != 0 || ifa == NULL) { + return GRPC_OS_ERROR(errno, "getifaddrs"); + } + for (ifa_it = ifa; ifa_it != NULL; ifa_it = ifa_it->ifa_next) { + grpc_resolved_address addr; + char *addr_str = NULL; + grpc_dualstack_mode dsmode; + grpc_tcp_listener *new_sp = NULL; + const char *ifa_name = (ifa_it->ifa_name ? ifa_it->ifa_name : ""); + if (ifa_it->ifa_addr == NULL) { + continue; + } else if (ifa_it->ifa_addr->sa_family == AF_INET) { + addr.len = sizeof(struct sockaddr_in); + } else if (ifa_it->ifa_addr->sa_family == AF_INET6) { + addr.len = sizeof(struct sockaddr_in6); + } else { + continue; + } + memcpy(addr.addr, ifa_it->ifa_addr, addr.len); + if (!grpc_sockaddr_set_port(&addr, requested_port)) { + /* Should never happen, because we check sa_family above. */ + err = GRPC_ERROR_CREATE("Failed to set port"); + break; + } + if (grpc_sockaddr_to_string(&addr_str, &addr, 0) < 0) { + addr_str = gpr_strdup(""); + } + gpr_log(GPR_DEBUG, + "Adding local addr from interface %s flags 0x%x to server: %s", + ifa_name, ifa_it->ifa_flags, addr_str); + /* We could have multiple interfaces with the same address (e.g., bonding), + so look for duplicates. */ + if (find_listener_with_addr(s, &addr) != NULL) { + gpr_log(GPR_DEBUG, "Skipping duplicate addr %s on interface %s", addr_str, + ifa_name); + gpr_free(addr_str); + continue; + } + if ((err = add_addr_to_server(s, &addr, port_index, fd_index, &dsmode, + &new_sp)) != GRPC_ERROR_NONE) { + char *err_str = NULL; + grpc_error *root_err; + if (gpr_asprintf(&err_str, "Failed to add listener: %s", addr_str) < 0) { + err_str = gpr_strdup("Failed to add listener"); + } + root_err = GRPC_ERROR_CREATE(err_str); + gpr_free(err_str); + gpr_free(addr_str); + err = grpc_error_add_child(root_err, err); + break; + } else { + GPR_ASSERT(requested_port == new_sp->port); + ++fd_index; + if (sp != NULL) { + new_sp->is_sibling = 1; + sp->sibling = new_sp; + } + sp = new_sp; + } + gpr_free(addr_str); + } + freeifaddrs(ifa); + if (err != GRPC_ERROR_NONE) { + return err; + } else if (sp == NULL) { + return GRPC_ERROR_CREATE("No local addresses"); + } else { + *out_port = sp->port; + return GRPC_ERROR_NONE; + } +} + +/* Treat :: or 0.0.0.0 as a family-agnostic wildcard. */ +static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s, + unsigned port_index, + int requested_port, + int *out_port) { + grpc_resolved_address wild4; + grpc_resolved_address wild6; + unsigned fd_index = 0; + grpc_dualstack_mode dsmode; + grpc_tcp_listener *sp = NULL; + grpc_tcp_listener *sp2 = NULL; + grpc_error *v6_err = GRPC_ERROR_NONE; + grpc_error *v4_err = GRPC_ERROR_NONE; + *out_port = -1; + if (s->expand_wildcard_addrs) { + return add_all_local_addrs_to_server(s, port_index, requested_port, + out_port); + } + grpc_sockaddr_make_wildcards(requested_port, &wild4, &wild6); + /* Try listening on IPv6 first. */ + if ((v6_err = add_addr_to_server(s, &wild6, port_index, fd_index, &dsmode, + &sp)) == GRPC_ERROR_NONE) { + ++fd_index; + requested_port = *out_port = sp->port; + if (dsmode == GRPC_DSMODE_DUALSTACK || dsmode == GRPC_DSMODE_IPV4) { + return GRPC_ERROR_NONE; + } + } + /* If we got a v6-only socket or nothing, try adding 0.0.0.0. */ + grpc_sockaddr_set_port(&wild4, requested_port); + if ((v4_err = add_addr_to_server(s, &wild4, port_index, fd_index, &dsmode, + &sp2)) == GRPC_ERROR_NONE) { + *out_port = sp2->port; + if (sp != NULL) { + sp2->is_sibling = 1; + sp->sibling = sp2; + } + } + if (*out_port > 0) { + GRPC_LOG_IF_ERROR("Failed to add :: listener", v6_err); + GRPC_LOG_IF_ERROR("Failed to add 0.0.0.0 listener", v4_err); + return GRPC_ERROR_NONE; + } else { + grpc_error *root_err = + GRPC_ERROR_CREATE("Failed to add any wildcard listeners"); + GPR_ASSERT(v6_err != GRPC_ERROR_NONE && v4_err != GRPC_ERROR_NONE); + root_err = grpc_error_add_child(root_err, v6_err); + root_err = grpc_error_add_child(root_err, v4_err); + return root_err; + } +} + static grpc_error *clone_port(grpc_tcp_listener *listener, unsigned count) { grpc_tcp_listener *sp = NULL; char *addr_str; @@ -559,19 +787,13 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s, const grpc_resolved_address *addr, int *out_port) { grpc_tcp_listener *sp; - grpc_tcp_listener *sp2 = NULL; - int fd; - grpc_dualstack_mode dsmode; - grpc_resolved_address addr6_v4mapped; - grpc_resolved_address wild4; - grpc_resolved_address wild6; - grpc_resolved_address addr4_copy; - grpc_resolved_address *allocated_addr = NULL; grpc_resolved_address sockname_temp; - int port; + grpc_resolved_address addr6_v4mapped; + int requested_port = grpc_sockaddr_get_port(addr); unsigned port_index = 0; - unsigned fd_index = 0; - grpc_error *errs[2] = {GRPC_ERROR_NONE, GRPC_ERROR_NONE}; + grpc_dualstack_mode dsmode; + grpc_error *err; + *out_port = -1; if (s->tail != NULL) { port_index = s->tail->port_index + 1; } @@ -579,85 +801,34 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s, /* Check if this is a wildcard port, and if so, try to keep the port the same as some previously created listener. */ - if (grpc_sockaddr_get_port(addr) == 0) { + if (requested_port == 0) { for (sp = s->head; sp; sp = sp->next) { sockname_temp.len = sizeof(struct sockaddr_storage); - if (0 == getsockname(sp->fd, (struct sockaddr *)sockname_temp.addr, + if (0 == getsockname(sp->fd, (struct sockaddr *)&sockname_temp.addr, (socklen_t *)&sockname_temp.len)) { - port = grpc_sockaddr_get_port(&sockname_temp); - if (port > 0) { - allocated_addr = gpr_malloc(sizeof(grpc_resolved_address)); - memcpy(allocated_addr, addr, addr->len); - grpc_sockaddr_set_port(allocated_addr, port); - addr = allocated_addr; + int used_port = grpc_sockaddr_get_port(&sockname_temp); + if (used_port > 0) { + memcpy(&sockname_temp, addr, sizeof(grpc_resolved_address)); + grpc_sockaddr_set_port(&sockname_temp, used_port); + requested_port = used_port; + addr = &sockname_temp; break; } } } } - - sp = NULL; - + if (grpc_sockaddr_is_wildcard(addr, &requested_port)) { + return add_wildcard_addrs_to_server(s, port_index, requested_port, + out_port); + } if (grpc_sockaddr_to_v4mapped(addr, &addr6_v4mapped)) { addr = &addr6_v4mapped; } - - /* Treat :: or 0.0.0.0 as a family-agnostic wildcard. */ - if (grpc_sockaddr_is_wildcard(addr, &port)) { - grpc_sockaddr_make_wildcards(port, &wild4, &wild6); - - /* Try listening on IPv6 first. */ - addr = &wild6; - errs[0] = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode, &fd); - if (errs[0] == GRPC_ERROR_NONE) { - errs[0] = add_socket_to_server(s, fd, addr, port_index, fd_index, &sp); - if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) { - goto done; - } - if (sp != NULL) { - ++fd_index; - } - /* If we didn't get a dualstack socket, also listen on 0.0.0.0. */ - if (port == 0 && sp != NULL) { - grpc_sockaddr_set_port(&wild4, sp->port); - } - } - addr = &wild4; - } - - errs[1] = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode, &fd); - if (errs[1] == GRPC_ERROR_NONE) { - if (dsmode == GRPC_DSMODE_IPV4 && - grpc_sockaddr_is_v4mapped(addr, &addr4_copy)) { - addr = &addr4_copy; - } - sp2 = sp; - errs[1] = add_socket_to_server(s, fd, addr, port_index, fd_index, &sp); - if (sp2 != NULL && sp != NULL) { - sp2->sibling = sp; - sp->is_sibling = 1; - } - } - -done: - gpr_free(allocated_addr); - if (sp != NULL) { + if ((err = add_addr_to_server(s, addr, port_index, 0, &dsmode, &sp)) == + GRPC_ERROR_NONE) { *out_port = sp->port; - GRPC_ERROR_UNREF(errs[0]); - GRPC_ERROR_UNREF(errs[1]); - return GRPC_ERROR_NONE; - } else { - *out_port = -1; - char *addr_str = grpc_sockaddr_to_uri(addr); - grpc_error *err = grpc_error_set_str( - GRPC_ERROR_CREATE_REFERENCING("Failed to add port to server", errs, - GPR_ARRAY_SIZE(errs)), - GRPC_ERROR_STR_TARGET_ADDRESS, addr_str); - GRPC_ERROR_UNREF(errs[0]); - GRPC_ERROR_UNREF(errs[1]); - gpr_free(addr_str); - return err; } + return err; } /* Return listener at port_index or NULL. Should only be called with s->mu diff --git a/test/core/handshake/client_ssl.c b/test/core/handshake/client_ssl.c index a22133e629f..1a06fd62558 100644 --- a/test/core/handshake/client_ssl.c +++ b/test/core/handshake/client_ssl.c @@ -59,13 +59,17 @@ typedef struct { char *alpn_preferred; } server_args; -// From https://wiki.openssl.org/index.php/Simple_TLS_Server. -static int create_socket(int port) { +// Based on https://wiki.openssl.org/index.php/Simple_TLS_Server. +// Pick an arbitrary unused port and return it in *out_port. Return +// an fd>=0 on success. +static int create_socket(int *out_port) { int s; struct sockaddr_in addr; + socklen_t addr_len; + *out_port = -1; addr.sin_family = AF_INET; - addr.sin_port = htons((uint16_t)port); + addr.sin_port = 0; addr.sin_addr.s_addr = htonl(INADDR_ANY); s = socket(AF_INET, SOCK_STREAM, 0); @@ -76,7 +80,7 @@ static int create_socket(int port) { if (bind(s, (struct sockaddr *)&addr, sizeof(addr)) < 0) { perror("Unable to bind"); - gpr_log(GPR_ERROR, "Unable to bind to %d", port); + gpr_log(GPR_ERROR, "%s", "Unable to bind to any port"); close(s); return -1; } @@ -87,6 +91,16 @@ static int create_socket(int port) { return -1; } + addr_len = sizeof(addr); + if (getsockname(s, (struct sockaddr *)&addr, &addr_len) != 0 || + addr_len > sizeof(addr)) { + perror("getsockname"); + gpr_log(GPR_ERROR, "%s", "Unable to get socket local address"); + close(s); + return -1; + } + + *out_port = ntohs(addr.sin_port); return s; } @@ -215,13 +229,12 @@ static bool client_ssl_test(char *server_alpn_preferred) { int server_socket = -1; int socket_retries = 30; while (server_socket == -1 && socket_retries-- > 0) { - port = grpc_pick_unused_port_or_die(); - server_socket = create_socket(port); + server_socket = create_socket(&port); if (server_socket == -1) { sleep(1); } } - GPR_ASSERT(server_socket > 0); + GPR_ASSERT(server_socket > 0 && port > 0); // Launch the TLS server thread. gpr_thd_options thdopt = gpr_thd_options_default(); diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 7122cd1e869..374e63f748d 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -39,9 +39,12 @@ #include "src/core/lib/iomgr/tcp_server.h" #include +#include #include +#include #include #include +#include #include #include @@ -50,6 +53,7 @@ #include #include +#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/iomgr/resolve_address.h" #include "src/core/lib/iomgr/sockaddr_utils.h" @@ -62,7 +66,7 @@ static gpr_mu *g_mu; static grpc_pollset *g_pollset; static int g_nconnects = 0; -typedef struct on_connect_result { +typedef struct { /* Owns a ref to server. */ grpc_tcp_server *server; unsigned port_index; @@ -70,15 +74,41 @@ typedef struct on_connect_result { int server_fd; } on_connect_result; -typedef struct server_weak_ref { +typedef struct { grpc_tcp_server *server; /* arg is this server_weak_ref. */ grpc_closure server_shutdown; } server_weak_ref; +#define MAX_URI 1024 +typedef struct { + grpc_resolved_address addr; + char str[MAX_URI]; +} test_addr; + +#define MAX_ADDRS 100 +typedef struct { + size_t naddrs; + test_addr addrs[MAX_ADDRS]; +} test_addrs; + static on_connect_result g_result = {NULL, 0, 0, -1}; +static char family_name_buf[1024]; +static const char *sock_family_name(int family) { + if (family == AF_INET) { + return "AF_INET"; + } else if (family == AF_INET6) { + return "AF_INET6"; + } else if (family == AF_UNSPEC) { + return "AF_UNSPEC"; + } else { + sprintf(family_name_buf, "%d", family); + return family_name_buf; + } +} + static void on_connect_result_init(on_connect_result *result) { result->server = NULL; result->port_index = 0; @@ -118,6 +148,18 @@ static void server_weak_ref_set(server_weak_ref *weak_ref, weak_ref->server = server; } +static void test_addr_init_str(test_addr *addr) { + char *str = NULL; + if (grpc_sockaddr_to_string(&str, &addr->addr, 0) != -1) { + size_t str_len; + memcpy(addr->str, str, (str_len = strnlen(str, sizeof(addr->str) - 1))); + addr->str[str_len] = '\0'; + gpr_free(str); + } else { + addr->str[0] = '\0'; + } +} + static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_pollset *pollset, grpc_tcp_server_acceptor *acceptor) { @@ -168,7 +210,7 @@ static void test_no_op_with_port(void) { memset(&resolved_addr, 0, sizeof(resolved_addr)); resolved_addr.len = sizeof(struct sockaddr_in); addr->sin_family = AF_INET; - int port; + int port = -1; GPR_ASSERT(grpc_tcp_server_add_port(s, &resolved_addr, &port) == GRPC_ERROR_NONE && port > 0); @@ -185,7 +227,7 @@ static void test_no_op_with_port_and_start(void) { GPR_ASSERT(GRPC_ERROR_NONE == grpc_tcp_server_create(&exec_ctx, NULL, NULL, &s)); LOG_TEST("test_no_op_with_port_and_start"); - int port; + int port = -1; memset(&resolved_addr, 0, sizeof(resolved_addr)); resolved_addr.len = sizeof(struct sockaddr_in); @@ -200,74 +242,115 @@ static void test_no_op_with_port_and_start(void) { grpc_exec_ctx_finish(&exec_ctx); } -static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, - socklen_t remote_len, on_connect_result *result) { +static grpc_error *tcp_connect(grpc_exec_ctx *exec_ctx, const test_addr *remote, + on_connect_result *result) { gpr_timespec deadline = grpc_timeout_seconds_to_deadline(10); - int clifd = socket(remote->sa_family, SOCK_STREAM, 0); + int clifd; int nconnects_before; + const struct sockaddr *remote_addr = + (const struct sockaddr *)remote->addr.addr; + gpr_log(GPR_INFO, "Connecting to %s", remote->str); gpr_mu_lock(g_mu); nconnects_before = g_nconnects; on_connect_result_init(&g_result); - GPR_ASSERT(clifd >= 0); - gpr_log(GPR_DEBUG, "start connect"); - GPR_ASSERT(connect(clifd, remote, remote_len) == 0); + clifd = socket(remote_addr->sa_family, SOCK_STREAM, 0); + if (clifd < 0) { + gpr_mu_unlock(g_mu); + return GRPC_OS_ERROR(errno, "Failed to create socket"); + } + gpr_log(GPR_DEBUG, "start connect to %s", remote->str); + if (connect(clifd, remote_addr, (socklen_t)remote->addr.len) != 0) { + gpr_mu_unlock(g_mu); + close(clifd); + return GRPC_OS_ERROR(errno, "connect"); + } gpr_log(GPR_DEBUG, "wait"); while (g_nconnects == nconnects_before && gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) { grpc_pollset_worker *worker = NULL; - GPR_ASSERT(GRPC_LOG_IF_ERROR( - "pollset_work", - grpc_pollset_work(exec_ctx, g_pollset, &worker, - gpr_now(GPR_CLOCK_MONOTONIC), deadline))); + grpc_error *err; + if ((err = grpc_pollset_work(exec_ctx, g_pollset, &worker, + gpr_now(GPR_CLOCK_MONOTONIC), deadline)) != + GRPC_ERROR_NONE) { + gpr_mu_unlock(g_mu); + close(clifd); + return err; + } gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(exec_ctx); gpr_mu_lock(g_mu); } gpr_log(GPR_DEBUG, "wait done"); - GPR_ASSERT(g_nconnects == nconnects_before + 1); + if (g_nconnects != nconnects_before + 1) { + gpr_mu_unlock(g_mu); + close(clifd); + return GRPC_ERROR_CREATE("Didn't connect"); + } close(clifd); *result = g_result; gpr_mu_unlock(g_mu); + gpr_log(GPR_INFO, "Result (%d, %d) fd %d", result->port_index, + result->fd_index, result->server_fd); + grpc_tcp_server_unref(exec_ctx, result->server); + return GRPC_ERROR_NONE; } -/* Tests a tcp server with multiple ports. TODO(daniel-j-born): Multiple fds for - the same port should be tested. */ -static void test_connect(unsigned n) { +/* Tests a tcp server on "::" listeners with multiple ports. If channel_args is + non-NULL, pass them to the server. If dst_addrs is non-NULL, use valid addrs + as destination addrs (port is not set). If dst_addrs is NULL, use listener + addrs as destination addrs. If test_dst_addrs is true, test connectivity with + each destination address, set grpc_resolved_address::len=0 for failures, but + don't fail the overall unitest. */ +static void test_connect(size_t num_connects, + const grpc_channel_args *channel_args, + test_addrs *dst_addrs, bool test_dst_addrs) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_resolved_address resolved_addr; grpc_resolved_address resolved_addr1; - struct sockaddr_storage *addr = (struct sockaddr_storage *)resolved_addr.addr; - struct sockaddr_storage *addr1 = + struct sockaddr_storage *const addr = + (struct sockaddr_storage *)resolved_addr.addr; + struct sockaddr_storage *const addr1 = (struct sockaddr_storage *)resolved_addr1.addr; unsigned svr_fd_count; + int port; int svr_port; unsigned svr1_fd_count; int svr1_port; grpc_tcp_server *s; + const unsigned num_ports = 2; GPR_ASSERT(GRPC_ERROR_NONE == - grpc_tcp_server_create(&exec_ctx, NULL, NULL, &s)); - unsigned i; + grpc_tcp_server_create(&exec_ctx, NULL, channel_args, &s)); + unsigned port_num; server_weak_ref weak_ref; server_weak_ref_init(&weak_ref); + server_weak_ref_set(&weak_ref, s); LOG_TEST("test_connect"); - gpr_log(GPR_INFO, "clients=%d", n); + gpr_log(GPR_INFO, + "clients=%lu, num chan args=%lu, remote IP=%s, test_dst_addrs=%d", + (unsigned long)num_connects, + (unsigned long)(channel_args != NULL ? channel_args->num_args : 0), + dst_addrs != NULL ? "" : "::", test_dst_addrs); memset(&resolved_addr, 0, sizeof(resolved_addr)); memset(&resolved_addr1, 0, sizeof(resolved_addr1)); resolved_addr.len = sizeof(struct sockaddr_storage); resolved_addr1.len = sizeof(struct sockaddr_storage); addr->ss_family = addr1->ss_family = AF_INET; - GPR_ASSERT(GRPC_ERROR_NONE == - grpc_tcp_server_add_port(s, &resolved_addr, &svr_port)); + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "grpc_tcp_server_add_port", + grpc_tcp_server_add_port(s, &resolved_addr, &svr_port))); + gpr_log(GPR_INFO, "Allocated port %d", svr_port); GPR_ASSERT(svr_port > 0); /* Cannot use wildcard (port==0), because add_port() will try to reuse the same port as a previous add_port(). */ svr1_port = grpc_pick_unused_port_or_die(); + GPR_ASSERT(svr1_port > 0); + gpr_log(GPR_INFO, "Picked unused port %d", svr1_port); grpc_sockaddr_set_port(&resolved_addr1, svr1_port); - GPR_ASSERT(grpc_tcp_server_add_port(s, &resolved_addr1, &svr_port) == + GPR_ASSERT(grpc_tcp_server_add_port(s, &resolved_addr1, &port) == GRPC_ERROR_NONE && - svr_port == svr1_port); + port == svr1_port); /* Bad port_index. */ GPR_ASSERT(grpc_tcp_server_port_fd_count(s, 2) == 0); @@ -283,58 +366,71 @@ static void test_connect(unsigned n) { svr1_fd_count = grpc_tcp_server_port_fd_count(s, 1); GPR_ASSERT(svr1_fd_count >= 1); - for (i = 0; i < svr_fd_count; ++i) { - int fd = grpc_tcp_server_port_fd(s, 0, i); - GPR_ASSERT(fd >= 0); - if (i == 0) { - GPR_ASSERT(getsockname(fd, (struct sockaddr *)addr, - (socklen_t *)&resolved_addr.len) == 0); - GPR_ASSERT(resolved_addr.len <= sizeof(*addr)); - } - } - for (i = 0; i < svr1_fd_count; ++i) { - int fd = grpc_tcp_server_port_fd(s, 1, i); - GPR_ASSERT(fd >= 0); - if (i == 0) { - GPR_ASSERT(getsockname(fd, (struct sockaddr *)addr1, - (socklen_t *)&resolved_addr1.len) == 0); - GPR_ASSERT(resolved_addr1.len <= sizeof(*addr1)); - } - } - grpc_tcp_server_start(&exec_ctx, s, &g_pollset, 1, on_connect, NULL); - for (i = 0; i < n; i++) { - on_connect_result result; - int svr_fd; - on_connect_result_init(&result); - tcp_connect(&exec_ctx, (struct sockaddr *)addr, - (socklen_t)resolved_addr.len, &result); - GPR_ASSERT(result.server_fd >= 0); - svr_fd = result.server_fd; - GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) == - result.server_fd); - GPR_ASSERT(result.port_index == 0); - GPR_ASSERT(result.fd_index < svr_fd_count); - GPR_ASSERT(result.server == s); - if (weak_ref.server == NULL) { - server_weak_ref_set(&weak_ref, result.server); + if (dst_addrs != NULL) { + int ports[] = {svr_port, svr1_port}; + for (port_num = 0; port_num < num_ports; ++port_num) { + size_t dst_idx; + size_t num_tested = 0; + for (dst_idx = 0; dst_idx < dst_addrs->naddrs; ++dst_idx) { + test_addr dst = dst_addrs->addrs[dst_idx]; + on_connect_result result; + grpc_error *err; + if (dst.addr.len == 0) { + gpr_log(GPR_DEBUG, "Skipping test of non-functional local IP %s", + dst.str); + continue; + } + GPR_ASSERT(grpc_sockaddr_set_port(&dst.addr, ports[port_num])); + test_addr_init_str(&dst); + ++num_tested; + on_connect_result_init(&result); + if ((err = tcp_connect(&exec_ctx, &dst, &result)) == GRPC_ERROR_NONE && + result.server_fd >= 0 && result.server == s) { + continue; + } + const char *err_str = grpc_error_string(err); + gpr_log(GPR_ERROR, "Failed to connect to %s: %s", dst.str, err_str); + grpc_error_free_string(err_str); + GPR_ASSERT(test_dst_addrs); + dst_addrs->addrs[dst_idx].addr.len = 0; + GRPC_ERROR_UNREF(err); + } + GPR_ASSERT(num_tested > 0); + } + } else { + for (port_num = 0; port_num < num_ports; ++port_num) { + const unsigned num_fds = grpc_tcp_server_port_fd_count(s, port_num); + unsigned fd_num; + for (fd_num = 0; fd_num < num_fds; ++fd_num) { + int fd = grpc_tcp_server_port_fd(s, port_num, fd_num); + size_t connect_num; + test_addr dst; + GPR_ASSERT(fd >= 0); + dst.addr.len = sizeof(dst.addr.addr); + GPR_ASSERT(getsockname(fd, (struct sockaddr *)dst.addr.addr, + (socklen_t *)&dst.addr.len) == 0); + GPR_ASSERT(dst.addr.len <= sizeof(dst.addr.addr)); + test_addr_init_str(&dst); + gpr_log(GPR_INFO, "(%d, %d) fd %d family %s listening on %s", port_num, + fd_num, fd, sock_family_name(addr->ss_family), dst.str); + for (connect_num = 0; connect_num < num_connects; ++connect_num) { + on_connect_result result; + on_connect_result_init(&result); + GPR_ASSERT(GRPC_LOG_IF_ERROR("tcp_connect", + tcp_connect(&exec_ctx, &dst, &result))); + GPR_ASSERT(result.server_fd == fd); + GPR_ASSERT(result.port_index == port_num); + GPR_ASSERT(result.fd_index == fd_num); + GPR_ASSERT(result.server == s); + GPR_ASSERT( + grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) == + result.server_fd); + } + } } - grpc_tcp_server_unref(&exec_ctx, result.server); - - on_connect_result_init(&result); - tcp_connect(&exec_ctx, (struct sockaddr *)addr1, - (socklen_t)resolved_addr1.len, &result); - GPR_ASSERT(result.server_fd >= 0); - GPR_ASSERT(result.server_fd != svr_fd); - GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) == - result.server_fd); - GPR_ASSERT(result.port_index == 1); - GPR_ASSERT(result.fd_index < svr_fd_count); - GPR_ASSERT(result.server == s); - grpc_tcp_server_unref(&exec_ctx, result.server); } - /* Weak ref to server valid until final unref. */ GPR_ASSERT(weak_ref.server != NULL); GPR_ASSERT(grpc_tcp_server_port_fd(s, 0, 0) >= 0); @@ -354,6 +450,12 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int main(int argc, char **argv) { grpc_closure destroyed; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_arg chan_args[] = { + {GRPC_ARG_INTEGER, GRPC_ARG_EXPAND_WILDCARD_ADDRS, {.integer = 1}}}; + const grpc_channel_args channel_args = {1, chan_args}; + struct ifaddrs *ifa = NULL; + struct ifaddrs *ifa_it; + test_addrs dst_addrs; grpc_test_init(argc, argv); grpc_init(); g_pollset = gpr_malloc(grpc_pollset_size()); @@ -363,8 +465,45 @@ int main(int argc, char **argv) { test_no_op_with_start(); test_no_op_with_port(); test_no_op_with_port_and_start(); - test_connect(1); - test_connect(10); + + if (getifaddrs(&ifa) != 0 || ifa == NULL) { + gpr_log(GPR_ERROR, "getifaddrs: %s", strerror(errno)); + return EXIT_FAILURE; + } + dst_addrs.naddrs = 0; + for (ifa_it = ifa; ifa_it != NULL && dst_addrs.naddrs < MAX_ADDRS; + ifa_it = ifa_it->ifa_next) { + if (ifa_it->ifa_addr == NULL) { + continue; + } else if (ifa_it->ifa_addr->sa_family == AF_INET) { + dst_addrs.addrs[dst_addrs.naddrs].addr.len = sizeof(struct sockaddr_in); + } else if (ifa_it->ifa_addr->sa_family == AF_INET6) { + dst_addrs.addrs[dst_addrs.naddrs].addr.len = sizeof(struct sockaddr_in6); + } else { + continue; + } + memcpy(dst_addrs.addrs[dst_addrs.naddrs].addr.addr, ifa_it->ifa_addr, + dst_addrs.addrs[dst_addrs.naddrs].addr.len); + GPR_ASSERT( + grpc_sockaddr_set_port(&dst_addrs.addrs[dst_addrs.naddrs].addr, 0)); + test_addr_init_str(&dst_addrs.addrs[dst_addrs.naddrs]); + ++dst_addrs.naddrs; + } + freeifaddrs(ifa); + ifa = NULL; + + /* Connect to same addresses as listeners. */ + test_connect(1, NULL, NULL, false); + test_connect(10, NULL, NULL, false); + + /* Set dst_addrs.addrs[i].len=0 for dst_addrs that are unreachable with a "::" + listener. */ + test_connect(1, NULL, &dst_addrs, true); + + /* Test connect(2) with dst_addrs. */ + test_connect(1, &channel_args, &dst_addrs, false); + /* Test connect(2) with dst_addrs. */ + test_connect(10, &channel_args, &dst_addrs, false); grpc_closure_init(&destroyed, destroy_pollset, g_pollset, grpc_schedule_on_exec_ctx); @@ -372,7 +511,7 @@ int main(int argc, char **argv) { grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); gpr_free(g_pollset); - return 0; + return EXIT_SUCCESS; } #else /* GRPC_POSIX_SOCKET */ From 3158f7bb47a4f04d17050d00951cd140799278fc Mon Sep 17 00:00:00 2001 From: Dan Born Date: Thu, 9 Feb 2017 20:32:02 -0800 Subject: [PATCH 103/167] API changes --- src/core/lib/iomgr/tcp_server_posix.c | 2 +- test/core/iomgr/tcp_server_posix_test.c | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index 786e6189d72..36f878fdd4d 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -191,7 +191,7 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, if (args->args[i].type == GRPC_ARG_INTEGER) { s->expand_wildcard_addrs = (args->args[i].value.integer != 0); } else { - grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota); + grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); gpr_free(s); return GRPC_ERROR_CREATE(GRPC_ARG_EXPAND_WILDCARD_ADDRS " must be an integer"); diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 374e63f748d..4a6570015f2 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -390,9 +390,8 @@ static void test_connect(size_t num_connects, result.server_fd >= 0 && result.server == s) { continue; } - const char *err_str = grpc_error_string(err); - gpr_log(GPR_ERROR, "Failed to connect to %s: %s", dst.str, err_str); - grpc_error_free_string(err_str); + gpr_log(GPR_ERROR, "Failed to connect to %s: %s", dst.str, + grpc_error_string(err)); GPR_ASSERT(test_dst_addrs); dst_addrs->addrs[dst_idx].addr.len = 0; GRPC_ERROR_UNREF(err); From c15d25a9dbc44c26b0755bb5078920db57c8afd7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 21:22:22 -0800 Subject: [PATCH 104/167] Fix substitution --- src/core/lib/channel/http_server_filter.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c index b475dd6b86d..bb185351a8e 100644 --- a/src/core/lib/channel/http_server_filter.c +++ b/src/core/lib/channel/http_server_filter.c @@ -199,13 +199,16 @@ static grpc_error *server_filter_incoming_metadata(grpc_exec_ctx *exec_ctx, } if (b->idx.named.host != NULL && b->idx.named.authority == NULL) { + grpc_linked_mdelem *el = b->idx.named.host; + grpc_mdelem md = GRPC_MDELEM_REF(el->md); + grpc_metadata_batch_remove(exec_ctx, b, el); add_error( error_name, &error, grpc_metadata_batch_add_head( - exec_ctx, b, b->idx.named.host, - grpc_mdelem_from_slices( - exec_ctx, GRPC_MDSTR_AUTHORITY, - grpc_slice_ref_internal(GRPC_MDVALUE(b->idx.named.host->md))))); + exec_ctx, b, el, grpc_mdelem_from_slices( + exec_ctx, GRPC_MDSTR_AUTHORITY, + grpc_slice_ref_internal(GRPC_MDVALUE(md))))); + GRPC_MDELEM_UNREF(exec_ctx, md); } if (b->idx.named.authority == NULL) { From f1021678e4de35babec7b09e64a3f1ce2750a841 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 21:29:50 -0800 Subject: [PATCH 105/167] Merge fixup --- src/core/ext/client_channel/client_channel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 907d81866dc..43f7e619cfc 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -584,7 +584,7 @@ static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx, } grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker); grpc_pollset_set_destroy(chand->interested_parties); - grpc_combiner_destroy(exec_ctx, chand->combiner); + GRPC_COMBINER_UNREF(exec_ctx, chand->combiner, "client_channel"); gpr_mu_destroy(&chand->info_mu); } From 07e8fb1775b51d178cb46f81da8518d190c4a066 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 21:36:24 -0800 Subject: [PATCH 106/167] Eliminate CQ freelist --- src/core/lib/iomgr/ev_epoll_linux.c | 11 ------ src/core/lib/iomgr/ev_poll_posix.c | 11 ------ src/core/lib/iomgr/ev_posix.c | 4 --- src/core/lib/iomgr/ev_posix.h | 1 - src/core/lib/iomgr/pollset.h | 3 -- src/core/lib/iomgr/pollset_uv.c | 5 --- src/core/lib/iomgr/pollset_windows.c | 10 ------ src/core/lib/surface/completion_queue.c | 46 +++++-------------------- src/core/lib/surface/completion_queue.h | 3 -- src/core/lib/surface/init.c | 2 -- test/core/internal_api_canaries/iomgr.c | 1 - 11 files changed, 9 insertions(+), 88 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 51842fc2081..fc56843128e 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1405,16 +1405,6 @@ static void pollset_destroy(grpc_pollset *pollset) { gpr_mu_destroy(&pollset->po.mu); } -static void pollset_reset(grpc_pollset *pollset) { - GPR_ASSERT(pollset->shutting_down); - GPR_ASSERT(!pollset_has_workers(pollset)); - pollset->shutting_down = false; - pollset->finish_shutdown_called = false; - pollset->kicked_without_pollers = false; - pollset->shutdown_done = NULL; - GPR_ASSERT(pollset->po.pi == NULL); -} - static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, polling_island *pi) { if (gpr_mu_trylock(&pi->workqueue_read_mu)) { @@ -1958,7 +1948,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_init = pollset_init, .pollset_shutdown = pollset_shutdown, - .pollset_reset = pollset_reset, .pollset_destroy = pollset_destroy, .pollset_work = pollset_work, .pollset_kick = pollset_kick, diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index ca129322194..21eb62753ef 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -815,16 +815,6 @@ static void pollset_destroy(grpc_pollset *pollset) { gpr_mu_destroy(&pollset->mu); } -static void pollset_reset(grpc_pollset *pollset) { - GPR_ASSERT(pollset->shutting_down); - GPR_ASSERT(!pollset_has_workers(pollset)); - GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); - GPR_ASSERT(pollset->fd_count == 0); - pollset->shutting_down = 0; - pollset->called_shutdown = 0; - pollset->kicked_without_pollers = 0; -} - static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { gpr_mu_lock(&pollset->mu); @@ -1514,7 +1504,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_init = pollset_init, .pollset_shutdown = pollset_shutdown, - .pollset_reset = pollset_reset, .pollset_destroy = pollset_destroy, .pollset_work = pollset_work, .pollset_kick = pollset_kick, diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 5bb55631d69..95b1d99d757 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -191,10 +191,6 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_event_engine->pollset_shutdown(exec_ctx, pollset, closure); } -void grpc_pollset_reset(grpc_pollset *pollset) { - g_event_engine->pollset_reset(pollset); -} - void grpc_pollset_destroy(grpc_pollset *pollset) { g_event_engine->pollset_destroy(pollset); } diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index a589efdeece..1aea7d61f32 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -64,7 +64,6 @@ typedef struct grpc_event_engine_vtable { void (*pollset_init)(grpc_pollset *pollset, gpr_mu **mu); void (*pollset_shutdown)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure); - void (*pollset_reset)(grpc_pollset *pollset); void (*pollset_destroy)(grpc_pollset *pollset); grpc_error *(*pollset_work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h index 8d9edc8406b..c4b62a27904 100644 --- a/src/core/lib/iomgr/pollset.h +++ b/src/core/lib/iomgr/pollset.h @@ -58,9 +58,6 @@ void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu); * pollset's mutex must be held */ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure); -/** Reset the pollset to its initial state (perhaps with some cached objects); - * must have been previously shutdown */ -void grpc_pollset_reset(grpc_pollset *pollset); void grpc_pollset_destroy(grpc_pollset *pollset); /* Do some work on a pollset. diff --git a/src/core/lib/iomgr/pollset_uv.c b/src/core/lib/iomgr/pollset_uv.c index ed3edeee949..5847771108a 100644 --- a/src/core/lib/iomgr/pollset_uv.c +++ b/src/core/lib/iomgr/pollset_uv.c @@ -97,11 +97,6 @@ void grpc_pollset_destroy(grpc_pollset *pollset) { } } -void grpc_pollset_reset(grpc_pollset *pollset) { - GPR_ASSERT(pollset->shutting_down); - pollset->shutting_down = 0; -} - static void timer_run_cb(uv_timer_t *timer) {} grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index 2a45e708df6..9be73a7ce8e 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -117,16 +117,6 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, void grpc_pollset_destroy(grpc_pollset *pollset) {} -void grpc_pollset_reset(grpc_pollset *pollset) { - GPR_ASSERT(pollset->shutting_down); - GPR_ASSERT( - !has_workers(&pollset->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET)); - pollset->shutting_down = 0; - pollset->is_iocp_worker = 0; - pollset->kicked_without_pollers = 0; - pollset->on_shutdown = NULL; -} - grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 1830842d00b..b80656aa8de 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -96,9 +96,6 @@ struct grpc_completion_queue { #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1)) #define CQ_FROM_POLLSET(ps) (((grpc_completion_queue *)ps) - 1) -static gpr_mu g_freelist_mu; -static grpc_completion_queue *g_freelist; - int grpc_cq_pluck_trace; int grpc_cq_event_timeout_trace; @@ -113,21 +110,6 @@ int grpc_cq_event_timeout_trace; static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc, grpc_error *error); -void grpc_cq_global_init(void) { gpr_mu_init(&g_freelist_mu); } - -void grpc_cq_global_shutdown(void) { - gpr_mu_destroy(&g_freelist_mu); - while (g_freelist) { - grpc_completion_queue *next = g_freelist->next_free; - grpc_pollset_destroy(POLLSET_FROM_CQ(g_freelist)); -#ifndef NDEBUG - gpr_free(g_freelist->outstanding_tags); -#endif - gpr_free(g_freelist); - g_freelist = next; - } -} - grpc_completion_queue *grpc_completion_queue_create(void *reserved) { grpc_completion_queue *cc; GPR_ASSERT(!reserved); @@ -136,22 +118,12 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) { GRPC_API_TRACE("grpc_completion_queue_create(reserved=%p)", 1, (reserved)); - gpr_mu_lock(&g_freelist_mu); - if (g_freelist == NULL) { - gpr_mu_unlock(&g_freelist_mu); - - cc = gpr_malloc(sizeof(grpc_completion_queue) + grpc_pollset_size()); - grpc_pollset_init(POLLSET_FROM_CQ(cc), &cc->mu); + cc = gpr_malloc(sizeof(grpc_completion_queue) + grpc_pollset_size()); + grpc_pollset_init(POLLSET_FROM_CQ(cc), &cc->mu); #ifndef NDEBUG - cc->outstanding_tags = NULL; - cc->outstanding_tag_capacity = 0; + cc->outstanding_tags = NULL; + cc->outstanding_tag_capacity = 0; #endif - } else { - cc = g_freelist; - g_freelist = g_freelist->next_free; - gpr_mu_unlock(&g_freelist_mu); - /* pollset already initialized */ - } /* Initial ref is dropped by grpc_completion_queue_shutdown */ gpr_ref_init(&cc->pending_events, 1); @@ -203,11 +175,11 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) { #endif if (gpr_unref(&cc->owning_refs)) { GPR_ASSERT(cc->completed_head.next == (uintptr_t)&cc->completed_head); - grpc_pollset_reset(POLLSET_FROM_CQ(cc)); - gpr_mu_lock(&g_freelist_mu); - cc->next_free = g_freelist; - g_freelist = cc; - gpr_mu_unlock(&g_freelist_mu); + grpc_pollset_destroy(POLLSET_FROM_CQ(cc)); +#ifndef NDEBUG + gpr_free(cc->outstanding_tags); +#endif + gpr_free(cc); } } diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h index c1cafba5f2c..5d73dd7216e 100644 --- a/src/core/lib/surface/completion_queue.h +++ b/src/core/lib/surface/completion_queue.h @@ -99,7 +99,4 @@ bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc); void grpc_cq_mark_server_cq(grpc_completion_queue *cc); int grpc_cq_is_server_cq(grpc_completion_queue *cc); -void grpc_cq_global_init(void); -void grpc_cq_global_shutdown(void); - #endif /* GRPC_CORE_LIB_SURFACE_COMPLETION_QUEUE_H */ diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c index b338ac4c486..91bd014a0e5 100644 --- a/src/core/lib/surface/init.c +++ b/src/core/lib/surface/init.c @@ -209,7 +209,6 @@ void grpc_init(void) { grpc_iomgr_init(); grpc_executor_init(); gpr_timers_global_init(); - grpc_cq_global_init(); grpc_handshaker_factory_registry_init(); grpc_security_init(); for (i = 0; i < g_number_of_plugins; i++) { @@ -236,7 +235,6 @@ void grpc_shutdown(void) { gpr_mu_lock(&g_init_mu); if (--g_initializations == 0) { grpc_executor_shutdown(&exec_ctx); - grpc_cq_global_shutdown(); grpc_iomgr_shutdown(&exec_ctx); gpr_timers_global_destroy(); grpc_tracer_shutdown(); diff --git a/test/core/internal_api_canaries/iomgr.c b/test/core/internal_api_canaries/iomgr.c index d73d5c175c9..6fdaf1f9806 100644 --- a/test/core/internal_api_canaries/iomgr.c +++ b/test/core/internal_api_canaries/iomgr.c @@ -105,7 +105,6 @@ static void test_code(void) { grpc_pollset_size(); grpc_pollset_init(NULL, NULL); grpc_pollset_shutdown(NULL, NULL, NULL); - grpc_pollset_reset(NULL); grpc_pollset_destroy(NULL); GRPC_ERROR_UNREF(grpc_pollset_work(NULL, NULL, NULL, gpr_now(GPR_CLOCK_REALTIME), From 557443ea83b91e4ee2210d708228f3b1ca3b2565 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Feb 2017 22:30:37 -0800 Subject: [PATCH 107/167] Avoid /tmp --- tools/run_tests/run_microbenchmark.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 096d7d78ab2..e0c62c81a8d 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -126,15 +126,15 @@ for bm_name in sys.argv[1:]: 'bins/mutrace/%s' % bm_name, '--benchmark_filter=^%s$' % line, '--benchmark_min_time=20']) - with open('/tmp/bm.perf', 'w') as f: + with open('bm.perf', 'w') as f: f.write(subprocess.check_output(['sudo', 'perf', 'script'])) - with open('/tmp/bm.folded', 'w') as f: + with open('bm.folded', 'w') as f: f.write(subprocess.check_output([ - '%s/stackcollapse-perf.pl' % flamegraph_dir, '/tmp/bm.perf'])) + '%s/stackcollapse-perf.pl' % flamegraph_dir, 'bm.perf'])) link(line, '%s.svg' % fnize(line)) with open('reports/%s.svg' % fnize(line), 'w') as f: f.write(subprocess.check_output([ - '%s/flamegraph.pl' % flamegraph_dir, '/tmp/bm.folded'])) + '%s/flamegraph.pl' % flamegraph_dir, 'bm.folded'])) index_html += "\n\n" with open('reports/index.html', 'w') as f: From 00cacbdd370717b49da2979600447ba3ca7fc541 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Feb 2017 10:51:30 -0800 Subject: [PATCH 108/167] add missing generated files --- Makefile | 2 ++ tools/doxygen/Doxyfile.c++ | 2 ++ tools/doxygen/Doxyfile.c++.internal | 2 ++ tools/run_tests/generated/sources_and_headers.json | 3 ++- vsprojects/vcxproj/grpc++/grpc++.vcxproj | 2 ++ vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters | 6 ++++++ 6 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0413cc0e009..54cb3b88d67 100644 --- a/Makefile +++ b/Makefile @@ -3939,6 +3939,8 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/sync_generic.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ + include/grpc++/impl/codegen/proto_utils.h \ + include/grpc++/impl/codegen/config_protobuf.h \ LIBGRPC++_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_SRC)))) diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index f215f1b5d1a..0706cdca0ff 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -810,12 +810,14 @@ include/grpc++/impl/codegen/client_unary_call.h \ include/grpc++/impl/codegen/completion_queue.h \ include/grpc++/impl/codegen/completion_queue_tag.h \ include/grpc++/impl/codegen/config.h \ +include/grpc++/impl/codegen/config_protobuf.h \ include/grpc++/impl/codegen/core_codegen.h \ include/grpc++/impl/codegen/core_codegen_interface.h \ include/grpc++/impl/codegen/create_auth_context.h \ include/grpc++/impl/codegen/grpc_library.h \ include/grpc++/impl/codegen/metadata_map.h \ include/grpc++/impl/codegen/method_handler_impl.h \ +include/grpc++/impl/codegen/proto_utils.h \ include/grpc++/impl/codegen/rpc_method.h \ include/grpc++/impl/codegen/rpc_service_method.h \ include/grpc++/impl/codegen/security/auth_context.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 19b4a9b02d4..cb0a3e55f8d 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -810,6 +810,7 @@ include/grpc++/impl/codegen/client_unary_call.h \ include/grpc++/impl/codegen/completion_queue.h \ include/grpc++/impl/codegen/completion_queue_tag.h \ include/grpc++/impl/codegen/config.h \ +include/grpc++/impl/codegen/config_protobuf.h \ include/grpc++/impl/codegen/core_codegen.h \ include/grpc++/impl/codegen/core_codegen.h \ include/grpc++/impl/codegen/core_codegen_interface.h \ @@ -817,6 +818,7 @@ include/grpc++/impl/codegen/create_auth_context.h \ include/grpc++/impl/codegen/grpc_library.h \ include/grpc++/impl/codegen/metadata_map.h \ include/grpc++/impl/codegen/method_handler_impl.h \ +include/grpc++/impl/codegen/proto_utils.h \ include/grpc++/impl/codegen/rpc_method.h \ include/grpc++/impl/codegen/rpc_service_method.h \ include/grpc++/impl/codegen/security/auth_context.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 52d8ae183b4..21f82a03ef3 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -5346,7 +5346,8 @@ "grpc", "grpc++_base", "grpc++_codegen_base", - "grpc++_codegen_base_src" + "grpc++_codegen_base_src", + "grpc++_codegen_proto" ], "headers": [ "include/grpc++/impl/codegen/core_codegen.h", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index fcd1d9def9a..58597744d8b 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -349,6 +349,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index dbf3ad90c62..4b501a1e74e 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -387,6 +387,12 @@ include\grpc\impl\codegen + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + From aead5aab010362c7dd6448acfd580dbae3966750 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Feb 2017 11:00:29 -0800 Subject: [PATCH 109/167] add grpc++_codegen_proto to bazelfile as well --- BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/BUILD b/BUILD index 58971ff8e35..5ef7bf8a5b9 100644 --- a/BUILD +++ b/BUILD @@ -140,6 +140,7 @@ grpc_cc_library( "grpc++_base", "grpc++_codegen_base", "grpc++_codegen_base_src", + "grpc++_codegen_proto", ], ) From dbf571ea07e20b3558a173f57e6ded7992120914 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Thu, 9 Feb 2017 14:10:03 -0800 Subject: [PATCH 110/167] Use unique namespace for each python protoc test Using the same namespace results in errors when using the c-extension version of protobuf due to name collisions --- .../protoc_plugin/_split_definitions_test.py | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py index bcc01f39787..db938e65459 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py @@ -48,6 +48,8 @@ from grpc_tools import protoc from tests.unit.framework.common import test_constants _MESSAGES_IMPORT = b'import "messages.proto";' +_SPLIT_NAMESPACE = b'package grpc_protoc_plugin.invocation_testing.split;' +_COMMON_NAMESPACE = b'package grpc_protoc_plugin.invocation_testing;' @contextlib.contextmanager @@ -163,7 +165,10 @@ class SameSeparateTest(unittest.TestCase, SeparateTestMixin): os.makedirs(self.grpc_python_out_directory) same_proto_file = os.path.join(self.proto_directory, 'same_separate.proto') - open(same_proto_file, 'wb').write(same_proto_contents) + open(same_proto_file, 'wb').write( + same_proto_contents.replace( + _COMMON_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.same_separate;')) protoc_result = protoc.main([ '', '--proto_path={}'.format(self.proto_directory), @@ -199,7 +204,11 @@ class SameCommonTest(unittest.TestCase, CommonTestMixin): os.makedirs(self.python_out_directory) same_proto_file = os.path.join(self.proto_directory, 'same_common.proto') - open(same_proto_file, 'wb').write(same_proto_contents) + open(same_proto_file, 'wb').write( + same_proto_contents.replace( + _COMMON_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.same_common;')) + protoc_result = protoc.main([ '', '--proto_path={}'.format(self.proto_directory), @@ -240,8 +249,14 @@ class SplitCommonTest(unittest.TestCase, CommonTestMixin): 'split_common_messages.proto') open(services_proto_file, 'wb').write( services_proto_contents.replace( - _MESSAGES_IMPORT, b'import "split_common_messages.proto";')) - open(messages_proto_file, 'wb').write(messages_proto_contents) + _MESSAGES_IMPORT, b'import "split_common_messages.proto";') + .replace( + _SPLIT_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.split_common;')) + open(messages_proto_file, 'wb').write( + messages_proto_contents.replace( + _SPLIT_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.split_common;')) protoc_result = protoc.main([ '', '--proto_path={}'.format(self.proto_directory), @@ -285,8 +300,16 @@ class SplitSeparateTest(unittest.TestCase, SeparateTestMixin): 'split_separate_messages.proto') open(services_proto_file, 'wb').write( services_proto_contents.replace( - _MESSAGES_IMPORT, b'import "split_separate_messages.proto";')) - open(messages_proto_file, 'wb').write(messages_proto_contents) + _MESSAGES_IMPORT, b'import "split_separate_messages.proto";') + .replace( + _SPLIT_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.split_separate;' + )) + open(messages_proto_file, 'wb').write( + messages_proto_contents.replace( + _SPLIT_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.split_separate;' + )) protoc_result = protoc.main([ '', '--proto_path={}'.format(self.proto_directory), From e7c60e62821bdf8cf980328a36499614458fd052 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Feb 2017 08:56:36 -0800 Subject: [PATCH 111/167] Make latency traces more useful --- test/cpp/microbenchmarks/bm_fullstack.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index 9d883e68d7f..549fdb5bc77 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -620,6 +620,7 @@ static void BM_StreamingPingPongMsgs(benchmark::State& state) { } while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); request_rw->Write(send_request, tag(0)); // Start client send response_rw.Read(&recv_request, tag(1)); // Start server recv request_rw->Read(&recv_response, tag(2)); // Start client recv From b3f34b6c2b24cbe3deac6c4efef0978fc9dc7c73 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Feb 2017 10:29:33 -0800 Subject: [PATCH 112/167] Add benchmarks of closures, combiners, exec_ctx primitives --- CMakeLists.txt | 39 ++ Makefile | 48 +++ build.yaml | 17 + test/cpp/microbenchmarks/bm_closure.cc | 341 ++++++++++++++++++ .../generated/sources_and_headers.json | 18 + tools/run_tests/generated/tests.json | 22 ++ 6 files changed, 485 insertions(+) create mode 100644 test/cpp/microbenchmarks/bm_closure.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 101f0ad2ef9..f2ff350e860 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -570,6 +570,9 @@ add_dependencies(buildtests_cxx alarm_cpp_test) add_dependencies(buildtests_cxx async_end2end_test) add_dependencies(buildtests_cxx auth_property_iterator_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_closure) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_fullstack) endif() add_dependencies(buildtests_cxx channel_arguments_test) @@ -7445,6 +7448,42 @@ endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_executable(bm_closure + test/cpp/microbenchmarks/bm_closure.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_closure + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_closure + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + benchmark + grpc_test_util + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + add_executable(bm_fullstack test/cpp/microbenchmarks/bm_fullstack.cc third_party/googletest/src/gtest-all.cc diff --git a/Makefile b/Makefile index 5dde577cca8..d9af4099a30 100644 --- a/Makefile +++ b/Makefile @@ -1043,6 +1043,7 @@ wakeup_fd_cv_test: $(BINDIR)/$(CONFIG)/wakeup_fd_cv_test alarm_cpp_test: $(BINDIR)/$(CONFIG)/alarm_cpp_test async_end2end_test: $(BINDIR)/$(CONFIG)/async_end2end_test auth_property_iterator_test: $(BINDIR)/$(CONFIG)/auth_property_iterator_test +bm_closure: $(BINDIR)/$(CONFIG)/bm_closure bm_fullstack: $(BINDIR)/$(CONFIG)/bm_fullstack channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test channel_filter_test: $(BINDIR)/$(CONFIG)/channel_filter_test @@ -1449,6 +1450,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/alarm_cpp_test \ $(BINDIR)/$(CONFIG)/async_end2end_test \ $(BINDIR)/$(CONFIG)/auth_property_iterator_test \ + $(BINDIR)/$(CONFIG)/bm_closure \ $(BINDIR)/$(CONFIG)/bm_fullstack \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ @@ -1553,6 +1555,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/alarm_cpp_test \ $(BINDIR)/$(CONFIG)/async_end2end_test \ $(BINDIR)/$(CONFIG)/auth_property_iterator_test \ + $(BINDIR)/$(CONFIG)/bm_closure \ $(BINDIR)/$(CONFIG)/bm_fullstack \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ @@ -1870,6 +1873,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/async_end2end_test || ( echo test async_end2end_test failed ; exit 1 ) $(E) "[RUN] Testing auth_property_iterator_test" $(Q) $(BINDIR)/$(CONFIG)/auth_property_iterator_test || ( echo test auth_property_iterator_test failed ; exit 1 ) + $(E) "[RUN] Testing bm_closure" + $(Q) $(BINDIR)/$(CONFIG)/bm_closure || ( echo test bm_closure failed ; exit 1 ) $(E) "[RUN] Testing bm_fullstack" $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack || ( echo test bm_fullstack failed ; exit 1 ) $(E) "[RUN] Testing channel_arguments_test" @@ -12389,6 +12394,49 @@ endif endif +BM_CLOSURE_SRC = \ + test/cpp/microbenchmarks/bm_closure.cc \ + +BM_CLOSURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_CLOSURE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_closure: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_closure: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_closure: $(PROTOBUF_DEP) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_closure + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_closure.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_closure: $(BM_CLOSURE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_CLOSURE_OBJS:.o=.dep) +endif +endif + + BM_FULLSTACK_SRC = \ test/cpp/microbenchmarks/bm_fullstack.cc \ diff --git a/build.yaml b/build.yaml index 2fbaa2e2262..42e599e1ada 100644 --- a/build.yaml +++ b/build.yaml @@ -2984,6 +2984,23 @@ targets: - grpc - gpr_test_util - gpr +- name: bm_closure + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_closure.cc + deps: + - benchmark + - grpc_test_util + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + platforms: + - mac + - linux + - posix - name: bm_fullstack build: test language: c++ diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc new file mode 100644 index 00000000000..de354410e52 --- /dev/null +++ b/test/cpp/microbenchmarks/bm_closure.cc @@ -0,0 +1,341 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Test various closure related operations */ + +#include + +extern "C" { +#include "src/core/lib/iomgr/closure.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/exec_ctx.h" +} + +#include "third_party/benchmark/include/benchmark/benchmark.h" + +static class InitializeStuff { + public: + InitializeStuff() { grpc_init(); } + ~InitializeStuff() { grpc_shutdown(); } +} initialize_stuff; + +static void BM_NoOpExecCtx(benchmark::State& state) { + while (state.KeepRunning()) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_exec_ctx_finish(&exec_ctx); + } +} +BENCHMARK(BM_NoOpExecCtx); + +static void BM_WellFlushed(benchmark::State& state) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + grpc_exec_ctx_flush(&exec_ctx); + } + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_WellFlushed); + +static void DoNothing(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {} + +static void BM_ClosureInitAgainstExecCtx(benchmark::State& state) { + grpc_closure c; + while (state.KeepRunning()) { + benchmark::DoNotOptimize( + grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx)); + } +} +BENCHMARK(BM_ClosureInitAgainstExecCtx); + +static void BM_ClosureInitAgainstCombiner(benchmark::State& state) { + grpc_combiner* combiner = grpc_combiner_create(NULL); + grpc_closure c; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(grpc_closure_init( + &c, DoNothing, NULL, grpc_combiner_scheduler(combiner, false))); + } + GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureInitAgainstCombiner); + +static void BM_ClosureRunOnExecCtx(benchmark::State& state) { + grpc_closure c; + grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + grpc_closure_run(&exec_ctx, &c, GRPC_ERROR_NONE); + grpc_exec_ctx_flush(&exec_ctx); + } + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureRunOnExecCtx); + +static void BM_ClosureCreateAndRun(benchmark::State& state) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + grpc_closure_run(&exec_ctx, grpc_closure_create(DoNothing, NULL, + grpc_schedule_on_exec_ctx), + GRPC_ERROR_NONE); + } + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureCreateAndRun); + +static void BM_ClosureInitAndRun(benchmark::State& state) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_closure c; + while (state.KeepRunning()) { + grpc_closure_run(&exec_ctx, grpc_closure_init(&c, DoNothing, NULL, + grpc_schedule_on_exec_ctx), + GRPC_ERROR_NONE); + } + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureInitAndRun); + +static void BM_ClosureSchedOnExecCtx(benchmark::State& state) { + grpc_closure c; + grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + grpc_closure_sched(&exec_ctx, &c, GRPC_ERROR_NONE); + grpc_exec_ctx_flush(&exec_ctx); + } + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureSchedOnExecCtx); + +static void BM_ClosureSched2OnExecCtx(benchmark::State& state) { + grpc_closure c1; + grpc_closure c2; + grpc_closure_init(&c1, DoNothing, NULL, grpc_schedule_on_exec_ctx); + grpc_closure_init(&c2, DoNothing, NULL, grpc_schedule_on_exec_ctx); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + grpc_closure_sched(&exec_ctx, &c1, GRPC_ERROR_NONE); + grpc_closure_sched(&exec_ctx, &c2, GRPC_ERROR_NONE); + grpc_exec_ctx_flush(&exec_ctx); + } + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureSched2OnExecCtx); + +static void BM_ClosureSched3OnExecCtx(benchmark::State& state) { + grpc_closure c1; + grpc_closure c2; + grpc_closure c3; + grpc_closure_init(&c1, DoNothing, NULL, grpc_schedule_on_exec_ctx); + grpc_closure_init(&c2, DoNothing, NULL, grpc_schedule_on_exec_ctx); + grpc_closure_init(&c3, DoNothing, NULL, grpc_schedule_on_exec_ctx); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + grpc_closure_sched(&exec_ctx, &c1, GRPC_ERROR_NONE); + grpc_closure_sched(&exec_ctx, &c2, GRPC_ERROR_NONE); + grpc_closure_sched(&exec_ctx, &c3, GRPC_ERROR_NONE); + grpc_exec_ctx_flush(&exec_ctx); + } + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureSched3OnExecCtx); + +static void BM_ClosureSchedOnCombiner(benchmark::State& state) { + grpc_combiner* combiner = grpc_combiner_create(NULL); + grpc_closure c; + grpc_closure_init(&c, DoNothing, NULL, + grpc_combiner_scheduler(combiner, false)); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + grpc_closure_sched(&exec_ctx, &c, GRPC_ERROR_NONE); + grpc_exec_ctx_flush(&exec_ctx); + } + GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureSchedOnCombiner); + +static void BM_ClosureSched2OnCombiner(benchmark::State& state) { + grpc_combiner* combiner = grpc_combiner_create(NULL); + grpc_closure c1; + grpc_closure c2; + grpc_closure_init(&c1, DoNothing, NULL, + grpc_combiner_scheduler(combiner, false)); + grpc_closure_init(&c2, DoNothing, NULL, + grpc_combiner_scheduler(combiner, false)); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + grpc_closure_sched(&exec_ctx, &c1, GRPC_ERROR_NONE); + grpc_closure_sched(&exec_ctx, &c2, GRPC_ERROR_NONE); + grpc_exec_ctx_flush(&exec_ctx); + } + GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureSched2OnCombiner); + +static void BM_ClosureSched3OnCombiner(benchmark::State& state) { + grpc_combiner* combiner = grpc_combiner_create(NULL); + grpc_closure c1; + grpc_closure c2; + grpc_closure c3; + grpc_closure_init(&c1, DoNothing, NULL, + grpc_combiner_scheduler(combiner, false)); + grpc_closure_init(&c2, DoNothing, NULL, + grpc_combiner_scheduler(combiner, false)); + grpc_closure_init(&c3, DoNothing, NULL, + grpc_combiner_scheduler(combiner, false)); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + grpc_closure_sched(&exec_ctx, &c1, GRPC_ERROR_NONE); + grpc_closure_sched(&exec_ctx, &c2, GRPC_ERROR_NONE); + grpc_closure_sched(&exec_ctx, &c3, GRPC_ERROR_NONE); + grpc_exec_ctx_flush(&exec_ctx); + } + GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureSched3OnCombiner); + +static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) { + grpc_combiner* combiner1 = grpc_combiner_create(NULL); + grpc_combiner* combiner2 = grpc_combiner_create(NULL); + grpc_closure c1; + grpc_closure c2; + grpc_closure_init(&c1, DoNothing, NULL, + grpc_combiner_scheduler(combiner1, false)); + grpc_closure_init(&c2, DoNothing, NULL, + grpc_combiner_scheduler(combiner2, false)); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + grpc_closure_sched(&exec_ctx, &c1, GRPC_ERROR_NONE); + grpc_closure_sched(&exec_ctx, &c2, GRPC_ERROR_NONE); + grpc_exec_ctx_flush(&exec_ctx); + } + GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished"); + GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished"); + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureSched2OnTwoCombiners); + +static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) { + grpc_combiner* combiner1 = grpc_combiner_create(NULL); + grpc_combiner* combiner2 = grpc_combiner_create(NULL); + grpc_closure c1; + grpc_closure c2; + grpc_closure c3; + grpc_closure c4; + grpc_closure_init(&c1, DoNothing, NULL, + grpc_combiner_scheduler(combiner1, false)); + grpc_closure_init(&c2, DoNothing, NULL, + grpc_combiner_scheduler(combiner2, false)); + grpc_closure_init(&c3, DoNothing, NULL, + grpc_combiner_scheduler(combiner1, false)); + grpc_closure_init(&c4, DoNothing, NULL, + grpc_combiner_scheduler(combiner2, false)); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + grpc_closure_sched(&exec_ctx, &c1, GRPC_ERROR_NONE); + grpc_closure_sched(&exec_ctx, &c2, GRPC_ERROR_NONE); + grpc_closure_sched(&exec_ctx, &c3, GRPC_ERROR_NONE); + grpc_closure_sched(&exec_ctx, &c4, GRPC_ERROR_NONE); + grpc_exec_ctx_flush(&exec_ctx); + } + GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished"); + GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished"); + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureSched4OnTwoCombiners); + +// Helper that continuously reschedules the same closure against something until +// the benchmark is complete +class Rescheduler { + public: + Rescheduler(benchmark::State& state, grpc_closure_scheduler* scheduler) + : state_(state) { + grpc_closure_init(&closure_, Step, this, scheduler); + } + + void ScheduleFirst(grpc_exec_ctx* exec_ctx) { + grpc_closure_sched(exec_ctx, &closure_, GRPC_ERROR_NONE); + } + + void ScheduleFirstAgainstDifferentScheduler( + grpc_exec_ctx* exec_ctx, grpc_closure_scheduler* scheduler) { + grpc_closure_sched(exec_ctx, grpc_closure_create(Step, this, scheduler), + GRPC_ERROR_NONE); + } + + private: + benchmark::State& state_; + grpc_closure closure_; + + static void Step(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { + Rescheduler* self = static_cast(arg); + if (self->state_.KeepRunning()) { + grpc_closure_sched(exec_ctx, &self->closure_, GRPC_ERROR_NONE); + } + } +}; + +static void BM_ClosureReschedOnExecCtx(benchmark::State& state) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + Rescheduler(state, grpc_schedule_on_exec_ctx).ScheduleFirst(&exec_ctx); + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureReschedOnExecCtx); + +static void BM_ClosureReschedOnCombiner(benchmark::State& state) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_combiner* combiner = grpc_combiner_create(NULL); + Rescheduler(state, grpc_combiner_scheduler(combiner, false)) + .ScheduleFirst(&exec_ctx); + grpc_exec_ctx_flush(&exec_ctx); + GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureReschedOnCombiner); + +static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_combiner* combiner = grpc_combiner_create(NULL); + Rescheduler(state, grpc_combiner_finally_scheduler(combiner, false)) + .ScheduleFirstAgainstDifferentScheduler( + &exec_ctx, grpc_combiner_scheduler(combiner, false)); + grpc_exec_ctx_flush(&exec_ctx); + GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_ClosureReschedOnCombinerFinally); + +BENCHMARK_MAIN(); diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 0d5aeb233c4..77799074da4 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2363,6 +2363,24 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_closure", + "src": [ + "test/cpp/microbenchmarks/bm_closure.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "benchmark", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index c193caa26f0..a161e8a670e 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2469,6 +2469,28 @@ "windows" ] }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_closure", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "--benchmark_min_time=0" From 2607763272bb3f508869de550288198a2807bf35 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Feb 2017 11:26:43 -0800 Subject: [PATCH 113/167] Add -o magicks --- tools/run_tests/run_microbenchmark.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index fe901056c59..df0944d144d 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -128,7 +128,8 @@ def collect_perf(bm_name, args): 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()]) for line in subprocess.check_output(['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines(): - subprocess.check_call(['sudo', 'perf', 'record', '-g', '-c', '1000', + subprocess.check_call(['sudo', 'perf', 'record', '-o', 'perf.data', + '-g', '-c', '1000', 'bins/mutrace/%s' % bm_name, '--benchmark_filter=^%s$' % line, '--benchmark_min_time=20']) From 1a967c31c3a7fbd2d6d5cb6c1e575fdb2d6bce9d Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 13 Feb 2017 11:38:17 -0800 Subject: [PATCH 114/167] Added Pylint to sanity tests Pylint is only enabled for "grpcio/grpc" package, and various specific checks that currently fail are disabled, each with a respective TODO item in the .pylintrc file. --- .pylintrc | 38 +++++++++++++++++++ tools/distrib/pylint_code.sh | 48 ++++++++++++++++++++++++ tools/run_tests/sanity/sanity_tests.yaml | 1 + 3 files changed, 87 insertions(+) create mode 100644 .pylintrc create mode 100755 tools/distrib/pylint_code.sh diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000000..a4f757f9bdd --- /dev/null +++ b/.pylintrc @@ -0,0 +1,38 @@ +[MESSAGES CONTROL] + +#TODO: Enable missing-docstring +#TODO: Enable too-few-public-methods +#TODO: Enable too-many-arguments +#TODO: Enable no-init +#TODO: Enable duplicate-code +#TODO: Enable invalid-name +#TODO: Enable suppressed-message +#TODO: Enable locally-disabled +#TODO: Enable protected-access +#TODO: Enable no-name-in-module +#TODO: Enable unused-argument +#TODO: Enable fixme +#TODO: Enable wrong-import-order +#TODO: Enable no-value-for-parameter +#TODO: Enable cyclic-import +#TODO: Enable unused-variable +#TODO: Enable redefined-outer-name +#TODO: Enable unused-import +#TODO: Enable too-many-instance-attributes +#TODO: Enable broad-except +#TODO: Enable too-many-locals +#TODO: Enable too-many-lines +#TODO: Enable redefined-variable-type +#TODO: Enable next-method-called +#TODO: Enable import-error +#TODO: Enable useless-else-on-loop +#TODO: Enable too-many-return-statements +#TODO: Enable too-many-nested-blocks +#TODO: Enable super-init-not-called +#TODO: Enable simplifiable-if-statement +#TODO: Enable no-self-use +#TODO: Enable no-member +#TODO: Enable logging-format-interpolation +#TODO: Enable dangerous-default-value + +disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,suppressed-message,locally-disabled,protected-access,no-name-in-module,unused-argument,fixme,wrong-import-order,no-value-for-parameter,cyclic-import,unused-variable,redefined-outer-name,unused-import,too-many-instance-attributes,broad-except,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called,simplifiable-if-statement,no-self-use,no-member,logging-format-interpolation,dangerous-default-value diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh new file mode 100755 index 00000000000..6369e605d53 --- /dev/null +++ b/tools/distrib/pylint_code.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +# change to root directory +cd $(dirname $0)/../.. + +DIRS=src/python/grpcio/grpc + +VIRTUALENV=python_pylint_venv + +virtualenv $VIRTUALENV +PYTHON=`realpath $VIRTUALENV/bin/python` +$PYTHON -m pip install pylint==1.6.5 + +for dir in $DIRS; do + $PYTHON -m pylint --rcfile=.pylintrc -rn $dir || exit $? +done + +exit 0 diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index ce41da802d0..445f53e5558 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -12,6 +12,7 @@ - script: tools/distrib/check_trailing_newlines.sh - script: tools/distrib/check_nanopb_output.sh - script: tools/distrib/check_include_guards.py +- script: tools/distrib/pylint_code.sh - script: tools/distrib/yapf_code.sh - script: tools/distrib/python/check_grpcio_tools.py From 96f8728b06740dd6b63ab463319b4795209dbec4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Feb 2017 12:39:12 -0800 Subject: [PATCH 115/167] Add a comparison benchmark of acquiring a mutex --- test/cpp/microbenchmarks/bm_closure.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc index de354410e52..dac7d9c6779 100644 --- a/test/cpp/microbenchmarks/bm_closure.cc +++ b/test/cpp/microbenchmarks/bm_closure.cc @@ -170,6 +170,20 @@ static void BM_ClosureSched3OnExecCtx(benchmark::State& state) { } BENCHMARK(BM_ClosureSched3OnExecCtx); +static void BM_AcquireMutex(benchmark::State& state) { + // for comparison with the combiner stuff below + gpr_mu mu; + gpr_mu_init(&mu); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (state.KeepRunning()) { + gpr_mu_lock(&mu); + DoNothing(&exec_ctx, NULL, GRPC_ERROR_NONE); + gpr_mu_unlock(&mu); + } + grpc_exec_ctx_finish(&exec_ctx); +} +BENCHMARK(BM_AcquireMutex); + static void BM_ClosureSchedOnCombiner(benchmark::State& state) { grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c; From d12731e4f1c877dbe892d3a0d7f6d4f9e1c74421 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Feb 2017 12:41:27 -0800 Subject: [PATCH 116/167] Fix path --- tools/run_tests/run_microbenchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index df0944d144d..5792df8ab6e 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -152,7 +152,7 @@ def collect_summary(bm_name, args): '--benchmark_out=out.json', '--benchmark_out_format=json'])) if args.bigquery_upload: - with open('/tmp/out.csv', 'w') as f: + with open('out.csv', 'w') as f: f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.json'])) subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', 'out.csv']) From 4dde8b277d5196962f5c08afd39db94f50242c29 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Feb 2017 12:44:14 -0800 Subject: [PATCH 117/167] Fix stack corruption --- test/cpp/microbenchmarks/bm_closure.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc index dac7d9c6779..80d6610e13b 100644 --- a/test/cpp/microbenchmarks/bm_closure.cc +++ b/test/cpp/microbenchmarks/bm_closure.cc @@ -324,7 +324,8 @@ class Rescheduler { static void BM_ClosureReschedOnExecCtx(benchmark::State& state) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - Rescheduler(state, grpc_schedule_on_exec_ctx).ScheduleFirst(&exec_ctx); + Rescheduler r(state, grpc_schedule_on_exec_ctx); + r.ScheduleFirst(&exec_ctx); grpc_exec_ctx_finish(&exec_ctx); } BENCHMARK(BM_ClosureReschedOnExecCtx); @@ -332,8 +333,8 @@ BENCHMARK(BM_ClosureReschedOnExecCtx); static void BM_ClosureReschedOnCombiner(benchmark::State& state) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_combiner* combiner = grpc_combiner_create(NULL); - Rescheduler(state, grpc_combiner_scheduler(combiner, false)) - .ScheduleFirst(&exec_ctx); + Rescheduler r(state, grpc_combiner_scheduler(combiner, false)); + r.ScheduleFirst(&exec_ctx); grpc_exec_ctx_flush(&exec_ctx); GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); @@ -343,9 +344,9 @@ BENCHMARK(BM_ClosureReschedOnCombiner); static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_combiner* combiner = grpc_combiner_create(NULL); - Rescheduler(state, grpc_combiner_finally_scheduler(combiner, false)) - .ScheduleFirstAgainstDifferentScheduler( - &exec_ctx, grpc_combiner_scheduler(combiner, false)); + Rescheduler r(state, grpc_combiner_finally_scheduler(combiner, false)); + r.ScheduleFirstAgainstDifferentScheduler( + &exec_ctx, grpc_combiner_scheduler(combiner, false)); grpc_exec_ctx_flush(&exec_ctx); GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); From a78871e4b33929b206188291bb989c034f3c1eab Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Feb 2017 14:48:30 -0800 Subject: [PATCH 118/167] Fix command line --- tools/run_tests/run_microbenchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 2a1b0246958..0d56e4c3553 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -134,7 +134,7 @@ def collect_perf(bm_name, args): '--benchmark_filter=^%s$' % line, '--benchmark_min_time=20']) with open('bm.perf', 'w') as f: - f.write(subprocess.check_output(['sudo', 'perf', 'script'])) + f.write(subprocess.check_output(['sudo', 'perf', 'script', '-i', 'perf.data'])) with open('bm.folded', 'w') as f: f.write(subprocess.check_output([ '%s/stackcollapse-perf.pl' % flamegraph_dir, 'bm.perf'])) From f31f0962740cd263dffe383f6bf1b8a56c4ddbdb Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Fri, 10 Feb 2017 14:53:58 -0800 Subject: [PATCH 119/167] Use call context to propagare LR costs --- CMakeLists.txt | 46 +- Makefile | 65 +- build.yaml | 19 +- gRPC-Core.podspec | 1 + grpc.def | 1 + grpc.gemspec | 1 + include/grpc++/impl/codegen/server_context.h | 9 +- .../grpc++/test/server_context_test_spouse.h | 14 +- include/grpc/grpc.h | 10 +- .../grpc/load_reporting.h | 44 +- package.xml | 1 + src/core/ext/load_reporting/load_reporting.c | 20 + src/core/ext/load_reporting/load_reporting.h | 17 +- .../load_reporting/load_reporting_filter.c | 18 +- src/core/lib/channel/context.h | 3 + src/core/lib/transport/static_metadata.c | 763 +++++++++--------- src/core/lib/transport/static_metadata.h | 221 +++-- src/cpp/server/server_context.cc | 16 + src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 + src/ruby/ext/grpc/rb_grpc_imports.generated.h | 3 + test/core/end2end/fuzzers/hpack.dictionary | 2 - test/core/end2end/tests/load_reporting_hook.c | 29 +- .../core/surface/public_headers_must_be_c89.c | 1 + tools/codegen/core/gen_static_metadata.py | 2 - tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 1 + .../generated/sources_and_headers.json | 35 +- .../vcxproj/grpc++_test/grpc++_test.vcxproj | 170 ---- .../grpc++_test/grpc++_test.vcxproj.filters | 35 - vsprojects/vcxproj/grpc/grpc.vcxproj | 1 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 3 + .../grpc_test_util/grpc_test_util.vcxproj | 1 + .../grpc_test_util.vcxproj.filters | 3 + .../grpc_unsecure/grpc_unsecure.vcxproj | 1 + .../grpc_unsecure.vcxproj.filters | 3 + .../server_context_test_spouse_test.vcxproj | 3 - 36 files changed, 663 insertions(+), 902 deletions(-) rename src/cpp/test/server_context_test_spouse.cc => include/grpc/load_reporting.h (63%) delete mode 100644 vsprojects/vcxproj/grpc++_test/grpc++_test.vcxproj delete mode 100644 vsprojects/vcxproj/grpc++_test/grpc++_test.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index 51e512c85c4..d232a7cc779 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1086,6 +1086,7 @@ foreach(_hdr include/grpc/grpc.h include/grpc/grpc_posix.h include/grpc/grpc_security_constants.h + include/grpc/load_reporting.h include/grpc/slice.h include/grpc/slice_buffer.h include/grpc/status.h @@ -1363,6 +1364,7 @@ foreach(_hdr include/grpc/grpc.h include/grpc/grpc_posix.h include/grpc/grpc_security_constants.h + include/grpc/load_reporting.h include/grpc/slice.h include/grpc/slice_buffer.h include/grpc/status.h @@ -1585,6 +1587,7 @@ foreach(_hdr include/grpc/grpc.h include/grpc/grpc_posix.h include/grpc/grpc_security_constants.h + include/grpc/load_reporting.h include/grpc/slice.h include/grpc/slice_buffer.h include/grpc/status.h @@ -1906,6 +1909,7 @@ foreach(_hdr include/grpc/grpc.h include/grpc/grpc_posix.h include/grpc/grpc_security_constants.h + include/grpc/load_reporting.h include/grpc/slice.h include/grpc/slice_buffer.h include/grpc/status.h @@ -2558,6 +2562,7 @@ foreach(_hdr include/grpc/grpc.h include/grpc/grpc_posix.h include/grpc/grpc_security_constants.h + include/grpc/load_reporting.h include/grpc/slice.h include/grpc/slice_buffer.h include/grpc/status.h @@ -2699,46 +2704,6 @@ endif() if (gRPC_BUILD_TESTS) -add_library(grpc++_test - src/cpp/test/server_context_test_spouse.cc -) - -if(WIN32 AND MSVC) - set_target_properties(grpc++_test PROPERTIES COMPILE_PDB_NAME "grpc++_test" - COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" - ) - if (gRPC_INSTALL) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++_test.pdb - DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL - ) - endif() -endif() - - -target_include_directories(grpc++_test - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include - PRIVATE ${BORINGSSL_ROOT_DIR}/include - PRIVATE ${PROTOBUF_ROOT_DIR}/src - PRIVATE ${ZLIB_INCLUDE_DIR} - PRIVATE ${BENCHMARK}/include - PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib - PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include - PRIVATE third_party/googletest/include - PRIVATE third_party/googletest - PRIVATE ${_gRPC_PROTO_GENS_DIR} -) - -target_link_libraries(grpc++_test - ${_gRPC_PROTOBUF_LIBRARIES} - ${_gRPC_ALLTARGETS_LIBRARIES} - grpc++ -) - - -endif (gRPC_BUILD_TESTS) -if (gRPC_BUILD_TESTS) - add_library(grpc++_test_config test/cpp/util/test_config_cc.cc ) @@ -9330,7 +9295,6 @@ target_link_libraries(server_context_test_spouse_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} grpc_test_util - grpc++_test grpc++ grpc gpr_test_util diff --git a/Makefile b/Makefile index 125512307c2..4f0797dab69 100644 --- a/Makefile +++ b/Makefile @@ -1257,9 +1257,9 @@ pc_cxx: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc pc_cxx_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc ifeq ($(EMBED_OPENSSL),true) -privatelibs_cxx: $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_chacha_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_spake25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_sign_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_verify_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_gcm_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_newhope_statistical_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_newhope_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_newhope_vectors_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_obj_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a $(LIBDIR)/$(CONFIG)/libbenchmark.a +privatelibs_cxx: $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_chacha_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_spake25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_sign_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_verify_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_gcm_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_newhope_statistical_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_newhope_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_newhope_vectors_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_obj_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a $(LIBDIR)/$(CONFIG)/libbenchmark.a else -privatelibs_cxx: $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libbenchmark.a +privatelibs_cxx: $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libbenchmark.a endif @@ -2882,6 +2882,7 @@ PUBLIC_HEADERS_C += \ include/grpc/grpc.h \ include/grpc/grpc_posix.h \ include/grpc/grpc_security_constants.h \ + include/grpc/load_reporting.h \ include/grpc/slice.h \ include/grpc/slice_buffer.h \ include/grpc/status.h \ @@ -3163,6 +3164,7 @@ PUBLIC_HEADERS_C += \ include/grpc/grpc.h \ include/grpc/grpc_posix.h \ include/grpc/grpc_security_constants.h \ + include/grpc/load_reporting.h \ include/grpc/slice.h \ include/grpc/slice_buffer.h \ include/grpc/status.h \ @@ -3388,6 +3390,7 @@ PUBLIC_HEADERS_C += \ include/grpc/grpc.h \ include/grpc/grpc_posix.h \ include/grpc/grpc_security_constants.h \ + include/grpc/load_reporting.h \ include/grpc/slice.h \ include/grpc/slice_buffer.h \ include/grpc/status.h \ @@ -3690,6 +3693,7 @@ PUBLIC_HEADERS_C += \ include/grpc/grpc.h \ include/grpc/grpc_posix.h \ include/grpc/grpc_security_constants.h \ + include/grpc/load_reporting.h \ include/grpc/slice.h \ include/grpc/slice_buffer.h \ include/grpc/status.h \ @@ -4340,6 +4344,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/grpc.h \ include/grpc/grpc_posix.h \ include/grpc/grpc_security_constants.h \ + include/grpc/load_reporting.h \ include/grpc/slice.h \ include/grpc/slice_buffer.h \ include/grpc/status.h \ @@ -4533,55 +4538,6 @@ $(OBJDIR)/$(CONFIG)/src/cpp/ext/proto_server_reflection.o: $(GENDIR)/src/proto/g $(OBJDIR)/$(CONFIG)/src/cpp/ext/proto_server_reflection_plugin.o: $(GENDIR)/src/proto/grpc/reflection/v1alpha/reflection.pb.cc $(GENDIR)/src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.cc -LIBGRPC++_TEST_SRC = \ - src/cpp/test/server_context_test_spouse.cc \ - -PUBLIC_HEADERS_CXX += \ - -LIBGRPC++_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_TEST_SRC)))) - - -ifeq ($(NO_SECURE),true) - -# You can't build secure libraries if you don't have OpenSSL. - -$(LIBDIR)/$(CONFIG)/libgrpc++_test.a: openssl_dep_error - - -else - -ifeq ($(NO_PROTOBUF),true) - -# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay. - -$(LIBDIR)/$(CONFIG)/libgrpc++_test.a: protobuf_dep_error - - -else - -$(LIBDIR)/$(CONFIG)/libgrpc++_test.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_TEST_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_test.a - $(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBGRPC++_TEST_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc++_test.a -endif - - - - -endif - -endif - -ifneq ($(NO_SECURE),true) -ifneq ($(NO_DEPS),true) --include $(LIBGRPC++_TEST_OBJS:.o=.dep) -endif -endif - - LIBGRPC++_TEST_CONFIG_SRC = \ test/cpp/util/test_config_cc.cc \ @@ -14476,16 +14432,16 @@ $(BINDIR)/$(CONFIG)/server_context_test_spouse_test: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/server_context_test_spouse_test: $(PROTOBUF_DEP) $(SERVER_CONTEXT_TEST_SPOUSE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/server_context_test_spouse_test: $(PROTOBUF_DEP) $(SERVER_CONTEXT_TEST_SPOUSE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(SERVER_CONTEXT_TEST_SPOUSE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/server_context_test_spouse_test + $(Q) $(LDXX) $(LDFLAGS) $(SERVER_CONTEXT_TEST_SPOUSE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/server_context_test_spouse_test endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/test/server_context_test_spouse_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/test/server_context_test_spouse_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_server_context_test_spouse_test: $(SERVER_CONTEXT_TEST_SPOUSE_TEST_OBJS:.o=.dep) @@ -17798,7 +17754,6 @@ src/cpp/common/secure_create_auth_context.cc: $(OPENSSL_DEP) src/cpp/ext/proto_server_reflection.cc: $(OPENSSL_DEP) src/cpp/ext/proto_server_reflection_plugin.cc: $(OPENSSL_DEP) src/cpp/server/secure_server_credentials.cc: $(OPENSSL_DEP) -src/cpp/test/server_context_test_spouse.cc: $(OPENSSL_DEP) src/csharp/ext/grpc_csharp_ext.c: $(OPENSSL_DEP) test/core/bad_client/bad_client.c: $(OPENSSL_DEP) test/core/bad_ssl/server_common.c: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 62321fafc26..750bb696d30 100644 --- a/build.yaml +++ b/build.yaml @@ -165,6 +165,7 @@ filegroups: - include/grpc/grpc.h - include/grpc/grpc_posix.h - include/grpc/grpc_security_constants.h + - include/grpc/load_reporting.h - include/grpc/slice.h - include/grpc/slice_buffer.h - include/grpc/status.h @@ -894,6 +895,12 @@ filegroups: language: c++ src: - src/proto/grpc/reflection/v1alpha/reflection.proto +- name: grpc++_test + language: c++ + public_headers: + - include/grpc++/test/server_context_test_spouse.h + deps: + - grpc++ - name: thrift_util language: c++ public_headers: @@ -1140,15 +1147,6 @@ libs: - grpc++ filegroups: - grpc++_reflection_proto -- name: grpc++_test - build: private - language: c++ - headers: - - include/grpc++/test/server_context_test_spouse.h - src: - - src/cpp/test/server_context_test_spouse.cc - deps: - - grpc++ - name: grpc++_test_config build: private language: c++ @@ -3663,11 +3661,12 @@ targets: - test/cpp/test/server_context_test_spouse_test.cc deps: - grpc_test_util - - grpc++_test - grpc++ - grpc - gpr_test_util - gpr + uses: + - grpc++_test - name: server_crash_test gtest: true cpu_cost: 0.1 diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 07b3b2bad53..384ef094f19 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -162,6 +162,7 @@ Pod::Spec.new do |s| 'include/grpc/grpc.h', 'include/grpc/grpc_posix.h', 'include/grpc/grpc_security_constants.h', + 'include/grpc/load_reporting.h', 'include/grpc/slice.h', 'include/grpc/slice_buffer.h', 'include/grpc/status.h', diff --git a/grpc.def b/grpc.def index 5ef59ab9561..e387c8b7125 100644 --- a/grpc.def +++ b/grpc.def @@ -69,6 +69,7 @@ EXPORTS grpc_channel_create_registered_call grpc_call_start_batch grpc_call_get_peer + grpc_call_set_load_reporting_cost_context grpc_census_call_set_context grpc_census_call_get_context grpc_channel_get_target diff --git a/grpc.gemspec b/grpc.gemspec index 335021fdc83..82c9d680801 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -143,6 +143,7 @@ Gem::Specification.new do |s| s.files += %w( include/grpc/grpc.h ) s.files += %w( include/grpc/grpc_posix.h ) s.files += %w( include/grpc/grpc_security_constants.h ) + s.files += %w( include/grpc/load_reporting.h ) s.files += %w( include/grpc/slice.h ) s.files += %w( include/grpc/slice_buffer.h ) s.files += %w( include/grpc/status.h ) diff --git a/include/grpc++/impl/codegen/server_context.h b/include/grpc++/impl/codegen/server_context.h index 8c7fe0809ea..43bbff6ce9c 100644 --- a/include/grpc++/impl/codegen/server_context.h +++ b/include/grpc++/impl/codegen/server_context.h @@ -36,6 +36,10 @@ #include #include +#include + +#include +#include #include #include @@ -43,14 +47,12 @@ #include #include #include -#include struct grpc_metadata; struct grpc_call; struct census_context; namespace grpc { - class ClientContext; template class ServerAsyncReader; @@ -143,6 +145,9 @@ class ServerContext { } void set_compression_algorithm(grpc_compression_algorithm algorithm); + // Set the load reporting costs in \a cost_data for the call. + void SetLoadReportingCosts(const std::vector& cost_data); + std::shared_ptr auth_context() const { if (auth_context_.get() == nullptr) { auth_context_ = CreateAuthContext(call_); diff --git a/include/grpc++/test/server_context_test_spouse.h b/include/grpc++/test/server_context_test_spouse.h index b2482854b50..4cb84e77c19 100644 --- a/include/grpc++/test/server_context_test_spouse.h +++ b/include/grpc++/test/server_context_test_spouse.h @@ -48,10 +48,22 @@ class ServerContextTestSpouse { // Inject client metadata to the ServerContext for the test. The test spouse // must be alive when ServerContext::client_metadata is called. - void AddClientMetadata(const grpc::string& key, const grpc::string& value); + void AddClientMetadata(const grpc::string& key, const grpc::string& value) { + client_metadata_storage_.insert( + std::pair(key, value)); + ctx_->client_metadata_.map()->clear(); + for (auto iter = client_metadata_storage_.begin(); + iter != client_metadata_storage_.end(); ++iter) { + ctx_->client_metadata_.map()->insert( + std::pair(iter->first.c_str(), + iter->second.c_str())); + } + } + std::multimap GetInitialMetadata() const { return ctx_->initial_metadata_; } + std::multimap GetTrailingMetadata() const { return ctx_->trailing_metadata_; } diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index 37b823ae1e9..1b33d48c023 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -229,14 +229,20 @@ GRPCAPI grpc_call_error grpc_call_start_batch(grpc_call *call, functionality. Instead, use grpc_auth_context. */ GRPCAPI char *grpc_call_get_peer(grpc_call *call); +struct grpc_load_reporting_cost_context; + +/* Associate costs contained in \a cost_context to \a call. */ +GRPCAPI void grpc_call_set_load_reporting_cost_context( + grpc_call *call, struct grpc_load_reporting_cost_context *context); + struct census_context; -/* Set census context for a call; Must be called before first call to +/** Set census context for a call; Must be called before first call to grpc_call_start_batch(). */ GRPCAPI void grpc_census_call_set_context(grpc_call *call, struct census_context *context); -/* Retrieve the calls current census context. */ +/** Retrieve the calls current census context. */ GRPCAPI struct census_context *grpc_census_call_get_context(grpc_call *call); /** Return a newly allocated string representing the target a channel was diff --git a/src/cpp/test/server_context_test_spouse.cc b/include/grpc/load_reporting.h similarity index 63% rename from src/cpp/test/server_context_test_spouse.cc rename to include/grpc/load_reporting.h index b812d169a52..c833ce5c63b 100644 --- a/src/cpp/test/server_context_test_spouse.cc +++ b/include/grpc/load_reporting.h @@ -1,6 +1,6 @@ /* * - * Copyright 2016, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,23 +31,33 @@ * */ -#include +#ifndef GRPC_LOAD_REPORTING_H +#define GRPC_LOAD_REPORTING_H -namespace grpc { -namespace testing { +#include +#include -void ServerContextTestSpouse::AddClientMetadata(const grpc::string& key, - const grpc::string& value) { - client_metadata_storage_.insert( - std::pair(key, value)); - ctx_->client_metadata_.map()->clear(); - for (auto iter = client_metadata_storage_.begin(); - iter != client_metadata_storage_.end(); ++iter) { - ctx_->client_metadata_.map()->insert( - std::pair(iter->first.c_str(), - iter->second.c_str())); - } +#ifdef __cplusplus +extern "C" { +#endif + +/** Metadata key for the gRPC LB load balancer token. + * + * The value corresponding to this key is an opaque token that is given to the + * frontend as part of each pick; the frontend sends this token to the backend + * in each request it sends when using that pick. The token is used by the + * backend to verify the request and to allow the backend to report load to the + * gRPC LB system. */ +#define GRPC_LB_TOKEN_MD_KEY "lb-token" + +/** A sequence of values for load reporting purposes */ +typedef struct grpc_load_reporting_cost_context { + grpc_slice *values; + size_t values_count; +} grpc_load_reporting_cost_context; + +#ifdef __cplusplus } +#endif -} // namespace testing -} // namespace grpc +#endif /* GRPC_LOAD_REPORTING_H */ diff --git a/package.xml b/package.xml index 6250c78ddc9..e4db6a7d2e5 100644 --- a/package.xml +++ b/package.xml @@ -152,6 +152,7 @@ + diff --git a/src/core/ext/load_reporting/load_reporting.c b/src/core/ext/load_reporting/load_reporting.c index 37b06a737fb..942aea4fd1c 100644 --- a/src/core/ext/load_reporting/load_reporting.c +++ b/src/core/ext/load_reporting/load_reporting.c @@ -34,14 +34,34 @@ #include #include +#include #include #include #include "src/core/ext/load_reporting/load_reporting.h" #include "src/core/ext/load_reporting/load_reporting_filter.h" #include "src/core/lib/channel/channel_stack_builder.h" +#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel_init.h" +static void destroy_lr_cost_context(void *c) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_load_reporting_cost_context *cost_ctx = c; + for (size_t i = 0; i < cost_ctx->values_count; ++i) { + grpc_slice_unref_internal(&exec_ctx, cost_ctx->values[i]); + } + grpc_exec_ctx_finish(&exec_ctx); + gpr_free(cost_ctx->values); + gpr_free(cost_ctx); +} + +void grpc_call_set_load_reporting_cost_context( + grpc_call *call, grpc_load_reporting_cost_context *ctx) { + grpc_call_context_set(call, GRPC_CONTEXT_LR_COST, ctx, + destroy_lr_cost_context); +} + static bool is_load_reporting_enabled(const grpc_channel_args *a) { if (a == NULL) return false; for (size_t i = 0; i < a->num_args; i++) { diff --git a/src/core/ext/load_reporting/load_reporting.h b/src/core/ext/load_reporting/load_reporting.h index a157844734e..22859a599a7 100644 --- a/src/core/ext/load_reporting/load_reporting.h +++ b/src/core/ext/load_reporting/load_reporting.h @@ -35,23 +35,8 @@ #define GRPC_CORE_EXT_LOAD_REPORTING_LOAD_REPORTING_H #include -#include "src/core/lib/channel/channel_stack.h" - -/** Metadata key for the gRPC LB load balancer token. - * - * The value corresponding to this key is an opaque token that is given to the - * frontend as part of each pick; the frontend sends this token to the backend - * in each request it sends when using that pick. The token is used by the - * backend to verify the request and to allow the backend to report load to the - * gRPC LB system. */ -#define GRPC_LB_TOKEN_MD_KEY "lb-token" -/** Metadata key for gRPC LB cost reporting. - * - * The value corresponding to this key is an opaque binary blob reported by the - * backend as part of its trailing metadata containing cost information for the - * call. */ -#define GRPC_LB_COST_MD_KEY "lb-cost-bin" +#include "src/core/lib/channel/channel_stack.h" /** Identifiers for the invocation point of the users LR callback */ typedef enum grpc_load_reporting_source { diff --git a/src/core/ext/load_reporting/load_reporting_filter.c b/src/core/ext/load_reporting/load_reporting_filter.c index 8af6191c3b8..7fc88db603a 100644 --- a/src/core/ext/load_reporting/load_reporting_filter.c +++ b/src/core/ext/load_reporting/load_reporting_filter.c @@ -31,11 +31,13 @@ * */ +#include + +#include #include #include #include #include -#include #include "src/core/ext/load_reporting/load_reporting.h" #include "src/core/ext/load_reporting/load_reporting_filter.h" @@ -46,8 +48,6 @@ typedef struct call_data { intptr_t id; /**< an id unique to the call */ - bool have_trailing_md_string; - grpc_slice trailing_md_string; bool have_initial_md_string; grpc_slice initial_md_string; bool have_service_method; @@ -142,9 +142,6 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, if (calld->have_initial_md_string) { grpc_slice_unref_internal(exec_ctx, calld->initial_md_string); } - if (calld->have_trailing_md_string) { - grpc_slice_unref_internal(exec_ctx, calld->trailing_md_string); - } if (calld->have_service_method) { grpc_slice_unref_internal(exec_ctx, calld->service_method); } @@ -201,15 +198,6 @@ static void lr_start_transport_stream_op(grpc_exec_ctx *exec_ctx, /* substitute our callback for the higher callback */ calld->ops_recv_initial_metadata_ready = op->recv_initial_metadata_ready; op->recv_initial_metadata_ready = &calld->on_initial_md_ready; - } else if (op->send_trailing_metadata) { - if (op->send_trailing_metadata->idx.named.lb_cost_bin != NULL) { - calld->trailing_md_string = grpc_slice_ref_internal( - GRPC_MDVALUE(op->send_trailing_metadata->idx.named.lb_cost_bin->md)); - calld->have_trailing_md_string = true; - grpc_metadata_batch_remove( - exec_ctx, op->send_trailing_metadata, - op->send_trailing_metadata->idx.named.lb_cost_bin); - } } grpc_call_next_op(exec_ctx, elem, op); diff --git a/src/core/lib/channel/context.h b/src/core/lib/channel/context.h index 6c931ad28ae..2c1174ce7a1 100644 --- a/src/core/lib/channel/context.h +++ b/src/core/lib/channel/context.h @@ -50,6 +50,9 @@ typedef enum { /// Reserved for traffic_class_context. GRPC_CONTEXT_TRAFFIC, + /// Costs for Load Reporting. + GRPC_CONTEXT_LR_COST, + GRPC_CONTEXT_COUNT } grpc_context_index; diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c index 750711befd7..c13ba230b32 100644 --- a/src/core/lib/transport/static_metadata.c +++ b/src/core/lib/transport/static_metadata.c @@ -55,64 +55,63 @@ static uint8_t g_bytes[] = { 112, 101, 103, 114, 112, 99, 45, 105, 110, 116, 101, 114, 110, 97, 108, 45, 101, 110, 99, 111, 100, 105, 110, 103, 45, 114, 101, 113, 117, 101, 115, 116, 117, 115, 101, 114, 45, 97, 103, 101, 110, 116, 104, 111, 115, - 116, 108, 98, 45, 116, 111, 107, 101, 110, 108, 98, 45, 99, 111, 115, - 116, 45, 98, 105, 110, 103, 114, 112, 99, 45, 116, 105, 109, 101, 111, - 117, 116, 103, 114, 112, 99, 45, 116, 114, 97, 99, 105, 110, 103, 45, - 98, 105, 110, 103, 114, 112, 99, 45, 115, 116, 97, 116, 115, 45, 98, - 105, 110, 103, 114, 112, 99, 46, 119, 97, 105, 116, 95, 102, 111, 114, - 95, 114, 101, 97, 100, 121, 103, 114, 112, 99, 46, 116, 105, 109, 101, - 111, 117, 116, 103, 114, 112, 99, 46, 109, 97, 120, 95, 114, 101, 113, - 117, 101, 115, 116, 95, 109, 101, 115, 115, 97, 103, 101, 95, 98, 121, - 116, 101, 115, 103, 114, 112, 99, 46, 109, 97, 120, 95, 114, 101, 115, - 112, 111, 110, 115, 101, 95, 109, 101, 115, 115, 97, 103, 101, 95, 98, - 121, 116, 101, 115, 47, 103, 114, 112, 99, 46, 108, 98, 46, 118, 49, - 46, 76, 111, 97, 100, 66, 97, 108, 97, 110, 99, 101, 114, 47, 66, - 97, 108, 97, 110, 99, 101, 76, 111, 97, 100, 48, 49, 50, 105, 100, - 101, 110, 116, 105, 116, 121, 103, 122, 105, 112, 100, 101, 102, 108, 97, - 116, 101, 116, 114, 97, 105, 108, 101, 114, 115, 97, 112, 112, 108, 105, - 99, 97, 116, 105, 111, 110, 47, 103, 114, 112, 99, 80, 79, 83, 84, - 50, 48, 48, 52, 48, 52, 104, 116, 116, 112, 104, 116, 116, 112, 115, - 103, 114, 112, 99, 71, 69, 84, 80, 85, 84, 47, 47, 105, 110, 100, - 101, 120, 46, 104, 116, 109, 108, 50, 48, 52, 50, 48, 54, 51, 48, - 52, 52, 48, 48, 53, 48, 48, 97, 99, 99, 101, 112, 116, 45, 99, - 104, 97, 114, 115, 101, 116, 97, 99, 99, 101, 112, 116, 45, 101, 110, - 99, 111, 100, 105, 110, 103, 103, 122, 105, 112, 44, 32, 100, 101, 102, - 108, 97, 116, 101, 97, 99, 99, 101, 112, 116, 45, 108, 97, 110, 103, - 117, 97, 103, 101, 97, 99, 99, 101, 112, 116, 45, 114, 97, 110, 103, - 101, 115, 97, 99, 99, 101, 112, 116, 97, 99, 99, 101, 115, 115, 45, - 99, 111, 110, 116, 114, 111, 108, 45, 97, 108, 108, 111, 119, 45, 111, - 114, 105, 103, 105, 110, 97, 103, 101, 97, 108, 108, 111, 119, 97, 117, - 116, 104, 111, 114, 105, 122, 97, 116, 105, 111, 110, 99, 97, 99, 104, - 101, 45, 99, 111, 110, 116, 114, 111, 108, 99, 111, 110, 116, 101, 110, - 116, 45, 100, 105, 115, 112, 111, 115, 105, 116, 105, 111, 110, 99, 111, - 110, 116, 101, 110, 116, 45, 101, 110, 99, 111, 100, 105, 110, 103, 99, - 111, 110, 116, 101, 110, 116, 45, 108, 97, 110, 103, 117, 97, 103, 101, - 99, 111, 110, 116, 101, 110, 116, 45, 108, 101, 110, 103, 116, 104, 99, - 111, 110, 116, 101, 110, 116, 45, 108, 111, 99, 97, 116, 105, 111, 110, - 99, 111, 110, 116, 101, 110, 116, 45, 114, 97, 110, 103, 101, 99, 111, - 111, 107, 105, 101, 100, 97, 116, 101, 101, 116, 97, 103, 101, 120, 112, - 101, 99, 116, 101, 120, 112, 105, 114, 101, 115, 102, 114, 111, 109, 105, - 102, 45, 109, 97, 116, 99, 104, 105, 102, 45, 109, 111, 100, 105, 102, - 105, 101, 100, 45, 115, 105, 110, 99, 101, 105, 102, 45, 110, 111, 110, - 101, 45, 109, 97, 116, 99, 104, 105, 102, 45, 114, 97, 110, 103, 101, - 105, 102, 45, 117, 110, 109, 111, 100, 105, 102, 105, 101, 100, 45, 115, - 105, 110, 99, 101, 108, 97, 115, 116, 45, 109, 111, 100, 105, 102, 105, - 101, 100, 108, 105, 110, 107, 108, 111, 99, 97, 116, 105, 111, 110, 109, - 97, 120, 45, 102, 111, 114, 119, 97, 114, 100, 115, 112, 114, 111, 120, - 121, 45, 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 101, 112, - 114, 111, 120, 121, 45, 97, 117, 116, 104, 111, 114, 105, 122, 97, 116, - 105, 111, 110, 114, 97, 110, 103, 101, 114, 101, 102, 101, 114, 101, 114, - 114, 101, 102, 114, 101, 115, 104, 114, 101, 116, 114, 121, 45, 97, 102, - 116, 101, 114, 115, 101, 114, 118, 101, 114, 115, 101, 116, 45, 99, 111, - 111, 107, 105, 101, 115, 116, 114, 105, 99, 116, 45, 116, 114, 97, 110, - 115, 112, 111, 114, 116, 45, 115, 101, 99, 117, 114, 105, 116, 121, 116, - 114, 97, 110, 115, 102, 101, 114, 45, 101, 110, 99, 111, 100, 105, 110, - 103, 118, 97, 114, 121, 118, 105, 97, 119, 119, 119, 45, 97, 117, 116, - 104, 101, 110, 116, 105, 99, 97, 116, 101, 105, 100, 101, 110, 116, 105, - 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 105, 100, 101, 110, 116, - 105, 116, 121, 44, 103, 122, 105, 112, 100, 101, 102, 108, 97, 116, 101, - 44, 103, 122, 105, 112, 105, 100, 101, 110, 116, 105, 116, 121, 44, 100, - 101, 102, 108, 97, 116, 101, 44, 103, 122, 105, 112}; + 116, 108, 98, 45, 116, 111, 107, 101, 110, 103, 114, 112, 99, 45, 116, + 105, 109, 101, 111, 117, 116, 103, 114, 112, 99, 45, 116, 114, 97, 99, + 105, 110, 103, 45, 98, 105, 110, 103, 114, 112, 99, 45, 115, 116, 97, + 116, 115, 45, 98, 105, 110, 103, 114, 112, 99, 46, 119, 97, 105, 116, + 95, 102, 111, 114, 95, 114, 101, 97, 100, 121, 103, 114, 112, 99, 46, + 116, 105, 109, 101, 111, 117, 116, 103, 114, 112, 99, 46, 109, 97, 120, + 95, 114, 101, 113, 117, 101, 115, 116, 95, 109, 101, 115, 115, 97, 103, + 101, 95, 98, 121, 116, 101, 115, 103, 114, 112, 99, 46, 109, 97, 120, + 95, 114, 101, 115, 112, 111, 110, 115, 101, 95, 109, 101, 115, 115, 97, + 103, 101, 95, 98, 121, 116, 101, 115, 47, 103, 114, 112, 99, 46, 108, + 98, 46, 118, 49, 46, 76, 111, 97, 100, 66, 97, 108, 97, 110, 99, + 101, 114, 47, 66, 97, 108, 97, 110, 99, 101, 76, 111, 97, 100, 48, + 49, 50, 105, 100, 101, 110, 116, 105, 116, 121, 103, 122, 105, 112, 100, + 101, 102, 108, 97, 116, 101, 116, 114, 97, 105, 108, 101, 114, 115, 97, + 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 103, 114, 112, 99, + 80, 79, 83, 84, 50, 48, 48, 52, 48, 52, 104, 116, 116, 112, 104, + 116, 116, 112, 115, 103, 114, 112, 99, 71, 69, 84, 80, 85, 84, 47, + 47, 105, 110, 100, 101, 120, 46, 104, 116, 109, 108, 50, 48, 52, 50, + 48, 54, 51, 48, 52, 52, 48, 48, 53, 48, 48, 97, 99, 99, 101, + 112, 116, 45, 99, 104, 97, 114, 115, 101, 116, 97, 99, 99, 101, 112, + 116, 45, 101, 110, 99, 111, 100, 105, 110, 103, 103, 122, 105, 112, 44, + 32, 100, 101, 102, 108, 97, 116, 101, 97, 99, 99, 101, 112, 116, 45, + 108, 97, 110, 103, 117, 97, 103, 101, 97, 99, 99, 101, 112, 116, 45, + 114, 97, 110, 103, 101, 115, 97, 99, 99, 101, 112, 116, 97, 99, 99, + 101, 115, 115, 45, 99, 111, 110, 116, 114, 111, 108, 45, 97, 108, 108, + 111, 119, 45, 111, 114, 105, 103, 105, 110, 97, 103, 101, 97, 108, 108, + 111, 119, 97, 117, 116, 104, 111, 114, 105, 122, 97, 116, 105, 111, 110, + 99, 97, 99, 104, 101, 45, 99, 111, 110, 116, 114, 111, 108, 99, 111, + 110, 116, 101, 110, 116, 45, 100, 105, 115, 112, 111, 115, 105, 116, 105, + 111, 110, 99, 111, 110, 116, 101, 110, 116, 45, 101, 110, 99, 111, 100, + 105, 110, 103, 99, 111, 110, 116, 101, 110, 116, 45, 108, 97, 110, 103, + 117, 97, 103, 101, 99, 111, 110, 116, 101, 110, 116, 45, 108, 101, 110, + 103, 116, 104, 99, 111, 110, 116, 101, 110, 116, 45, 108, 111, 99, 97, + 116, 105, 111, 110, 99, 111, 110, 116, 101, 110, 116, 45, 114, 97, 110, + 103, 101, 99, 111, 111, 107, 105, 101, 100, 97, 116, 101, 101, 116, 97, + 103, 101, 120, 112, 101, 99, 116, 101, 120, 112, 105, 114, 101, 115, 102, + 114, 111, 109, 105, 102, 45, 109, 97, 116, 99, 104, 105, 102, 45, 109, + 111, 100, 105, 102, 105, 101, 100, 45, 115, 105, 110, 99, 101, 105, 102, + 45, 110, 111, 110, 101, 45, 109, 97, 116, 99, 104, 105, 102, 45, 114, + 97, 110, 103, 101, 105, 102, 45, 117, 110, 109, 111, 100, 105, 102, 105, + 101, 100, 45, 115, 105, 110, 99, 101, 108, 97, 115, 116, 45, 109, 111, + 100, 105, 102, 105, 101, 100, 108, 105, 110, 107, 108, 111, 99, 97, 116, + 105, 111, 110, 109, 97, 120, 45, 102, 111, 114, 119, 97, 114, 100, 115, + 112, 114, 111, 120, 121, 45, 97, 117, 116, 104, 101, 110, 116, 105, 99, + 97, 116, 101, 112, 114, 111, 120, 121, 45, 97, 117, 116, 104, 111, 114, + 105, 122, 97, 116, 105, 111, 110, 114, 97, 110, 103, 101, 114, 101, 102, + 101, 114, 101, 114, 114, 101, 102, 114, 101, 115, 104, 114, 101, 116, 114, + 121, 45, 97, 102, 116, 101, 114, 115, 101, 114, 118, 101, 114, 115, 101, + 116, 45, 99, 111, 111, 107, 105, 101, 115, 116, 114, 105, 99, 116, 45, + 116, 114, 97, 110, 115, 112, 111, 114, 116, 45, 115, 101, 99, 117, 114, + 105, 116, 121, 116, 114, 97, 110, 115, 102, 101, 114, 45, 101, 110, 99, + 111, 100, 105, 110, 103, 118, 97, 114, 121, 118, 105, 97, 119, 119, 119, + 45, 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 101, 105, 100, + 101, 110, 116, 105, 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 105, + 100, 101, 110, 116, 105, 116, 121, 44, 103, 122, 105, 112, 100, 101, 102, + 108, 97, 116, 101, 44, 103, 122, 105, 112, 105, 100, 101, 110, 116, 105, + 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 44, 103, 122, 105, 112}; static void static_ref(void *unused) {} static void static_unref(grpc_exec_ctx *exec_ctx, void *unused) {} @@ -221,7 +220,6 @@ grpc_slice_refcount grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = { {&grpc_static_metadata_vtable, &static_sub_refcnt}, {&grpc_static_metadata_vtable, &static_sub_refcnt}, {&grpc_static_metadata_vtable, &static_sub_refcnt}, - {&grpc_static_metadata_vtable, &static_sub_refcnt}, }; const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { @@ -258,188 +256,186 @@ const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { {.refcount = &grpc_static_metadata_refcounts[15], .data.refcounted = {g_bytes + 166, 8}}, {.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 174, 11}}, + .data.refcounted = {g_bytes + 174, 12}}, {.refcount = &grpc_static_metadata_refcounts[17], - .data.refcounted = {g_bytes + 185, 12}}, + .data.refcounted = {g_bytes + 186, 16}}, {.refcount = &grpc_static_metadata_refcounts[18], - .data.refcounted = {g_bytes + 197, 16}}, + .data.refcounted = {g_bytes + 202, 14}}, {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 213, 14}}, + .data.refcounted = {g_bytes + 216, 0}}, {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}, + .data.refcounted = {g_bytes + 216, 19}}, {.refcount = &grpc_static_metadata_refcounts[21], - .data.refcounted = {g_bytes + 227, 19}}, + .data.refcounted = {g_bytes + 235, 12}}, {.refcount = &grpc_static_metadata_refcounts[22], - .data.refcounted = {g_bytes + 246, 12}}, + .data.refcounted = {g_bytes + 247, 30}}, {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 258, 30}}, + .data.refcounted = {g_bytes + 277, 31}}, {.refcount = &grpc_static_metadata_refcounts[24], - .data.refcounted = {g_bytes + 288, 31}}, + .data.refcounted = {g_bytes + 308, 36}}, {.refcount = &grpc_static_metadata_refcounts[25], - .data.refcounted = {g_bytes + 319, 36}}, + .data.refcounted = {g_bytes + 344, 1}}, {.refcount = &grpc_static_metadata_refcounts[26], - .data.refcounted = {g_bytes + 355, 1}}, + .data.refcounted = {g_bytes + 345, 1}}, {.refcount = &grpc_static_metadata_refcounts[27], - .data.refcounted = {g_bytes + 356, 1}}, + .data.refcounted = {g_bytes + 346, 1}}, {.refcount = &grpc_static_metadata_refcounts[28], - .data.refcounted = {g_bytes + 357, 1}}, + .data.refcounted = {g_bytes + 347, 8}}, {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 358, 8}}, + .data.refcounted = {g_bytes + 355, 4}}, {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 366, 4}}, + .data.refcounted = {g_bytes + 359, 7}}, {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 370, 7}}, + .data.refcounted = {g_bytes + 366, 8}}, {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 377, 8}}, + .data.refcounted = {g_bytes + 374, 16}}, {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 385, 16}}, + .data.refcounted = {g_bytes + 390, 4}}, {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 401, 4}}, + .data.refcounted = {g_bytes + 394, 3}}, {.refcount = &grpc_static_metadata_refcounts[35], - .data.refcounted = {g_bytes + 405, 3}}, + .data.refcounted = {g_bytes + 397, 3}}, {.refcount = &grpc_static_metadata_refcounts[36], - .data.refcounted = {g_bytes + 408, 3}}, + .data.refcounted = {g_bytes + 400, 4}}, {.refcount = &grpc_static_metadata_refcounts[37], - .data.refcounted = {g_bytes + 411, 4}}, + .data.refcounted = {g_bytes + 404, 5}}, {.refcount = &grpc_static_metadata_refcounts[38], - .data.refcounted = {g_bytes + 415, 5}}, + .data.refcounted = {g_bytes + 409, 4}}, {.refcount = &grpc_static_metadata_refcounts[39], - .data.refcounted = {g_bytes + 420, 4}}, + .data.refcounted = {g_bytes + 413, 3}}, {.refcount = &grpc_static_metadata_refcounts[40], - .data.refcounted = {g_bytes + 424, 3}}, + .data.refcounted = {g_bytes + 416, 3}}, {.refcount = &grpc_static_metadata_refcounts[41], - .data.refcounted = {g_bytes + 427, 3}}, + .data.refcounted = {g_bytes + 419, 1}}, {.refcount = &grpc_static_metadata_refcounts[42], - .data.refcounted = {g_bytes + 430, 1}}, + .data.refcounted = {g_bytes + 420, 11}}, {.refcount = &grpc_static_metadata_refcounts[43], - .data.refcounted = {g_bytes + 431, 11}}, + .data.refcounted = {g_bytes + 431, 3}}, {.refcount = &grpc_static_metadata_refcounts[44], - .data.refcounted = {g_bytes + 442, 3}}, + .data.refcounted = {g_bytes + 434, 3}}, {.refcount = &grpc_static_metadata_refcounts[45], - .data.refcounted = {g_bytes + 445, 3}}, + .data.refcounted = {g_bytes + 437, 3}}, {.refcount = &grpc_static_metadata_refcounts[46], - .data.refcounted = {g_bytes + 448, 3}}, + .data.refcounted = {g_bytes + 440, 3}}, {.refcount = &grpc_static_metadata_refcounts[47], - .data.refcounted = {g_bytes + 451, 3}}, + .data.refcounted = {g_bytes + 443, 3}}, {.refcount = &grpc_static_metadata_refcounts[48], - .data.refcounted = {g_bytes + 454, 3}}, + .data.refcounted = {g_bytes + 446, 14}}, {.refcount = &grpc_static_metadata_refcounts[49], - .data.refcounted = {g_bytes + 457, 14}}, + .data.refcounted = {g_bytes + 460, 15}}, {.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 471, 15}}, + .data.refcounted = {g_bytes + 475, 13}}, {.refcount = &grpc_static_metadata_refcounts[51], - .data.refcounted = {g_bytes + 486, 13}}, + .data.refcounted = {g_bytes + 488, 15}}, {.refcount = &grpc_static_metadata_refcounts[52], - .data.refcounted = {g_bytes + 499, 15}}, + .data.refcounted = {g_bytes + 503, 13}}, {.refcount = &grpc_static_metadata_refcounts[53], - .data.refcounted = {g_bytes + 514, 13}}, + .data.refcounted = {g_bytes + 516, 6}}, {.refcount = &grpc_static_metadata_refcounts[54], - .data.refcounted = {g_bytes + 527, 6}}, + .data.refcounted = {g_bytes + 522, 27}}, {.refcount = &grpc_static_metadata_refcounts[55], - .data.refcounted = {g_bytes + 533, 27}}, + .data.refcounted = {g_bytes + 549, 3}}, {.refcount = &grpc_static_metadata_refcounts[56], - .data.refcounted = {g_bytes + 560, 3}}, + .data.refcounted = {g_bytes + 552, 5}}, {.refcount = &grpc_static_metadata_refcounts[57], - .data.refcounted = {g_bytes + 563, 5}}, + .data.refcounted = {g_bytes + 557, 13}}, {.refcount = &grpc_static_metadata_refcounts[58], - .data.refcounted = {g_bytes + 568, 13}}, + .data.refcounted = {g_bytes + 570, 13}}, {.refcount = &grpc_static_metadata_refcounts[59], - .data.refcounted = {g_bytes + 581, 13}}, + .data.refcounted = {g_bytes + 583, 19}}, {.refcount = &grpc_static_metadata_refcounts[60], - .data.refcounted = {g_bytes + 594, 19}}, + .data.refcounted = {g_bytes + 602, 16}}, {.refcount = &grpc_static_metadata_refcounts[61], - .data.refcounted = {g_bytes + 613, 16}}, + .data.refcounted = {g_bytes + 618, 16}}, {.refcount = &grpc_static_metadata_refcounts[62], - .data.refcounted = {g_bytes + 629, 16}}, + .data.refcounted = {g_bytes + 634, 14}}, {.refcount = &grpc_static_metadata_refcounts[63], - .data.refcounted = {g_bytes + 645, 14}}, + .data.refcounted = {g_bytes + 648, 16}}, {.refcount = &grpc_static_metadata_refcounts[64], - .data.refcounted = {g_bytes + 659, 16}}, + .data.refcounted = {g_bytes + 664, 13}}, {.refcount = &grpc_static_metadata_refcounts[65], - .data.refcounted = {g_bytes + 675, 13}}, + .data.refcounted = {g_bytes + 677, 6}}, {.refcount = &grpc_static_metadata_refcounts[66], - .data.refcounted = {g_bytes + 688, 6}}, + .data.refcounted = {g_bytes + 683, 4}}, {.refcount = &grpc_static_metadata_refcounts[67], - .data.refcounted = {g_bytes + 694, 4}}, + .data.refcounted = {g_bytes + 687, 4}}, {.refcount = &grpc_static_metadata_refcounts[68], - .data.refcounted = {g_bytes + 698, 4}}, + .data.refcounted = {g_bytes + 691, 6}}, {.refcount = &grpc_static_metadata_refcounts[69], - .data.refcounted = {g_bytes + 702, 6}}, + .data.refcounted = {g_bytes + 697, 7}}, {.refcount = &grpc_static_metadata_refcounts[70], - .data.refcounted = {g_bytes + 708, 7}}, + .data.refcounted = {g_bytes + 704, 4}}, {.refcount = &grpc_static_metadata_refcounts[71], - .data.refcounted = {g_bytes + 715, 4}}, + .data.refcounted = {g_bytes + 708, 8}}, {.refcount = &grpc_static_metadata_refcounts[72], - .data.refcounted = {g_bytes + 719, 8}}, + .data.refcounted = {g_bytes + 716, 17}}, {.refcount = &grpc_static_metadata_refcounts[73], - .data.refcounted = {g_bytes + 727, 17}}, + .data.refcounted = {g_bytes + 733, 13}}, {.refcount = &grpc_static_metadata_refcounts[74], - .data.refcounted = {g_bytes + 744, 13}}, + .data.refcounted = {g_bytes + 746, 8}}, {.refcount = &grpc_static_metadata_refcounts[75], - .data.refcounted = {g_bytes + 757, 8}}, + .data.refcounted = {g_bytes + 754, 19}}, {.refcount = &grpc_static_metadata_refcounts[76], - .data.refcounted = {g_bytes + 765, 19}}, + .data.refcounted = {g_bytes + 773, 13}}, {.refcount = &grpc_static_metadata_refcounts[77], - .data.refcounted = {g_bytes + 784, 13}}, + .data.refcounted = {g_bytes + 786, 4}}, {.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 797, 4}}, + .data.refcounted = {g_bytes + 790, 8}}, {.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 801, 8}}, + .data.refcounted = {g_bytes + 798, 12}}, {.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 809, 12}}, + .data.refcounted = {g_bytes + 810, 18}}, {.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 821, 18}}, + .data.refcounted = {g_bytes + 828, 19}}, {.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 839, 19}}, + .data.refcounted = {g_bytes + 847, 5}}, {.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 858, 5}}, + .data.refcounted = {g_bytes + 852, 7}}, {.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 863, 7}}, + .data.refcounted = {g_bytes + 859, 7}}, {.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 870, 7}}, + .data.refcounted = {g_bytes + 866, 11}}, {.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 877, 11}}, + .data.refcounted = {g_bytes + 877, 6}}, {.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 888, 6}}, + .data.refcounted = {g_bytes + 883, 10}}, {.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 894, 10}}, + .data.refcounted = {g_bytes + 893, 25}}, {.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 904, 25}}, + .data.refcounted = {g_bytes + 918, 17}}, {.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 929, 17}}, + .data.refcounted = {g_bytes + 935, 4}}, {.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 946, 4}}, + .data.refcounted = {g_bytes + 939, 3}}, {.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 950, 3}}, + .data.refcounted = {g_bytes + 942, 16}}, {.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 953, 16}}, + .data.refcounted = {g_bytes + 958, 16}}, {.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 969, 16}}, + .data.refcounted = {g_bytes + 974, 13}}, {.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 985, 13}}, + .data.refcounted = {g_bytes + 987, 12}}, {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 998, 12}}, - {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1010, 21}}, + .data.refcounted = {g_bytes + 999, 21}}, }; uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 6, 6, 8, 8}; + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 6, 6, 8, 8}; static const int8_t elems_r[] = { - 10, 8, -3, 0, 9, 21, -76, 22, 0, 10, -7, 20, 0, 19, 18, 17, - 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, -49, -50, 16, -52, -53, -54, -54, -55, -56, -57, 0, 38, 37, 36, 35, - 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, - 18, 17, 16, 15, 14, 13, 12, 15, 14, 13, 12, 11, 10, 9, 8, 0}; + 10, 8, -3, 0, 9, 21, -75, 22, 0, 10, -7, 20, 0, 19, 18, 17, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + -48, -49, 16, -51, -52, -53, -54, -54, -55, -56, -57, 0, 37, 36, 35, 34, + 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, + 17, 16, 15, 14, 13, 12, 11, 14, 13, 12, 11, 10, 9, 8, 0}; static uint32_t elems_phash(uint32_t i) { - i -= 42; - uint32_t x = i % 96; - uint32_t y = i / 96; + i -= 41; + uint32_t x = i % 95; + uint32_t y = i / 95; uint32_t h = x; if (y < GPR_ARRAY_SIZE(elems_r)) { uint32_t delta = (uint32_t)elems_r[y]; @@ -449,30 +445,29 @@ static uint32_t elems_phash(uint32_t i) { } static const uint16_t elem_keys[] = { - 1009, 1010, 1011, 240, 241, 242, 243, 244, 138, 139, 42, 43, - 429, 430, 431, 911, 912, 913, 712, 713, 1098, 522, 714, 1294, - 1392, 1490, 1588, 4822, 4920, 4951, 5116, 5214, 5312, 1111, 5410, 5508, - 5606, 5704, 5802, 5900, 5998, 6096, 6194, 6292, 6390, 6488, 6586, 6684, - 6782, 6880, 6978, 7076, 7174, 7272, 7370, 7468, 7566, 7664, 7762, 7860, - 7958, 8056, 8154, 8252, 8350, 1074, 1075, 1076, 1077, 8448, 8546, 8644, - 8742, 8840, 8938, 9036, 9134, 314, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 132, 231, 232, 0, 0, 0, 0, 0, + 998, 999, 1000, 237, 238, 239, 240, 241, 136, 137, 41, 42, + 424, 425, 426, 901, 902, 903, 704, 705, 1086, 516, 706, 1280, + 1377, 1474, 4675, 4772, 4803, 4966, 5063, 5160, 5257, 1099, 5354, 5451, + 5548, 5645, 5742, 5839, 5936, 6033, 6130, 6227, 6324, 6421, 6518, 6615, + 6712, 6809, 6906, 7003, 7100, 7197, 7294, 7391, 7488, 7585, 7682, 7779, + 7876, 7973, 8070, 8167, 8264, 1063, 1064, 1065, 1066, 8361, 8458, 8555, + 8652, 8749, 8846, 8943, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 130, 228, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0}; + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; static const uint8_t elem_idxs[] = { - 74, 77, 75, 19, 20, 21, 22, 23, 15, 16, 17, 18, 11, 12, 13, - 3, 4, 5, 0, 1, 41, 6, 2, 70, 48, 55, 56, 24, 25, 26, - 27, 28, 29, 7, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 42, 43, 44, 45, 46, 47, 49, 50, 51, 52, 53, 54, 57, 58, 59, - 60, 61, 62, 63, 64, 76, 78, 79, 80, 65, 66, 67, 68, 69, 71, - 72, 73, 14, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 8, 9, 10}; + 73, 76, 74, 19, 20, 21, 22, 23, 15, 16, 17, 18, 11, 12, 13, + 3, 4, 5, 0, 1, 41, 6, 2, 69, 48, 55, 24, 25, 26, 27, + 28, 29, 30, 7, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 42, + 43, 44, 45, 46, 47, 49, 50, 51, 52, 53, 54, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 75, 77, 78, 79, 65, 66, 67, 68, 70, 71, + 72, 14, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 8, 9, 10}; grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) { if (a == -1 || b == -1) return GRPC_MDNULL; - uint32_t k = (uint32_t)(a * 98 + b); + uint32_t k = (uint32_t)(a * 97 + b); uint32_t h = elems_phash(k); return h < GPR_ARRAY_SIZE(elem_keys) && elem_keys[h] == k ? GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[elem_idxs[h]], @@ -483,328 +478,324 @@ grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) { grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = { {{.refcount = &grpc_static_metadata_refcounts[7], .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[26], - .data.refcounted = {g_bytes + 355, 1}}}, + {.refcount = &grpc_static_metadata_refcounts[25], + .data.refcounted = {g_bytes + 344, 1}}}, {{.refcount = &grpc_static_metadata_refcounts[7], .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[27], - .data.refcounted = {g_bytes + 356, 1}}}, + {.refcount = &grpc_static_metadata_refcounts[26], + .data.refcounted = {g_bytes + 345, 1}}}, {{.refcount = &grpc_static_metadata_refcounts[7], .data.refcounted = {g_bytes + 50, 11}}, + {.refcount = &grpc_static_metadata_refcounts[27], + .data.refcounted = {g_bytes + 346, 1}}}, + {{.refcount = &grpc_static_metadata_refcounts[9], + .data.refcounted = {g_bytes + 77, 13}}, {.refcount = &grpc_static_metadata_refcounts[28], - .data.refcounted = {g_bytes + 357, 1}}}, + .data.refcounted = {g_bytes + 347, 8}}}, {{.refcount = &grpc_static_metadata_refcounts[9], .data.refcounted = {g_bytes + 77, 13}}, {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 358, 8}}}, + .data.refcounted = {g_bytes + 355, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[9], .data.refcounted = {g_bytes + 77, 13}}, {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 366, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[9], - .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 370, 7}}}, + .data.refcounted = {g_bytes + 359, 7}}}, {{.refcount = &grpc_static_metadata_refcounts[5], .data.refcounted = {g_bytes + 36, 2}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 377, 8}}}, + {.refcount = &grpc_static_metadata_refcounts[31], + .data.refcounted = {g_bytes + 366, 8}}}, {{.refcount = &grpc_static_metadata_refcounts[11], .data.refcounted = {g_bytes + 110, 12}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 385, 16}}}, + {.refcount = &grpc_static_metadata_refcounts[32], + .data.refcounted = {g_bytes + 374, 16}}}, {{.refcount = &grpc_static_metadata_refcounts[1], .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 401, 4}}}, + {.refcount = &grpc_static_metadata_refcounts[33], + .data.refcounted = {g_bytes + 390, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[35], - .data.refcounted = {g_bytes + 405, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[34], + .data.refcounted = {g_bytes + 394, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, + {.refcount = &grpc_static_metadata_refcounts[35], + .data.refcounted = {g_bytes + 397, 3}}}, + {{.refcount = &grpc_static_metadata_refcounts[4], + .data.refcounted = {g_bytes + 29, 7}}, {.refcount = &grpc_static_metadata_refcounts[36], - .data.refcounted = {g_bytes + 408, 3}}}, + .data.refcounted = {g_bytes + 400, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[4], .data.refcounted = {g_bytes + 29, 7}}, {.refcount = &grpc_static_metadata_refcounts[37], - .data.refcounted = {g_bytes + 411, 4}}}, + .data.refcounted = {g_bytes + 404, 5}}}, {{.refcount = &grpc_static_metadata_refcounts[4], .data.refcounted = {g_bytes + 29, 7}}, {.refcount = &grpc_static_metadata_refcounts[38], - .data.refcounted = {g_bytes + 415, 5}}}, - {{.refcount = &grpc_static_metadata_refcounts[4], - .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[39], - .data.refcounted = {g_bytes + 420, 4}}}, + .data.refcounted = {g_bytes + 409, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[3], .data.refcounted = {g_bytes + 19, 10}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[1], .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[40], - .data.refcounted = {g_bytes + 424, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[39], + .data.refcounted = {g_bytes + 413, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[1], .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[41], - .data.refcounted = {g_bytes + 427, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[40], + .data.refcounted = {g_bytes + 416, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[0], .data.refcounted = {g_bytes + 0, 5}}, - {.refcount = &grpc_static_metadata_refcounts[42], - .data.refcounted = {g_bytes + 430, 1}}}, + {.refcount = &grpc_static_metadata_refcounts[41], + .data.refcounted = {g_bytes + 419, 1}}}, {{.refcount = &grpc_static_metadata_refcounts[0], .data.refcounted = {g_bytes + 0, 5}}, + {.refcount = &grpc_static_metadata_refcounts[42], + .data.refcounted = {g_bytes + 420, 11}}}, + {{.refcount = &grpc_static_metadata_refcounts[2], + .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[43], - .data.refcounted = {g_bytes + 431, 11}}}, + .data.refcounted = {g_bytes + 431, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[44], - .data.refcounted = {g_bytes + 442, 3}}}, + .data.refcounted = {g_bytes + 434, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[45], - .data.refcounted = {g_bytes + 445, 3}}}, + .data.refcounted = {g_bytes + 437, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[46], - .data.refcounted = {g_bytes + 448, 3}}}, + .data.refcounted = {g_bytes + 440, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[47], - .data.refcounted = {g_bytes + 451, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[48], - .data.refcounted = {g_bytes + 454, 3}}}, + .data.refcounted = {g_bytes + 443, 3}}}, + {{.refcount = &grpc_static_metadata_refcounts[48], + .data.refcounted = {g_bytes + 446, 14}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[49], - .data.refcounted = {g_bytes + 457, 14}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 471, 15}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 471, 15}}, - {.refcount = &grpc_static_metadata_refcounts[51], - .data.refcounted = {g_bytes + 486, 13}}}, + .data.refcounted = {g_bytes + 460, 15}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[49], + .data.refcounted = {g_bytes + 460, 15}}, + {.refcount = &grpc_static_metadata_refcounts[50], + .data.refcounted = {g_bytes + 475, 13}}}, + {{.refcount = &grpc_static_metadata_refcounts[51], + .data.refcounted = {g_bytes + 488, 15}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[52], - .data.refcounted = {g_bytes + 499, 15}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 503, 13}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[53], - .data.refcounted = {g_bytes + 514, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 516, 6}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[54], - .data.refcounted = {g_bytes + 527, 6}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 522, 27}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[55], - .data.refcounted = {g_bytes + 533, 27}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 549, 3}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[56], - .data.refcounted = {g_bytes + 560, 3}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 552, 5}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[57], - .data.refcounted = {g_bytes + 563, 5}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 557, 13}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[58], - .data.refcounted = {g_bytes + 568, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 570, 13}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[59], - .data.refcounted = {g_bytes + 581, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 583, 19}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[60], - .data.refcounted = {g_bytes + 594, 19}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 602, 16}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[61], - .data.refcounted = {g_bytes + 613, 16}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 618, 16}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[62], - .data.refcounted = {g_bytes + 629, 16}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 634, 14}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[63], - .data.refcounted = {g_bytes + 645, 14}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 648, 16}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[64], - .data.refcounted = {g_bytes + 659, 16}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[65], - .data.refcounted = {g_bytes + 675, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 664, 13}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[11], .data.refcounted = {g_bytes + 110, 12}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[65], + .data.refcounted = {g_bytes + 677, 6}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[66], - .data.refcounted = {g_bytes + 688, 6}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 683, 4}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[67], - .data.refcounted = {g_bytes + 694, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 687, 4}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[68], - .data.refcounted = {g_bytes + 698, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 691, 6}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[69], - .data.refcounted = {g_bytes + 702, 6}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 697, 7}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[70], - .data.refcounted = {g_bytes + 708, 7}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[71], - .data.refcounted = {g_bytes + 715, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 704, 4}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[14], .data.refcounted = {g_bytes + 162, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[71], + .data.refcounted = {g_bytes + 708, 8}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[72], - .data.refcounted = {g_bytes + 719, 8}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 716, 17}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[73], - .data.refcounted = {g_bytes + 727, 17}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 733, 13}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[74], - .data.refcounted = {g_bytes + 744, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 746, 8}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[75], - .data.refcounted = {g_bytes + 757, 8}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 754, 19}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[76], - .data.refcounted = {g_bytes + 765, 19}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[77], - .data.refcounted = {g_bytes + 784, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 773, 13}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[15], .data.refcounted = {g_bytes + 166, 8}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 174, 11}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[77], + .data.refcounted = {g_bytes + 786, 4}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 797, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 790, 8}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 801, 8}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 798, 12}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 809, 12}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 810, 18}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 821, 18}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 828, 19}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 839, 19}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 847, 5}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 858, 5}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 852, 7}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 863, 7}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 859, 7}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 870, 7}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 866, 11}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 877, 11}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 877, 6}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 888, 6}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 883, 10}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 894, 10}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 893, 25}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 904, 25}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 929, 17}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 918, 17}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[13], .data.refcounted = {g_bytes + 152, 10}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[90], + .data.refcounted = {g_bytes + 935, 4}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 946, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 939, 3}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 950, 3}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 953, 16}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 227, 0}}}, + .data.refcounted = {g_bytes + 942, 16}}, + {.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 216, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 358, 8}}}, + {.refcount = &grpc_static_metadata_refcounts[28], + .data.refcounted = {g_bytes + 347, 8}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 370, 7}}}, + {.refcount = &grpc_static_metadata_refcounts[30], + .data.refcounted = {g_bytes + 359, 7}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 969, 16}}}, + {.refcount = &grpc_static_metadata_refcounts[93], + .data.refcounted = {g_bytes + 958, 16}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 366, 4}}}, + {.refcount = &grpc_static_metadata_refcounts[29], + .data.refcounted = {g_bytes + 355, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 985, 13}}}, + {.refcount = &grpc_static_metadata_refcounts[94], + .data.refcounted = {g_bytes + 974, 13}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 998, 12}}}, + {.refcount = &grpc_static_metadata_refcounts[95], + .data.refcounted = {g_bytes + 987, 12}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1010, 21}}}, + {.refcount = &grpc_static_metadata_refcounts[96], + .data.refcounted = {g_bytes + 999, 21}}}, }; -const uint8_t grpc_static_accept_encoding_metadata[8] = {0, 74, 75, 76, - 77, 78, 79, 80}; +const uint8_t grpc_static_accept_encoding_metadata[8] = {0, 73, 74, 75, + 76, 77, 78, 79}; diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h index 7649ccd5d2d..f9600ee2e4e 100644 --- a/src/core/lib/transport/static_metadata.h +++ b/src/core/lib/transport/static_metadata.h @@ -44,7 +44,7 @@ #include "src/core/lib/transport/metadata.h" -#define GRPC_STATIC_MDSTR_COUNT 98 +#define GRPC_STATIC_MDSTR_COUNT 97 extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]; /* ":path" */ #define GRPC_MDSTR_PATH (grpc_static_slice_table[0]) @@ -78,174 +78,172 @@ extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]; #define GRPC_MDSTR_HOST (grpc_static_slice_table[14]) /* "lb-token" */ #define GRPC_MDSTR_LB_TOKEN (grpc_static_slice_table[15]) -/* "lb-cost-bin" */ -#define GRPC_MDSTR_LB_COST_BIN (grpc_static_slice_table[16]) /* "grpc-timeout" */ -#define GRPC_MDSTR_GRPC_TIMEOUT (grpc_static_slice_table[17]) +#define GRPC_MDSTR_GRPC_TIMEOUT (grpc_static_slice_table[16]) /* "grpc-tracing-bin" */ -#define GRPC_MDSTR_GRPC_TRACING_BIN (grpc_static_slice_table[18]) +#define GRPC_MDSTR_GRPC_TRACING_BIN (grpc_static_slice_table[17]) /* "grpc-stats-bin" */ -#define GRPC_MDSTR_GRPC_STATS_BIN (grpc_static_slice_table[19]) +#define GRPC_MDSTR_GRPC_STATS_BIN (grpc_static_slice_table[18]) /* "" */ -#define GRPC_MDSTR_EMPTY (grpc_static_slice_table[20]) +#define GRPC_MDSTR_EMPTY (grpc_static_slice_table[19]) /* "grpc.wait_for_ready" */ -#define GRPC_MDSTR_GRPC_DOT_WAIT_FOR_READY (grpc_static_slice_table[21]) +#define GRPC_MDSTR_GRPC_DOT_WAIT_FOR_READY (grpc_static_slice_table[20]) /* "grpc.timeout" */ -#define GRPC_MDSTR_GRPC_DOT_TIMEOUT (grpc_static_slice_table[22]) +#define GRPC_MDSTR_GRPC_DOT_TIMEOUT (grpc_static_slice_table[21]) /* "grpc.max_request_message_bytes" */ #define GRPC_MDSTR_GRPC_DOT_MAX_REQUEST_MESSAGE_BYTES \ - (grpc_static_slice_table[23]) + (grpc_static_slice_table[22]) /* "grpc.max_response_message_bytes" */ #define GRPC_MDSTR_GRPC_DOT_MAX_RESPONSE_MESSAGE_BYTES \ - (grpc_static_slice_table[24]) + (grpc_static_slice_table[23]) /* "/grpc.lb.v1.LoadBalancer/BalanceLoad" */ #define GRPC_MDSTR_SLASH_GRPC_DOT_LB_DOT_V1_DOT_LOADBALANCER_SLASH_BALANCELOAD \ - (grpc_static_slice_table[25]) + (grpc_static_slice_table[24]) /* "0" */ -#define GRPC_MDSTR_0 (grpc_static_slice_table[26]) +#define GRPC_MDSTR_0 (grpc_static_slice_table[25]) /* "1" */ -#define GRPC_MDSTR_1 (grpc_static_slice_table[27]) +#define GRPC_MDSTR_1 (grpc_static_slice_table[26]) /* "2" */ -#define GRPC_MDSTR_2 (grpc_static_slice_table[28]) +#define GRPC_MDSTR_2 (grpc_static_slice_table[27]) /* "identity" */ -#define GRPC_MDSTR_IDENTITY (grpc_static_slice_table[29]) +#define GRPC_MDSTR_IDENTITY (grpc_static_slice_table[28]) /* "gzip" */ -#define GRPC_MDSTR_GZIP (grpc_static_slice_table[30]) +#define GRPC_MDSTR_GZIP (grpc_static_slice_table[29]) /* "deflate" */ -#define GRPC_MDSTR_DEFLATE (grpc_static_slice_table[31]) +#define GRPC_MDSTR_DEFLATE (grpc_static_slice_table[30]) /* "trailers" */ -#define GRPC_MDSTR_TRAILERS (grpc_static_slice_table[32]) +#define GRPC_MDSTR_TRAILERS (grpc_static_slice_table[31]) /* "application/grpc" */ -#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (grpc_static_slice_table[33]) +#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (grpc_static_slice_table[32]) /* "POST" */ -#define GRPC_MDSTR_POST (grpc_static_slice_table[34]) +#define GRPC_MDSTR_POST (grpc_static_slice_table[33]) /* "200" */ -#define GRPC_MDSTR_200 (grpc_static_slice_table[35]) +#define GRPC_MDSTR_200 (grpc_static_slice_table[34]) /* "404" */ -#define GRPC_MDSTR_404 (grpc_static_slice_table[36]) +#define GRPC_MDSTR_404 (grpc_static_slice_table[35]) /* "http" */ -#define GRPC_MDSTR_HTTP (grpc_static_slice_table[37]) +#define GRPC_MDSTR_HTTP (grpc_static_slice_table[36]) /* "https" */ -#define GRPC_MDSTR_HTTPS (grpc_static_slice_table[38]) +#define GRPC_MDSTR_HTTPS (grpc_static_slice_table[37]) /* "grpc" */ -#define GRPC_MDSTR_GRPC (grpc_static_slice_table[39]) +#define GRPC_MDSTR_GRPC (grpc_static_slice_table[38]) /* "GET" */ -#define GRPC_MDSTR_GET (grpc_static_slice_table[40]) +#define GRPC_MDSTR_GET (grpc_static_slice_table[39]) /* "PUT" */ -#define GRPC_MDSTR_PUT (grpc_static_slice_table[41]) +#define GRPC_MDSTR_PUT (grpc_static_slice_table[40]) /* "/" */ -#define GRPC_MDSTR_SLASH (grpc_static_slice_table[42]) +#define GRPC_MDSTR_SLASH (grpc_static_slice_table[41]) /* "/index.html" */ -#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (grpc_static_slice_table[43]) +#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (grpc_static_slice_table[42]) /* "204" */ -#define GRPC_MDSTR_204 (grpc_static_slice_table[44]) +#define GRPC_MDSTR_204 (grpc_static_slice_table[43]) /* "206" */ -#define GRPC_MDSTR_206 (grpc_static_slice_table[45]) +#define GRPC_MDSTR_206 (grpc_static_slice_table[44]) /* "304" */ -#define GRPC_MDSTR_304 (grpc_static_slice_table[46]) +#define GRPC_MDSTR_304 (grpc_static_slice_table[45]) /* "400" */ -#define GRPC_MDSTR_400 (grpc_static_slice_table[47]) +#define GRPC_MDSTR_400 (grpc_static_slice_table[46]) /* "500" */ -#define GRPC_MDSTR_500 (grpc_static_slice_table[48]) +#define GRPC_MDSTR_500 (grpc_static_slice_table[47]) /* "accept-charset" */ -#define GRPC_MDSTR_ACCEPT_CHARSET (grpc_static_slice_table[49]) +#define GRPC_MDSTR_ACCEPT_CHARSET (grpc_static_slice_table[48]) /* "accept-encoding" */ -#define GRPC_MDSTR_ACCEPT_ENCODING (grpc_static_slice_table[50]) +#define GRPC_MDSTR_ACCEPT_ENCODING (grpc_static_slice_table[49]) /* "gzip, deflate" */ -#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (grpc_static_slice_table[51]) +#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (grpc_static_slice_table[50]) /* "accept-language" */ -#define GRPC_MDSTR_ACCEPT_LANGUAGE (grpc_static_slice_table[52]) +#define GRPC_MDSTR_ACCEPT_LANGUAGE (grpc_static_slice_table[51]) /* "accept-ranges" */ -#define GRPC_MDSTR_ACCEPT_RANGES (grpc_static_slice_table[53]) +#define GRPC_MDSTR_ACCEPT_RANGES (grpc_static_slice_table[52]) /* "accept" */ -#define GRPC_MDSTR_ACCEPT (grpc_static_slice_table[54]) +#define GRPC_MDSTR_ACCEPT (grpc_static_slice_table[53]) /* "access-control-allow-origin" */ -#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (grpc_static_slice_table[55]) +#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (grpc_static_slice_table[54]) /* "age" */ -#define GRPC_MDSTR_AGE (grpc_static_slice_table[56]) +#define GRPC_MDSTR_AGE (grpc_static_slice_table[55]) /* "allow" */ -#define GRPC_MDSTR_ALLOW (grpc_static_slice_table[57]) +#define GRPC_MDSTR_ALLOW (grpc_static_slice_table[56]) /* "authorization" */ -#define GRPC_MDSTR_AUTHORIZATION (grpc_static_slice_table[58]) +#define GRPC_MDSTR_AUTHORIZATION (grpc_static_slice_table[57]) /* "cache-control" */ -#define GRPC_MDSTR_CACHE_CONTROL (grpc_static_slice_table[59]) +#define GRPC_MDSTR_CACHE_CONTROL (grpc_static_slice_table[58]) /* "content-disposition" */ -#define GRPC_MDSTR_CONTENT_DISPOSITION (grpc_static_slice_table[60]) +#define GRPC_MDSTR_CONTENT_DISPOSITION (grpc_static_slice_table[59]) /* "content-encoding" */ -#define GRPC_MDSTR_CONTENT_ENCODING (grpc_static_slice_table[61]) +#define GRPC_MDSTR_CONTENT_ENCODING (grpc_static_slice_table[60]) /* "content-language" */ -#define GRPC_MDSTR_CONTENT_LANGUAGE (grpc_static_slice_table[62]) +#define GRPC_MDSTR_CONTENT_LANGUAGE (grpc_static_slice_table[61]) /* "content-length" */ -#define GRPC_MDSTR_CONTENT_LENGTH (grpc_static_slice_table[63]) +#define GRPC_MDSTR_CONTENT_LENGTH (grpc_static_slice_table[62]) /* "content-location" */ -#define GRPC_MDSTR_CONTENT_LOCATION (grpc_static_slice_table[64]) +#define GRPC_MDSTR_CONTENT_LOCATION (grpc_static_slice_table[63]) /* "content-range" */ -#define GRPC_MDSTR_CONTENT_RANGE (grpc_static_slice_table[65]) +#define GRPC_MDSTR_CONTENT_RANGE (grpc_static_slice_table[64]) /* "cookie" */ -#define GRPC_MDSTR_COOKIE (grpc_static_slice_table[66]) +#define GRPC_MDSTR_COOKIE (grpc_static_slice_table[65]) /* "date" */ -#define GRPC_MDSTR_DATE (grpc_static_slice_table[67]) +#define GRPC_MDSTR_DATE (grpc_static_slice_table[66]) /* "etag" */ -#define GRPC_MDSTR_ETAG (grpc_static_slice_table[68]) +#define GRPC_MDSTR_ETAG (grpc_static_slice_table[67]) /* "expect" */ -#define GRPC_MDSTR_EXPECT (grpc_static_slice_table[69]) +#define GRPC_MDSTR_EXPECT (grpc_static_slice_table[68]) /* "expires" */ -#define GRPC_MDSTR_EXPIRES (grpc_static_slice_table[70]) +#define GRPC_MDSTR_EXPIRES (grpc_static_slice_table[69]) /* "from" */ -#define GRPC_MDSTR_FROM (grpc_static_slice_table[71]) +#define GRPC_MDSTR_FROM (grpc_static_slice_table[70]) /* "if-match" */ -#define GRPC_MDSTR_IF_MATCH (grpc_static_slice_table[72]) +#define GRPC_MDSTR_IF_MATCH (grpc_static_slice_table[71]) /* "if-modified-since" */ -#define GRPC_MDSTR_IF_MODIFIED_SINCE (grpc_static_slice_table[73]) +#define GRPC_MDSTR_IF_MODIFIED_SINCE (grpc_static_slice_table[72]) /* "if-none-match" */ -#define GRPC_MDSTR_IF_NONE_MATCH (grpc_static_slice_table[74]) +#define GRPC_MDSTR_IF_NONE_MATCH (grpc_static_slice_table[73]) /* "if-range" */ -#define GRPC_MDSTR_IF_RANGE (grpc_static_slice_table[75]) +#define GRPC_MDSTR_IF_RANGE (grpc_static_slice_table[74]) /* "if-unmodified-since" */ -#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (grpc_static_slice_table[76]) +#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (grpc_static_slice_table[75]) /* "last-modified" */ -#define GRPC_MDSTR_LAST_MODIFIED (grpc_static_slice_table[77]) +#define GRPC_MDSTR_LAST_MODIFIED (grpc_static_slice_table[76]) /* "link" */ -#define GRPC_MDSTR_LINK (grpc_static_slice_table[78]) +#define GRPC_MDSTR_LINK (grpc_static_slice_table[77]) /* "location" */ -#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[79]) +#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[78]) /* "max-forwards" */ -#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[80]) +#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[79]) /* "proxy-authenticate" */ -#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[81]) +#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[80]) /* "proxy-authorization" */ -#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[82]) +#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[81]) /* "range" */ -#define GRPC_MDSTR_RANGE (grpc_static_slice_table[83]) +#define GRPC_MDSTR_RANGE (grpc_static_slice_table[82]) /* "referer" */ -#define GRPC_MDSTR_REFERER (grpc_static_slice_table[84]) +#define GRPC_MDSTR_REFERER (grpc_static_slice_table[83]) /* "refresh" */ -#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[85]) +#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[84]) /* "retry-after" */ -#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[86]) +#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[85]) /* "server" */ -#define GRPC_MDSTR_SERVER (grpc_static_slice_table[87]) +#define GRPC_MDSTR_SERVER (grpc_static_slice_table[86]) /* "set-cookie" */ -#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[88]) +#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[87]) /* "strict-transport-security" */ -#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[89]) +#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[88]) /* "transfer-encoding" */ -#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[90]) +#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[89]) /* "vary" */ -#define GRPC_MDSTR_VARY (grpc_static_slice_table[91]) +#define GRPC_MDSTR_VARY (grpc_static_slice_table[90]) /* "via" */ -#define GRPC_MDSTR_VIA (grpc_static_slice_table[92]) +#define GRPC_MDSTR_VIA (grpc_static_slice_table[91]) /* "www-authenticate" */ -#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[93]) +#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[92]) /* "identity,deflate" */ -#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[94]) +#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[93]) /* "identity,gzip" */ -#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[95]) +#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[94]) /* "deflate,gzip" */ -#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[96]) +#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[95]) /* "identity,deflate,gzip" */ #define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \ - (grpc_static_slice_table[97]) + (grpc_static_slice_table[96]) extern const grpc_slice_refcount_vtable grpc_static_metadata_vtable; extern grpc_slice_refcount @@ -257,7 +255,7 @@ extern grpc_slice_refcount #define GRPC_STATIC_METADATA_INDEX(static_slice) \ ((int)((static_slice).refcount - grpc_static_metadata_refcounts)) -#define GRPC_STATIC_MDELEM_COUNT 81 +#define GRPC_STATIC_MDELEM_COUNT 80 extern grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT]; extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; /* "grpc-status": "0" */ @@ -428,81 +426,78 @@ extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; /* "lb-token": "" */ #define GRPC_MDELEM_LB_TOKEN_EMPTY \ (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[55], GRPC_MDELEM_STORAGE_STATIC)) -/* "lb-cost-bin": "" */ -#define GRPC_MDELEM_LB_COST_BIN_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[56], GRPC_MDELEM_STORAGE_STATIC)) /* "link": "" */ #define GRPC_MDELEM_LINK_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[57], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[56], GRPC_MDELEM_STORAGE_STATIC)) /* "location": "" */ #define GRPC_MDELEM_LOCATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[58], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[57], GRPC_MDELEM_STORAGE_STATIC)) /* "max-forwards": "" */ #define GRPC_MDELEM_MAX_FORWARDS_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[59], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[58], GRPC_MDELEM_STORAGE_STATIC)) /* "proxy-authenticate": "" */ #define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[60], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[59], GRPC_MDELEM_STORAGE_STATIC)) /* "proxy-authorization": "" */ #define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[61], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[60], GRPC_MDELEM_STORAGE_STATIC)) /* "range": "" */ #define GRPC_MDELEM_RANGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[62], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[61], GRPC_MDELEM_STORAGE_STATIC)) /* "referer": "" */ #define GRPC_MDELEM_REFERER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[63], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[62], GRPC_MDELEM_STORAGE_STATIC)) /* "refresh": "" */ #define GRPC_MDELEM_REFRESH_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[64], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[63], GRPC_MDELEM_STORAGE_STATIC)) /* "retry-after": "" */ #define GRPC_MDELEM_RETRY_AFTER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[65], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[64], GRPC_MDELEM_STORAGE_STATIC)) /* "server": "" */ #define GRPC_MDELEM_SERVER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[66], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[65], GRPC_MDELEM_STORAGE_STATIC)) /* "set-cookie": "" */ #define GRPC_MDELEM_SET_COOKIE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[67], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[66], GRPC_MDELEM_STORAGE_STATIC)) /* "strict-transport-security": "" */ #define GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[68], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[67], GRPC_MDELEM_STORAGE_STATIC)) /* "transfer-encoding": "" */ #define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[69], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[68], GRPC_MDELEM_STORAGE_STATIC)) /* "user-agent": "" */ #define GRPC_MDELEM_USER_AGENT_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[70], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[69], GRPC_MDELEM_STORAGE_STATIC)) /* "vary": "" */ #define GRPC_MDELEM_VARY_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[71], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[70], GRPC_MDELEM_STORAGE_STATIC)) /* "via": "" */ #define GRPC_MDELEM_VIA_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[72], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[71], GRPC_MDELEM_STORAGE_STATIC)) /* "www-authenticate": "" */ #define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[73], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[72], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[74], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[73], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "deflate" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[75], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[74], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,deflate" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[76], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[75], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[77], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[76], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[78], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[77], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "deflate,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[79], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[78], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,deflate,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[80], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[79], GRPC_MDELEM_STORAGE_STATIC)) grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b); typedef enum { @@ -522,7 +517,6 @@ typedef enum { GRPC_BATCH_USER_AGENT, GRPC_BATCH_HOST, GRPC_BATCH_LB_TOKEN, - GRPC_BATCH_LB_COST_BIN, GRPC_BATCH_CALLOUTS_COUNT } grpc_metadata_batch_callouts_index; @@ -545,7 +539,6 @@ typedef union { struct grpc_linked_mdelem *user_agent; struct grpc_linked_mdelem *host; struct grpc_linked_mdelem *lb_token; - struct grpc_linked_mdelem *lb_cost_bin; } named; } grpc_metadata_batch_callouts; diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index a7aaa255724..6edbc909277 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -224,4 +224,20 @@ const struct census_context* ServerContext::census_context() const { return grpc_census_call_get_context(call_); } +void ServerContext::SetLoadReportingCosts( + const std::vector& cost_data) { + if (call_ == nullptr) return; + grpc_load_reporting_cost_context* cost_ctx = + static_cast( + gpr_malloc(sizeof(*cost_ctx))); + cost_ctx->values_count = cost_data.size(); + cost_ctx->values = static_cast( + gpr_malloc(sizeof(*cost_ctx->values) * cost_ctx->values_count)); + for (size_t i = 0; i < cost_ctx->values_count; ++i) { + cost_ctx->values[i] = + grpc_slice_from_copied_buffer(cost_data[i].data(), cost_data[i].size()); + } + grpc_call_set_load_reporting_cost_context(call_, cost_ctx); +} + } // namespace grpc diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index d8d76cc1c53..b8af8008336 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -107,6 +107,7 @@ grpc_channel_register_call_type grpc_channel_register_call_import; grpc_channel_create_registered_call_type grpc_channel_create_registered_call_import; grpc_call_start_batch_type grpc_call_start_batch_import; grpc_call_get_peer_type grpc_call_get_peer_import; +grpc_call_set_load_reporting_cost_context_type grpc_call_set_load_reporting_cost_context_import; grpc_census_call_set_context_type grpc_census_call_set_context_import; grpc_census_call_get_context_type grpc_census_call_get_context_import; grpc_channel_get_target_type grpc_channel_get_target_import; @@ -398,6 +399,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_channel_create_registered_call_import = (grpc_channel_create_registered_call_type) GetProcAddress(library, "grpc_channel_create_registered_call"); grpc_call_start_batch_import = (grpc_call_start_batch_type) GetProcAddress(library, "grpc_call_start_batch"); grpc_call_get_peer_import = (grpc_call_get_peer_type) GetProcAddress(library, "grpc_call_get_peer"); + grpc_call_set_load_reporting_cost_context_import = (grpc_call_set_load_reporting_cost_context_type) GetProcAddress(library, "grpc_call_set_load_reporting_cost_context"); grpc_census_call_set_context_import = (grpc_census_call_set_context_type) GetProcAddress(library, "grpc_census_call_set_context"); grpc_census_call_get_context_import = (grpc_census_call_get_context_type) GetProcAddress(library, "grpc_census_call_get_context"); grpc_channel_get_target_import = (grpc_channel_get_target_type) GetProcAddress(library, "grpc_channel_get_target"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 4eab9639ea7..3229e2a77dc 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -272,6 +272,9 @@ extern grpc_call_start_batch_type grpc_call_start_batch_import; typedef char *(*grpc_call_get_peer_type)(grpc_call *call); extern grpc_call_get_peer_type grpc_call_get_peer_import; #define grpc_call_get_peer grpc_call_get_peer_import +typedef void(*grpc_call_set_load_reporting_cost_context_type)(grpc_call *call, struct grpc_load_reporting_cost_context *context); +extern grpc_call_set_load_reporting_cost_context_type grpc_call_set_load_reporting_cost_context_import; +#define grpc_call_set_load_reporting_cost_context grpc_call_set_load_reporting_cost_context_import typedef void(*grpc_census_call_set_context_type)(grpc_call *call, struct census_context *context); extern grpc_census_call_set_context_type grpc_census_call_set_context_import; #define grpc_census_call_set_context grpc_census_call_set_context_import diff --git a/test/core/end2end/fuzzers/hpack.dictionary b/test/core/end2end/fuzzers/hpack.dictionary index 81a2419d120..6b967854191 100644 --- a/test/core/end2end/fuzzers/hpack.dictionary +++ b/test/core/end2end/fuzzers/hpack.dictionary @@ -15,7 +15,6 @@ "\x0Auser-agent" "\x04host" "\x08lb-token" -"\x0Blb-cost-bin" "\x0Cgrpc-timeout" "\x10grpc-tracing-bin" "\x0Egrpc-stats-bin" @@ -153,7 +152,6 @@ "\x00\x13if-unmodified-since\x00" "\x00\x0Dlast-modified\x00" "\x00\x08lb-token\x00" -"\x00\x0Blb-cost-bin\x00" "\x00\x04link\x00" "\x00\x08location\x00" "\x00\x0Cmax-forwards\x00" diff --git a/test/core/end2end/tests/load_reporting_hook.c b/test/core/end2end/tests/load_reporting_hook.c index 085a563fb8f..d1ee26fe502 100644 --- a/test/core/end2end/tests/load_reporting_hook.c +++ b/test/core/end2end/tests/load_reporting_hook.c @@ -31,23 +31,24 @@ * */ -#include "test/core/end2end/end2end_tests.h" - #include #include +#include #include #include #include #include #include -#include "test/core/end2end/cq_verifier.h" #include "src/core/ext/load_reporting/load_reporting.h" #include "src/core/ext/load_reporting/load_reporting_filter.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/transport/static_metadata.h" +#include "test/core/end2end/cq_verifier.h" +#include "test/core/end2end/end2end_tests.h" + enum { TIMEOUT = 200000 }; static void *tag(intptr_t t) { return (void *)t; } @@ -124,7 +125,8 @@ static void end_test(grpc_end2end_test_fixture *f) { static void request_response_with_payload( grpc_end2end_test_config config, grpc_end2end_test_fixture f, const char *method_name, const char *request_msg, const char *response_msg, - grpc_metadata *initial_lr_metadata, grpc_metadata *trailing_lr_metadata) { + grpc_metadata *initial_lr_metadata, + grpc_load_reporting_cost_context *cost_ctx) { grpc_slice request_payload_slice = grpc_slice_from_static_string(request_msg); grpc_slice response_payload_slice = grpc_slice_from_static_string(response_msg); @@ -237,9 +239,8 @@ static void request_response_with_payload( op->reserved = NULL; op++; op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; - GPR_ASSERT(trailing_lr_metadata != NULL); - op->data.send_status_from_server.trailing_metadata_count = 1; - op->data.send_status_from_server.trailing_metadata = trailing_lr_metadata; + GPR_ASSERT(cost_ctx != NULL); + grpc_call_set_load_reporting_cost_context(s, cost_ctx); op->data.send_status_from_server.status = GRPC_STATUS_OK; grpc_slice status_details = grpc_slice_from_static_string("xyz"); op->data.send_status_from_server.status_details = &status_details; @@ -293,21 +294,21 @@ static void test_load_reporting_hook(grpc_end2end_test_config config) { const char *response_msg = "... and the response from the server"; grpc_metadata initial_lr_metadata; - grpc_metadata trailing_lr_metadata; initial_lr_metadata.key = GRPC_MDSTR_LB_TOKEN; initial_lr_metadata.value = grpc_slice_from_static_string("client-token"); memset(&initial_lr_metadata.internal_data, 0, sizeof(initial_lr_metadata.internal_data)); - trailing_lr_metadata.key = GRPC_MDSTR_LB_COST_BIN; - trailing_lr_metadata.value = grpc_slice_from_static_string("server-token"); - memset(&trailing_lr_metadata.internal_data, 0, - sizeof(trailing_lr_metadata.internal_data)); + grpc_load_reporting_cost_context *cost_ctx = gpr_malloc(sizeof(*cost_ctx)); + memset(cost_ctx, 0, sizeof(*cost_ctx)); + cost_ctx->values_count = 1; + cost_ctx->values = + gpr_malloc(sizeof(*cost_ctx->values) * cost_ctx->values_count); + cost_ctx->values[0] = grpc_slice_from_static_string("cost-token"); request_response_with_payload(config, f, method_name, request_msg, - response_msg, &initial_lr_metadata, - &trailing_lr_metadata); + response_msg, &initial_lr_metadata, cost_ctx); end_test(&f); { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index e0a2c94216d..330da468490 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index 4703708aaa2..109701f740b 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -123,7 +123,6 @@ CONFIG = [ ('if-unmodified-since', ''), ('last-modified', ''), ('lb-token', ''), - ('lb-cost-bin', ''), ('link', ''), ('location', ''), ('max-forwards', ''), @@ -160,7 +159,6 @@ METADATA_BATCH_CALLOUTS = [ 'user-agent', 'host', 'lb-token', - 'lb-cost-bin', ] COMPRESSION_ALGORITHMS = [ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 060e09869d9..896ee0213b3 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -827,6 +827,7 @@ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ include/grpc/impl/codegen/sync_windows.h \ +include/grpc/load_reporting.h \ include/grpc/slice.h \ include/grpc/slice_buffer.h \ include/grpc/status.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index ce7e1036478..10801254ef9 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -827,6 +827,7 @@ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ include/grpc/impl/codegen/sync_windows.h \ +include/grpc/load_reporting.h \ include/grpc/slice.h \ include/grpc/slice_buffer.h \ include/grpc/status.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 046076bd105..cd96e051698 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -5472,23 +5472,6 @@ "third_party": false, "type": "lib" }, - { - "deps": [ - "grpc++" - ], - "headers": [ - "include/grpc++/test/server_context_test_spouse.h" - ], - "is_filegroup": false, - "language": "c++", - "name": "grpc++_test", - "src": [ - "include/grpc++/test/server_context_test_spouse.h", - "src/cpp/test/server_context_test_spouse.cc" - ], - "third_party": false, - "type": "lib" - }, { "deps": [], "headers": [ @@ -7081,6 +7064,7 @@ "include/grpc/grpc.h", "include/grpc/grpc_posix.h", "include/grpc/grpc_security_constants.h", + "include/grpc/load_reporting.h", "include/grpc/slice.h", "include/grpc/slice_buffer.h", "include/grpc/status.h", @@ -7201,6 +7185,7 @@ "include/grpc/grpc.h", "include/grpc/grpc_posix.h", "include/grpc/grpc_security_constants.h", + "include/grpc/load_reporting.h", "include/grpc/slice.h", "include/grpc/slice_buffer.h", "include/grpc/status.h", @@ -8354,6 +8339,22 @@ "third_party": false, "type": "filegroup" }, + { + "deps": [ + "grpc++" + ], + "headers": [ + "include/grpc++/test/server_context_test_spouse.h" + ], + "is_filegroup": true, + "language": "c++", + "name": "grpc++_test", + "src": [ + "include/grpc++/test/server_context_test_spouse.h" + ], + "third_party": false, + "type": "filegroup" + }, { "deps": [ "grpc++_codegen_base" diff --git a/vsprojects/vcxproj/grpc++_test/grpc++_test.vcxproj b/vsprojects/vcxproj/grpc++_test/grpc++_test.vcxproj deleted file mode 100644 index cd3e406f9dc..00000000000 --- a/vsprojects/vcxproj/grpc++_test/grpc++_test.vcxproj +++ /dev/null @@ -1,170 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {3D3EAEA9-76C4-0CFE-4718-5A1F6B7F72C8} - true - $(SolutionDir)IntDir\$(MSBuildProjectName)\ - - - - v100 - - - v110 - - - v120 - - - v140 - - - StaticLibrary - true - Unicode - - - StaticLibrary - false - true - Unicode - - - - - - - - - - - - grpc++_test - - - grpc++_test - - - - NotUsing - Level3 - Disabled - WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) - true - MultiThreadedDebug - true - None - false - - - Windows - true - false - - - - - - NotUsing - Level3 - Disabled - WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) - true - MultiThreadedDebug - true - None - false - - - Windows - true - false - - - - - - NotUsing - Level3 - MaxSpeed - WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) - true - true - true - MultiThreaded - true - None - false - - - Windows - true - false - true - true - - - - - - NotUsing - Level3 - MaxSpeed - WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) - true - true - true - MultiThreaded - true - None - false - - - Windows - true - false - true - true - - - - - - - - - - - - - {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} - - - - - - - - This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - - diff --git a/vsprojects/vcxproj/grpc++_test/grpc++_test.vcxproj.filters b/vsprojects/vcxproj/grpc++_test/grpc++_test.vcxproj.filters deleted file mode 100644 index e1d8fe1ef32..00000000000 --- a/vsprojects/vcxproj/grpc++_test/grpc++_test.vcxproj.filters +++ /dev/null @@ -1,35 +0,0 @@ - - - - - src\cpp\test - - - - - include\grpc++\test - - - - - - {418e2be9-9a04-af8a-8878-956543c507a1} - - - {bcaf3a2d-e884-bfbd-477b-09634054bcc5} - - - {3a14b66f-1d31-2d06-e9a1-c6d1199ab04d} - - - {bc08c75b-ee7e-85a0-fb0f-7586bbd33c6f} - - - {9797a57d-93ac-ff9c-5ac6-fd335777d782} - - - {58d4b4e0-5617-abf4-48a6-f1264d127b27} - - - - diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 73d511ef7ce..fde60be3e20 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -273,6 +273,7 @@ + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index dae55041a0b..8edbbc22bed 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -696,6 +696,9 @@ include\grpc + + include\grpc + include\grpc diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 57cbb2c496b..17fb026cf07 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -153,6 +153,7 @@ + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 620649a06c4..5331ede4eaa 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -438,6 +438,9 @@ include\grpc + + include\grpc + include\grpc diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 5bdbff9a6ac..22f4740b8fa 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -264,6 +264,7 @@ + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index b2b46b052b7..5021cb47d8d 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -609,6 +609,9 @@ include\grpc + + include\grpc + include\grpc diff --git a/vsprojects/vcxproj/test/server_context_test_spouse_test/server_context_test_spouse_test.vcxproj b/vsprojects/vcxproj/test/server_context_test_spouse_test/server_context_test_spouse_test.vcxproj index 9e586c647c7..fe2b1abc152 100644 --- a/vsprojects/vcxproj/test/server_context_test_spouse_test/server_context_test_spouse_test.vcxproj +++ b/vsprojects/vcxproj/test/server_context_test_spouse_test/server_context_test_spouse_test.vcxproj @@ -167,9 +167,6 @@ {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} - - {3D3EAEA9-76C4-0CFE-4718-5A1F6B7F72C8} - {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} From fe5f497f77b1800f837e79c595aeedb62e0719b7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Feb 2017 16:26:27 -0800 Subject: [PATCH 120/167] Add a test that measures flow control stalls in a deterministic way --- CMakeLists.txt | 2 + Makefile | 2 + build.yaml | 2 + .../chttp2/transport/frame_settings.h | 2 +- test/core/util/trickle_endpoint.c | 178 +++++++++++++++++ test/core/util/trickle_endpoint.h | 45 +++++ test/cpp/microbenchmarks/bm_fullstack.cc | 180 +++++++++++++++++- .../generated/sources_and_headers.json | 7 +- .../grpc_test_util/grpc_test_util.vcxproj | 3 + .../grpc_test_util.vcxproj.filters | 6 + .../grpc_test_util_unsecure.vcxproj | 3 + .../grpc_test_util_unsecure.vcxproj.filters | 6 + 12 files changed, 423 insertions(+), 13 deletions(-) create mode 100644 test/core/util/trickle_endpoint.c create mode 100644 test/core/util/trickle_endpoint.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 101f0ad2ef9..8edc29e9cc9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1428,6 +1428,7 @@ add_library(grpc_test_util test/core/util/port_uv.c test/core/util/port_windows.c test/core/util/slice_splitter.c + test/core/util/trickle_endpoint.c src/core/lib/channel/channel_args.c src/core/lib/channel/channel_stack.c src/core/lib/channel/channel_stack_builder.c @@ -1635,6 +1636,7 @@ add_library(grpc_test_util_unsecure test/core/util/port_uv.c test/core/util/port_windows.c test/core/util/slice_splitter.c + test/core/util/trickle_endpoint.c ) if(WIN32 AND MSVC) diff --git a/Makefile b/Makefile index 5dde577cca8..9bae21dc455 100644 --- a/Makefile +++ b/Makefile @@ -3262,6 +3262,7 @@ LIBGRPC_TEST_UTIL_SRC = \ test/core/util/port_uv.c \ test/core/util/port_windows.c \ test/core/util/slice_splitter.c \ + test/core/util/trickle_endpoint.c \ src/core/lib/channel/channel_args.c \ src/core/lib/channel/channel_stack.c \ src/core/lib/channel/channel_stack_builder.c \ @@ -3462,6 +3463,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ test/core/util/port_uv.c \ test/core/util/port_windows.c \ test/core/util/slice_splitter.c \ + test/core/util/trickle_endpoint.c \ PUBLIC_HEADERS_C += \ diff --git a/build.yaml b/build.yaml index 2fbaa2e2262..2f390b93198 100644 --- a/build.yaml +++ b/build.yaml @@ -596,6 +596,7 @@ filegroups: - test/core/util/port.h - test/core/util/port_server_client.h - test/core/util/slice_splitter.h + - test/core/util/trickle_endpoint.h src: - test/core/end2end/cq_verifier.c - test/core/end2end/fake_resolver.c @@ -613,6 +614,7 @@ filegroups: - test/core/util/port_uv.c - test/core/util/port_windows.c - test/core/util/slice_splitter.c + - test/core/util/trickle_endpoint.c deps: - grpc - gpr_test_util diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.h b/src/core/ext/transport/chttp2/transport/frame_settings.h index a29dc821060..44137798c06 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.h +++ b/src/core/ext/transport/chttp2/transport/frame_settings.h @@ -87,7 +87,7 @@ extern const grpc_chttp2_setting_parameters grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS]; /* Create a settings frame by diffing old & new, and updating old to be new */ -grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, +grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *newval, uint32_t force_mask, size_t count); /* Create an ack settings frame */ grpc_slice grpc_chttp2_settings_ack_create(void); diff --git a/test/core/util/trickle_endpoint.c b/test/core/util/trickle_endpoint.c new file mode 100644 index 00000000000..8d661e04c69 --- /dev/null +++ b/test/core/util/trickle_endpoint.c @@ -0,0 +1,178 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "test/core/util/passthru_endpoint.h" + +#include +#include + +#include +#include +#include +#include + +#include "src/core/lib/iomgr/sockaddr.h" + +#include "src/core/lib/slice/slice_internal.h" + +typedef struct { + grpc_endpoint base; + grpc_endpoint *wrapped; + + gpr_mu mu; + grpc_slice_buffer write_buffer; + grpc_slice_buffer writing_buffer; + grpc_error *error; + bool writing; +} trickle_endpoint; + +static void te_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_slice_buffer *slices, grpc_closure *cb) { + trickle_endpoint *te = (trickle_endpoint *)ep; + grpc_endpoint_read(exec_ctx, te->wrapped, slices, cb); +} + +static void te_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_slice_buffer *slices, grpc_closure *cb) { + trickle_endpoint *te = (trickle_endpoint *)ep; + for (size_t i = 0; i < slices->count; i++) { + grpc_slice_ref_internal(slices->slices[i]); + } + grpc_slice_buffer_addn(&te->write_buffer, slices->slices, slices->count); + gpr_mu_lock(&te->mu); + grpc_closure_sched(exec_ctx, cb, GRPC_ERROR_REF(te->error)); + gpr_mu_unlock(&te->mu); +} + +static grpc_workqueue *te_get_workqueue(grpc_endpoint *ep) { + trickle_endpoint *te = (trickle_endpoint *)ep; + return grpc_endpoint_get_workqueue(te->wrapped); +} + +static void te_add_to_pollset(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_pollset *pollset) { + trickle_endpoint *te = (trickle_endpoint *)ep; + grpc_endpoint_add_to_pollset(exec_ctx, te->wrapped, pollset); +} + +static void te_add_to_pollset_set(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_pollset_set *pollset_set) { + trickle_endpoint *te = (trickle_endpoint *)ep; + grpc_endpoint_add_to_pollset_set(exec_ctx, te->wrapped, pollset_set); +} + +static void te_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_error *why) { + trickle_endpoint *te = (trickle_endpoint *)ep; + gpr_mu_lock(&te->mu); + if (te->error == GRPC_ERROR_NONE) { + te->error = GRPC_ERROR_REF(why); + } + gpr_mu_unlock(&te->mu); + grpc_endpoint_shutdown(exec_ctx, te->wrapped, why); +} + +static void te_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { + trickle_endpoint *te = (trickle_endpoint *)ep; + grpc_endpoint_destroy(exec_ctx, te->wrapped); + gpr_mu_destroy(&te->mu); + grpc_slice_buffer_destroy_internal(exec_ctx, &te->write_buffer); + grpc_slice_buffer_destroy_internal(exec_ctx, &te->writing_buffer); + GRPC_ERROR_UNREF(te->error); + gpr_free(te); +} + +static grpc_resource_user *te_get_resource_user(grpc_endpoint *ep) { + trickle_endpoint *te = (trickle_endpoint *)ep; + return grpc_endpoint_get_resource_user(te->wrapped); +} + +static char *te_get_peer(grpc_endpoint *ep) { + trickle_endpoint *te = (trickle_endpoint *)ep; + return grpc_endpoint_get_peer(te->wrapped); +} + +static int te_get_fd(grpc_endpoint *ep) { + trickle_endpoint *te = (trickle_endpoint *)ep; + return grpc_endpoint_get_fd(te->wrapped); +} + +static void te_finish_write(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + trickle_endpoint *te = arg; + gpr_mu_lock(&te->mu); + te->writing = false; + grpc_slice_buffer_reset_and_unref(&te->writing_buffer); + gpr_mu_unlock(&te->mu); +} + +static const grpc_endpoint_vtable vtable = {te_read, + te_write, + te_get_workqueue, + te_add_to_pollset, + te_add_to_pollset_set, + te_shutdown, + te_destroy, + te_get_resource_user, + te_get_peer, + te_get_fd}; + +grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap) { + trickle_endpoint *te = gpr_malloc(sizeof(*te)); + te->base.vtable = &vtable; + te->wrapped = wrap; + gpr_mu_init(&te->mu); + grpc_slice_buffer_init(&te->write_buffer); + grpc_slice_buffer_init(&te->writing_buffer); + te->error = GRPC_ERROR_NONE; + te->writing = false; + return &te->base; +} + +size_t grpc_trickle_endpoint_trickle(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + size_t bytes) { + trickle_endpoint *te = (trickle_endpoint *)ep; + gpr_mu_lock(&te->mu); + if (bytes > 0 && !te->writing) { + grpc_slice_buffer_move_first(&te->write_buffer, + GPR_MIN(bytes, te->write_buffer.length), + &te->writing_buffer); + te->writing = true; + grpc_endpoint_write( + exec_ctx, te->wrapped, &te->writing_buffer, + grpc_closure_create(te_finish_write, te, grpc_schedule_on_exec_ctx)); + } + size_t backlog = te->write_buffer.length; + gpr_mu_unlock(&te->mu); + return backlog; +} diff --git a/test/core/util/trickle_endpoint.h b/test/core/util/trickle_endpoint.h new file mode 100644 index 00000000000..5f16818ebb7 --- /dev/null +++ b/test/core/util/trickle_endpoint.h @@ -0,0 +1,45 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef TRICKLE_ENDPOINT_H +#define TRICKLE_ENDPOINT_H + +#include "src/core/lib/iomgr/endpoint.h" + +grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap); + +/* Allow up to \a bytes through the endpoint. Returns the new backlog. */ +size_t grpc_trickle_endpoint_trickle(grpc_exec_ctx *exec_ctx, + grpc_endpoint *endpoint, size_t bytes); + +#endif diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index 9d883e68d7f..f83c3599616 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -46,6 +46,7 @@ extern "C" { #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" +#include "src/core/ext/transport/chttp2/transport/internal.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/endpoint_pair.h" @@ -57,6 +58,7 @@ extern "C" { #include "test/core/util/memory_counters.h" #include "test/core/util/passthru_endpoint.h" #include "test/core/util/port.h" +#include "test/core/util/trickle_endpoint.h" } #include "src/core/lib/profiling/timers.h" #include "src/cpp/client/create_channel_internal.h" @@ -197,7 +199,8 @@ class UDS : public FullstackFixture { class EndpointPairFixture : public BaseFixture { public: - EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) { + EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) + : endpoint_pair_(endpoints) { ServerBuilder b; cq_ = b.AddCompletionQueue(true); b.RegisterService(service); @@ -210,7 +213,7 @@ class EndpointPairFixture : public BaseFixture { { const grpc_channel_args* server_args = grpc_server_get_channel_args(server_->c_server()); - grpc_transport* transport = grpc_create_chttp2_transport( + server_transport_ = grpc_create_chttp2_transport( &exec_ctx, server_args, endpoints.server, 0 /* is_client */); grpc_pollset** pollsets; @@ -221,9 +224,9 @@ class EndpointPairFixture : public BaseFixture { grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]); } - grpc_server_setup_transport(&exec_ctx, server_->c_server(), transport, - NULL, server_args); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); + grpc_server_setup_transport(&exec_ctx, server_->c_server(), + server_transport_, NULL, server_args); + grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL); } /* create channel */ @@ -233,12 +236,13 @@ class EndpointPairFixture : public BaseFixture { ApplyCommonChannelArguments(&args); grpc_channel_args c_args = args.c_channel_args(); - grpc_transport* transport = + client_transport_ = grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1); - GPR_ASSERT(transport); - grpc_channel* channel = grpc_channel_create( - &exec_ctx, "target", &c_args, GRPC_CLIENT_DIRECT_CHANNEL, transport); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); + GPR_ASSERT(client_transport_); + grpc_channel* channel = + grpc_channel_create(&exec_ctx, "target", &c_args, + GRPC_CLIENT_DIRECT_CHANNEL, client_transport_); + grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL); channel_ = CreateChannelInternal("", channel); } @@ -258,6 +262,11 @@ class EndpointPairFixture : public BaseFixture { ServerCompletionQueue* cq() { return cq_.get(); } std::shared_ptr channel() { return channel_; } + protected: + grpc_endpoint_pair endpoint_pair_; + grpc_transport* client_transport_; + grpc_transport* server_transport_; + private: std::unique_ptr server_; std::unique_ptr cq_; @@ -295,6 +304,74 @@ class InProcessCHTTP2 : public EndpointPairFixture { } }; +class TrickledCHTTP2 : public EndpointPairFixture { + public: + TrickledCHTTP2(Service* service) + : EndpointPairFixture(service, MakeEndpoints()) {} + + void AddToLabel(std::ostream& out, benchmark::State& state) { + out << " writes/iter:" + << ((double)stats_.num_writes / (double)state.iterations()) + << " cli-transport-stalls/iter:" + << ((double) + client_stats_.streams_stalled_due_to_transport_flow_control / + (double)state.iterations()) + << " cli-stream-stalls/iter:" + << ((double)client_stats_.streams_stalled_due_to_stream_flow_control / + (double)state.iterations()) + << " svr-transport-stalls/iter:" + << ((double) + server_stats_.streams_stalled_due_to_transport_flow_control / + (double)state.iterations()) + << " svr-stream-stalls/iter:" + << ((double)server_stats_.streams_stalled_due_to_stream_flow_control / + (double)state.iterations()); + } + + void Step(size_t write_size) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + size_t client_backlog = grpc_trickle_endpoint_trickle( + &exec_ctx, endpoint_pair_.client, write_size); + size_t server_backlog = grpc_trickle_endpoint_trickle( + &exec_ctx, endpoint_pair_.server, write_size); + grpc_exec_ctx_finish(&exec_ctx); + + UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_, + client_backlog); + UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_, + server_backlog); + } + + private: + grpc_passthru_endpoint_stats stats_; + struct Stats { + int streams_stalled_due_to_stream_flow_control = 0; + int streams_stalled_due_to_transport_flow_control = 0; + }; + Stats client_stats_; + Stats server_stats_; + + grpc_endpoint_pair MakeEndpoints() { + grpc_endpoint_pair p; + grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(), + &stats_); + p.client = grpc_trickle_endpoint_create(p.client); + p.server = grpc_trickle_endpoint_create(p.server); + return p; + } + + void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) { + if (backlog == 0) { + if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) { + s->streams_stalled_due_to_stream_flow_control++; + } + if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) { + s->streams_stalled_due_to_transport_flow_control++; + } + } + } +}; + /******************************************************************************* * CONTEXT MUTATORS */ @@ -777,6 +854,79 @@ static void BM_PumpStreamServerToClient(benchmark::State& state) { state.SetBytesProcessed(state.range(0) * state.iterations()); } +static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok, + size_t size) { + while (true) { + switch (fixture->cq()->AsyncNext(t, ok, gpr_now(GPR_CLOCK_MONOTONIC))) { + case CompletionQueue::TIMEOUT: + fixture->Step(size); + break; + case CompletionQueue::SHUTDOWN: + GPR_ASSERT(false); + break; + case CompletionQueue::GOT_EVENT: + return; + } + } +} + +static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new TrickledCHTTP2(&service)); + { + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_response.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + TrickleCQNext(fixture.get(), &t, &ok, state.range(1)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + request_rw->Read(&recv_response, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + response_rw.Write(send_response, tag(1)); + while (true) { + TrickleCQNext(fixture.get(), &t, &ok, state.range(1)); + if (t == tag(0)) { + request_rw->Read(&recv_response, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + response_rw.Finish(Status::OK, tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + TrickleCQNext(fixture.get(), &t, &ok, state.range(1)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + /******************************************************************************* * CONFIGURATIONS */ @@ -866,6 +1016,16 @@ BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) ->Range(0, 128 * 1024 * 1024); +static void TrickleArgs(benchmark::internal::Benchmark* b) { + for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { + for (int j = 1024; j <= 8 * 1024 * 1024; j *= 8) { + b->Args({i, j}); + } + } +} + +BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs); + // Generate Args for StreamingPingPong benchmarks. Currently generates args for // only "small streams" (i.e streams with 0, 1 or 2 messages) static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) { diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 0d5aeb233c4..cc4a2da6289 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7779,7 +7779,8 @@ "test/core/util/passthru_endpoint.h", "test/core/util/port.h", "test/core/util/port_server_client.h", - "test/core/util/slice_splitter.h" + "test/core/util/slice_splitter.h", + "test/core/util/trickle_endpoint.h" ], "is_filegroup": true, "language": "c", @@ -7814,7 +7815,9 @@ "test/core/util/port_uv.c", "test/core/util/port_windows.c", "test/core/util/slice_splitter.c", - "test/core/util/slice_splitter.h" + "test/core/util/slice_splitter.h", + "test/core/util/trickle_endpoint.c", + "test/core/util/trickle_endpoint.h" ], "third_party": false, "type": "filegroup" diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 57cbb2c496b..504a3fc9272 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -193,6 +193,7 @@ + @@ -343,6 +344,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 620649a06c4..8e51a641f13 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -64,6 +64,9 @@ test\core\util + + test\core\util + src\core\lib\channel @@ -554,6 +557,9 @@ test\core\util + + test\core\util + src\core\lib\channel diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj index daf92305c4c..1ea64654e5a 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj @@ -161,6 +161,7 @@ + @@ -195,6 +196,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters index c9a1b4c10d6..e2ad88c96e1 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters @@ -49,6 +49,9 @@ test\core\util + + test\core\util + @@ -93,6 +96,9 @@ test\core\util + + test\core\util + From 4cb0a7a5a12802f7deaf77659adb9b07ee58393f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Feb 2017 17:14:20 -0800 Subject: [PATCH 121/167] Better test --- test/core/util/trickle_endpoint.c | 40 +++++++++++++++------- test/core/util/trickle_endpoint.h | 5 +-- test/cpp/microbenchmarks/bm_fullstack.cc | 43 +++++++++++++----------- 3 files changed, 55 insertions(+), 33 deletions(-) diff --git a/test/core/util/trickle_endpoint.c b/test/core/util/trickle_endpoint.c index 8d661e04c69..5b6c666950a 100644 --- a/test/core/util/trickle_endpoint.c +++ b/test/core/util/trickle_endpoint.c @@ -47,7 +47,9 @@ typedef struct { grpc_endpoint base; + double bytes_per_second; grpc_endpoint *wrapped; + gpr_timespec last_write; gpr_mu mu; grpc_slice_buffer write_buffer; @@ -68,8 +70,11 @@ static void te_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, for (size_t i = 0; i < slices->count; i++) { grpc_slice_ref_internal(slices->slices[i]); } - grpc_slice_buffer_addn(&te->write_buffer, slices->slices, slices->count); gpr_mu_lock(&te->mu); + if (te->write_buffer.length == 0) { + te->last_write = gpr_now(GPR_CLOCK_MONOTONIC); + } + grpc_slice_buffer_addn(&te->write_buffer, slices->slices, slices->count); grpc_closure_sched(exec_ctx, cb, GRPC_ERROR_REF(te->error)); gpr_mu_unlock(&te->mu); } @@ -147,10 +152,12 @@ static const grpc_endpoint_vtable vtable = {te_read, te_get_peer, te_get_fd}; -grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap) { +grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap, + double bytes_per_second) { trickle_endpoint *te = gpr_malloc(sizeof(*te)); te->base.vtable = &vtable; te->wrapped = wrap; + te->bytes_per_second = bytes_per_second; gpr_mu_init(&te->mu); grpc_slice_buffer_init(&te->write_buffer); grpc_slice_buffer_init(&te->writing_buffer); @@ -159,18 +166,27 @@ grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap) { return &te->base; } -size_t grpc_trickle_endpoint_trickle(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, - size_t bytes) { +static double ts2dbl(gpr_timespec s) { return s.tv_sec + 1e-9 * s.tv_nsec; } + +size_t grpc_trickle_endpoint_trickle(grpc_exec_ctx *exec_ctx, + grpc_endpoint *ep) { trickle_endpoint *te = (trickle_endpoint *)ep; gpr_mu_lock(&te->mu); - if (bytes > 0 && !te->writing) { - grpc_slice_buffer_move_first(&te->write_buffer, - GPR_MIN(bytes, te->write_buffer.length), - &te->writing_buffer); - te->writing = true; - grpc_endpoint_write( - exec_ctx, te->wrapped, &te->writing_buffer, - grpc_closure_create(te_finish_write, te, grpc_schedule_on_exec_ctx)); + if (!te->writing && te->write_buffer.length > 0) { + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + double elapsed = ts2dbl(gpr_time_sub(now, te->last_write)); + size_t bytes = (size_t)(te->bytes_per_second * elapsed); + // gpr_log(GPR_DEBUG, "%lf elapsed --> %" PRIdPTR " bytes", elapsed, bytes); + if (bytes > 0) { + grpc_slice_buffer_move_first(&te->write_buffer, + GPR_MIN(bytes, te->write_buffer.length), + &te->writing_buffer); + te->writing = true; + te->last_write = now; + grpc_endpoint_write( + exec_ctx, te->wrapped, &te->writing_buffer, + grpc_closure_create(te_finish_write, te, grpc_schedule_on_exec_ctx)); + } } size_t backlog = te->write_buffer.length; gpr_mu_unlock(&te->mu); diff --git a/test/core/util/trickle_endpoint.h b/test/core/util/trickle_endpoint.h index 5f16818ebb7..7e8d9d91e33 100644 --- a/test/core/util/trickle_endpoint.h +++ b/test/core/util/trickle_endpoint.h @@ -36,10 +36,11 @@ #include "src/core/lib/iomgr/endpoint.h" -grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap); +grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap, + double bytes_per_second); /* Allow up to \a bytes through the endpoint. Returns the new backlog. */ size_t grpc_trickle_endpoint_trickle(grpc_exec_ctx *exec_ctx, - grpc_endpoint *endpoint, size_t bytes); + grpc_endpoint *endpoint); #endif diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index f83c3599616..b4f57c3e476 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -306,8 +306,8 @@ class InProcessCHTTP2 : public EndpointPairFixture { class TrickledCHTTP2 : public EndpointPairFixture { public: - TrickledCHTTP2(Service* service) - : EndpointPairFixture(service, MakeEndpoints()) {} + TrickledCHTTP2(Service* service, size_t megabits_per_second) + : EndpointPairFixture(service, MakeEndpoints(megabits_per_second)) {} void AddToLabel(std::ostream& out, benchmark::State& state) { out << " writes/iter:" @@ -328,12 +328,12 @@ class TrickledCHTTP2 : public EndpointPairFixture { (double)state.iterations()); } - void Step(size_t write_size) { + void Step() { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - size_t client_backlog = grpc_trickle_endpoint_trickle( - &exec_ctx, endpoint_pair_.client, write_size); - size_t server_backlog = grpc_trickle_endpoint_trickle( - &exec_ctx, endpoint_pair_.server, write_size); + size_t client_backlog = + grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.client); + size_t server_backlog = + grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server); grpc_exec_ctx_finish(&exec_ctx); UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_, @@ -351,12 +351,13 @@ class TrickledCHTTP2 : public EndpointPairFixture { Stats client_stats_; Stats server_stats_; - grpc_endpoint_pair MakeEndpoints() { + grpc_endpoint_pair MakeEndpoints(size_t kilobits) { grpc_endpoint_pair p; grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(), &stats_); - p.client = grpc_trickle_endpoint_create(p.client); - p.server = grpc_trickle_endpoint_create(p.server); + double bytes_per_second = 125 * kilobits; + p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second); + p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second); return p; } @@ -854,12 +855,13 @@ static void BM_PumpStreamServerToClient(benchmark::State& state) { state.SetBytesProcessed(state.range(0) * state.iterations()); } -static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok, - size_t size) { +static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) { while (true) { - switch (fixture->cq()->AsyncNext(t, ok, gpr_now(GPR_CLOCK_MONOTONIC))) { + switch (fixture->cq()->AsyncNext( + t, ok, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(100, GPR_TIMESPAN)))) { case CompletionQueue::TIMEOUT: - fixture->Step(size); + fixture->Step(); break; case CompletionQueue::SHUTDOWN: GPR_ASSERT(false); @@ -872,7 +874,8 @@ static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok, static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { EchoTestService::AsyncService service; - std::unique_ptr fixture(new TrickledCHTTP2(&service)); + std::unique_ptr fixture( + new TrickledCHTTP2(&service, state.range(1))); { EchoResponse send_response; EchoResponse recv_response; @@ -892,7 +895,7 @@ static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { void* t; bool ok; while (need_tags) { - TrickleCQNext(fixture.get(), &t, &ok, state.range(1)); + TrickleCQNext(fixture.get(), &t, &ok); GPR_ASSERT(ok); int i = (int)(intptr_t)t; GPR_ASSERT(need_tags & (1 << i)); @@ -903,7 +906,7 @@ static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { GPR_TIMER_SCOPE("BenchmarkCycle", 0); response_rw.Write(send_response, tag(1)); while (true) { - TrickleCQNext(fixture.get(), &t, &ok, state.range(1)); + TrickleCQNext(fixture.get(), &t, &ok); if (t == tag(0)) { request_rw->Read(&recv_response, tag(0)); } else if (t == tag(1)) { @@ -916,7 +919,7 @@ static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { response_rw.Finish(Status::OK, tag(1)); need_tags = (1 << 0) | (1 << 1); while (need_tags) { - TrickleCQNext(fixture.get(), &t, &ok, state.range(1)); + TrickleCQNext(fixture.get(), &t, &ok); int i = (int)(intptr_t)t; GPR_ASSERT(need_tags & (1 << i)); need_tags &= ~(1 << i); @@ -1018,7 +1021,9 @@ BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) static void TrickleArgs(benchmark::internal::Benchmark* b) { for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { - for (int j = 1024; j <= 8 * 1024 * 1024; j *= 8) { + for (int j = 1; j <= 40000; j *= 8) { + double expected_time = (double)(20 + i) / (125 * (double)j); + if (expected_time > 0.1) continue; b->Args({i, j}); } } From ef6d744c23f772f32c6f299291a504e15a10644f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Feb 2017 19:23:09 -0800 Subject: [PATCH 122/167] Include faster networks --- test/cpp/microbenchmarks/bm_fullstack.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index b4f57c3e476..619a03d4740 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -1021,9 +1021,9 @@ BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) static void TrickleArgs(benchmark::internal::Benchmark* b) { for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { - for (int j = 1; j <= 40000; j *= 8) { - double expected_time = (double)(20 + i) / (125 * (double)j); - if (expected_time > 0.1) continue; + for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) { + double expected_time = (double)(14 + i) / (125 * (double)j); + if (expected_time > 0.01) continue; b->Args({i, j}); } } From a48e6904985754c4364d95fa51675a11ce670102 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Feb 2017 19:31:49 -0800 Subject: [PATCH 123/167] Use shell to redirect a few big files --- tools/run_tests/run_microbenchmark.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 0d56e4c3553..a9a563c1bec 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -133,11 +133,9 @@ def collect_perf(bm_name, args): 'bins/mutrace/%s' % bm_name, '--benchmark_filter=^%s$' % line, '--benchmark_min_time=20']) - with open('bm.perf', 'w') as f: - f.write(subprocess.check_output(['sudo', 'perf', 'script', '-i', 'perf.data'])) - with open('bm.folded', 'w') as f: - f.write(subprocess.check_output([ - '%s/stackcollapse-perf.pl' % flamegraph_dir, 'bm.perf'])) + subprocess.check_call(['sudo', 'perf', 'script', '-i', 'perf.data', '>', 'bm.perf'], shell=True) + subprocess.check_call([ + '%s/stackcollapse-perf.pl' % flamegraph_dir, 'bm.perf', '>', 'bm.folded'], shell=True) link(line, '%s.svg' % fnize(line)) with open('reports/%s.svg' % fnize(line), 'w') as f: f.write(subprocess.check_output([ From 9f3ba09d3cb4dd8c122a34615d6e3779047eea6f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Feb 2017 08:16:09 -0800 Subject: [PATCH 124/167] Use pre-existing script for flamegraph generation --- tools/run_tests/run_microbenchmark.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index a9a563c1bec..ea87b6cd986 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -128,18 +128,19 @@ def collect_perf(bm_name, args): 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()]) for line in subprocess.check_output(['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines(): - subprocess.check_call(['sudo', 'perf', 'record', '-o', 'perf.data', + subprocess.check_call(['perf', 'record', '-o', '%s-perf.data' % fnize(line), '-g', '-c', '1000', 'bins/mutrace/%s' % bm_name, '--benchmark_filter=^%s$' % line, '--benchmark_min_time=20']) - subprocess.check_call(['sudo', 'perf', 'script', '-i', 'perf.data', '>', 'bm.perf'], shell=True) - subprocess.check_call([ - '%s/stackcollapse-perf.pl' % flamegraph_dir, 'bm.perf', '>', 'bm.folded'], shell=True) - link(line, '%s.svg' % fnize(line)) - with open('reports/%s.svg' % fnize(line), 'w') as f: - f.write(subprocess.check_output([ - '%s/flamegraph.pl' % flamegraph_dir, 'bm.folded'])) + env = os.environ.copy() + env.update({ + 'PERF_BASE_NAME': fnize(line), + 'OUTPUT_DIR': 'reports', + 'OUTPUT_FILENAME': fnize(line), + }) + subprocess.check_call(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'], + env=env) def collect_summary(bm_name, args): heading('Summary: %s' % bm_name) From 4a84bdda7f42e303c570e2ee73cc14c57df6c81b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Feb 2017 09:48:41 -0800 Subject: [PATCH 125/167] s/transport_private/handler_private -- use is no longer restricted to transports --- src/core/ext/client_channel/client_channel.c | 6 +++--- .../ext/transport/chttp2/transport/chttp2_transport.c | 10 +++++----- src/core/lib/transport/transport.h | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 43f7e619cfc..49ea7f06be9 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -873,7 +873,7 @@ static void cc_start_transport_stream_op_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error_ignored) { grpc_transport_stream_op *op = arg; - grpc_call_element *elem = op->transport_private.args[0]; + grpc_call_element *elem = op->handler_private.args[0]; call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; grpc_subchannel_call *call; @@ -992,10 +992,10 @@ static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, } /* we failed; lock and figure out what to do */ GRPC_CALL_STACK_REF(calld->owning_call, "start_transport_stream_op"); - op->transport_private.args[0] = elem; + op->handler_private.args[0] = elem; grpc_closure_sched( exec_ctx, - grpc_closure_init(&op->transport_private.closure, + grpc_closure_init(&op->handler_private.closure, cc_start_transport_stream_op_locked, op, grpc_combiner_scheduler(chand->combiner, false)), GRPC_ERROR_NONE); diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 3ee5e976f84..c00e64f0338 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1033,8 +1033,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, GPR_TIMER_BEGIN("perform_stream_op_locked", 0); grpc_transport_stream_op *op = stream_op; - grpc_chttp2_transport *t = op->transport_private.args[0]; - grpc_chttp2_stream *s = op->transport_private.args[1]; + grpc_chttp2_transport *t = op->handler_private.args[0]; + grpc_chttp2_stream *s = op->handler_private.args[1]; if (grpc_http_trace) { char *str = grpc_transport_stream_op_string(op); @@ -1255,13 +1255,13 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, gpr_free(str); } - op->transport_private.args[0] = gt; - op->transport_private.args[1] = gs; + op->handler_private.args[0] = gt; + op->handler_private.args[1] = gs; GRPC_CHTTP2_STREAM_REF(s, "perform_stream_op"); grpc_closure_sched( exec_ctx, grpc_closure_init( - &op->transport_private.closure, perform_stream_op_locked, op, + &op->handler_private.closure, perform_stream_op_locked, op, grpc_combiner_scheduler(t->combiner, op->covered_by_poller)), GRPC_ERROR_NONE); GPR_TIMER_END("perform_stream_op", 0); diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 9a0abe1ca46..e56bf2780ab 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -167,9 +167,9 @@ typedef struct grpc_transport_stream_op { /*************************************************************************** * remaining fields are initialized and used at the discretion of the - * transport implementation */ + * current handler of the op */ - grpc_transport_private_op_data transport_private; + grpc_transport_private_op_data handler_private; } grpc_transport_stream_op; /** Transport op: a set of operations to perform on a transport as a whole */ From a11bfc85e76dc1371b4c209259eb3449fd4b0dde Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Feb 2017 09:56:33 -0800 Subject: [PATCH 126/167] Simplify loop --- src/core/ext/client_channel/client_channel.c | 49 ++++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 49ea7f06be9..28ce224e618 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -869,32 +869,34 @@ static bool pick_subchannel_locked( return false; } -static void cc_start_transport_stream_op_locked(grpc_exec_ctx *exec_ctx, - void *arg, - grpc_error *error_ignored) { - grpc_transport_stream_op *op = arg; - grpc_call_element *elem = op->handler_private.args[0]; - call_data *calld = elem->call_data; +static void cc_start_transport_stream_op_locked_inner( + grpc_exec_ctx *exec_ctx, grpc_transport_stream_op *op, + grpc_call_element *elem) { channel_data *chand = elem->channel_data; + call_data *calld = elem->call_data; grpc_subchannel_call *call; -retry: /* need to recheck that another thread hasn't set the call */ call = GET_CALL(calld); if (call == CANCELLED_CALL) { grpc_transport_stream_op_finish_with_failure( exec_ctx, op, GRPC_ERROR_REF(calld->cancel_error)); - goto done; + /* early out */ + return; } if (call != NULL) { grpc_subchannel_call_process_op(exec_ctx, call, op); - goto done; + /* early out */ + return; } /* if this is a cancellation, then we can raise our cancelled flag */ if (op->cancel_error != GRPC_ERROR_NONE) { if (!gpr_atm_rel_cas(&calld->subchannel_call, 0, (gpr_atm)(uintptr_t)CANCELLED_CALL)) { - goto retry; + /* recurse to retry */ + cc_start_transport_stream_op_locked_inner(exec_ctx, op, elem); + /* early out */ + return; } else { // Stash a copy of cancel_error in our call data, so that we can use // it for subsequent operations. This ensures that if the call is @@ -914,7 +916,8 @@ retry: } grpc_transport_stream_op_finish_with_failure( exec_ctx, op, GRPC_ERROR_REF(op->cancel_error)); - goto done; + /* early out */ + return; } } /* if we don't have a subchannel, try to get one */ @@ -954,14 +957,29 @@ retry: gpr_atm_rel_store(&calld->subchannel_call, (gpr_atm)(uintptr_t)subchannel_call); retry_waiting_locked(exec_ctx, calld); - goto retry; + /* recurse to retry */ + cc_start_transport_stream_op_locked_inner(exec_ctx, op, elem); + /* early out */ + return; } /* nothing to be done but wait */ add_waiting_locked(calld, op); -done: +} + +static void cc_start_transport_stream_op_locked(grpc_exec_ctx *exec_ctx, + void *arg, + grpc_error *error_ignored) { + GPR_TIMER_BEGIN("cc_start_transport_stream_op_locked", 0); + + grpc_transport_stream_op *op = arg; + grpc_call_element *elem = op->handler_private.args[0]; + call_data *calld = elem->call_data; + + cc_start_transport_stream_op_locked_inner(exec_ctx, op, elem); + GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "start_transport_stream_op"); - GPR_TIMER_END("cc_start_transport_stream_op", 0); + GPR_TIMER_END("cc_start_transport_stream_op_locked", 0); } // The logic here is fairly complicated, due to (a) the fact that we @@ -983,11 +1001,13 @@ static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport_stream_op_finish_with_failure( exec_ctx, op, GRPC_ERROR_REF(calld->cancel_error)); GPR_TIMER_END("cc_start_transport_stream_op", 0); + /* early out */ return; } if (call != NULL) { grpc_subchannel_call_process_op(exec_ctx, call, op); GPR_TIMER_END("cc_start_transport_stream_op", 0); + /* early out */ return; } /* we failed; lock and figure out what to do */ @@ -999,6 +1019,7 @@ static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, cc_start_transport_stream_op_locked, op, grpc_combiner_scheduler(chand->combiner, false)), GRPC_ERROR_NONE); + GPR_TIMER_END("cc_start_transport_stream_op", 0); } // Gets data from the service config. Invoked when the resolver returns From be9691af99b438c72b64dd070cf0d45fac29792e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Feb 2017 10:00:42 -0800 Subject: [PATCH 127/167] Add commentary --- src/core/ext/client_channel/client_channel.c | 23 +++++++++++--------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 28ce224e618..e500187975c 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -898,11 +898,11 @@ static void cc_start_transport_stream_op_locked_inner( /* early out */ return; } else { - // Stash a copy of cancel_error in our call data, so that we can use - // it for subsequent operations. This ensures that if the call is - // cancelled before any ops are passed down (e.g., if the deadline - // is in the past when the call starts), we can return the right - // error to the caller when the first op does get passed down. + /* Stash a copy of cancel_error in our call data, so that we can use + it for subsequent operations. This ensures that if the call is + cancelled before any ops are passed down (e.g., if the deadline + is in the past when the call starts), we can return the right + error to the caller when the first op does get passed down. */ calld->cancel_error = GRPC_ERROR_REF(op->cancel_error); switch (calld->creation_phase) { case GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING: @@ -982,11 +982,14 @@ static void cc_start_transport_stream_op_locked(grpc_exec_ctx *exec_ctx, GPR_TIMER_END("cc_start_transport_stream_op_locked", 0); } -// The logic here is fairly complicated, due to (a) the fact that we -// need to handle the case where we receive the send op before the -// initial metadata op, and (b) the need for efficiency, especially in -// the streaming case. -// TODO(ctiller): Explain this more thoroughly. +/* The logic here is fairly complicated, due to (a) the fact that we + need to handle the case where we receive the send op before the + initial metadata op, and (b) the need for efficiency, especially in + the streaming case. + + We use double-checked locking to initially see if initialization has been + performed. If it has not, we acquire the combiner and perform initialization. + If it has, we proceed on the fast path. */ static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op) { From a8610c094d8bdf60188b9fd4c9934b2a9d3c0b56 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Feb 2017 10:05:11 -0800 Subject: [PATCH 128/167] Review nits --- src/core/ext/client_channel/client_channel.c | 34 +++++++++----------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index e500187975c..bedb13d9741 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -421,8 +421,8 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(state_error); } -static void cc_start_transport_op_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error_ignored) { +static void start_transport_op_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error_ignored) { grpc_transport_op *op = arg; grpc_channel_element *elem = op->transport_private.args[0]; channel_data *chand = elem->channel_data; @@ -488,10 +488,9 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, op->transport_private.args[0] = elem; GRPC_CHANNEL_STACK_REF(chand->owning_stack, "start_transport_op"); grpc_closure_sched( - exec_ctx, - grpc_closure_init(&op->transport_private.closure, - cc_start_transport_op_locked, op, - grpc_combiner_scheduler(chand->combiner, false)), + exec_ctx, grpc_closure_init( + &op->transport_private.closure, start_transport_op_locked, + op, grpc_combiner_scheduler(chand->combiner, false)), GRPC_ERROR_NONE); } @@ -869,9 +868,9 @@ static bool pick_subchannel_locked( return false; } -static void cc_start_transport_stream_op_locked_inner( - grpc_exec_ctx *exec_ctx, grpc_transport_stream_op *op, - grpc_call_element *elem) { +static void start_transport_stream_op_locked_inner(grpc_exec_ctx *exec_ctx, + grpc_transport_stream_op *op, + grpc_call_element *elem) { channel_data *chand = elem->channel_data; call_data *calld = elem->call_data; grpc_subchannel_call *call; @@ -894,7 +893,7 @@ static void cc_start_transport_stream_op_locked_inner( if (!gpr_atm_rel_cas(&calld->subchannel_call, 0, (gpr_atm)(uintptr_t)CANCELLED_CALL)) { /* recurse to retry */ - cc_start_transport_stream_op_locked_inner(exec_ctx, op, elem); + start_transport_stream_op_locked_inner(exec_ctx, op, elem); /* early out */ return; } else { @@ -958,7 +957,7 @@ static void cc_start_transport_stream_op_locked_inner( (gpr_atm)(uintptr_t)subchannel_call); retry_waiting_locked(exec_ctx, calld); /* recurse to retry */ - cc_start_transport_stream_op_locked_inner(exec_ctx, op, elem); + start_transport_stream_op_locked_inner(exec_ctx, op, elem); /* early out */ return; } @@ -975,7 +974,7 @@ static void cc_start_transport_stream_op_locked(grpc_exec_ctx *exec_ctx, grpc_call_element *elem = op->handler_private.args[0]; call_data *calld = elem->call_data; - cc_start_transport_stream_op_locked_inner(exec_ctx, op, elem); + start_transport_stream_op_locked_inner(exec_ctx, op, elem); GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "start_transport_stream_op"); @@ -1221,8 +1220,8 @@ static void try_to_connect_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_connectivity_state grpc_client_channel_check_connectivity_state( grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, int try_to_connect) { channel_data *chand = elem->channel_data; - grpc_connectivity_state out; - out = grpc_connectivity_state_check(&chand->state_tracker); + grpc_connectivity_state out = + grpc_connectivity_state_check(&chand->state_tracker); if (out == GRPC_CHANNEL_IDLE && try_to_connect) { GRPC_CHANNEL_STACK_REF(chand->owning_stack, "try_to_connect"); grpc_closure_sched( @@ -1254,9 +1253,8 @@ static void on_external_watch_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_closure_run(exec_ctx, follow_up, GRPC_ERROR_REF(error)); } -static void cc_watch_connectivity_state_locked(grpc_exec_ctx *exec_ctx, - void *arg, - grpc_error *error_ignored) { +static void watch_connectivity_state_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error_ignored) { external_connectivity_watcher *w = arg; grpc_closure_init(&w->my_closure, on_external_watch_complete, w, grpc_schedule_on_exec_ctx); @@ -1278,7 +1276,7 @@ void grpc_client_channel_watch_connectivity_state( "external_connectivity_watcher"); grpc_closure_sched( exec_ctx, - grpc_closure_init(&w->my_closure, cc_watch_connectivity_state_locked, w, + grpc_closure_init(&w->my_closure, watch_connectivity_state_locked, w, grpc_combiner_scheduler(chand->combiner, true)), GRPC_ERROR_NONE); } From a74ccafc93022d40210d70dbec392ceddfa3e612 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 14 Feb 2017 10:24:38 -0800 Subject: [PATCH 129/167] Add new dummy Cronet API --- .../transport/cronet/transport/cronet_api_dummy.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c index da6c0b4fbcc..0dc6a5152fd 100644 --- a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c +++ b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c @@ -80,4 +80,16 @@ void bidirectional_stream_cancel(bidirectional_stream* stream) { GPR_ASSERT(0); } +void bidirectional_stream_disable_auto_flush(bidirectional_stream* stream, + bool disable_auto_flush) { + GPR_ASSERT(0); +} + +void bidirectional_stream_delay_request_headers_until_flush( + bidirectional_stream* stream, bool delay_headers_until_flush) { + GPR_ASSERT(0); +} + +void bidirectional_stream_flush(bidirectional_stream* stream) { GPR_ASSERT(0); } + #endif /* GRPC_COMPILE_WITH_CRONET */ From 225435972c10da4dfb561ed7b11743022187708a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Feb 2017 10:29:36 -0800 Subject: [PATCH 130/167] Fix bug in poll polling engine causing pollsets to be deleted before they are removed from pollset_sets --- src/core/lib/iomgr/ev_poll_posix.c | 38 ++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 21eb62753ef..6bae99a3443 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -191,6 +191,7 @@ struct grpc_pollset { int kicked_without_pollers; grpc_closure *shutdown_done; grpc_closure_list idle_jobs; + int pollset_set_count; /* all polled fds */ size_t fd_count; size_t fd_capacity; @@ -228,7 +229,7 @@ static grpc_error *pollset_kick_ext(grpc_pollset *p, /* Return 1 if the pollset has active threads in pollset_work (pollset must * be locked) */ -static int pollset_has_workers(grpc_pollset *pollset); +static bool pollset_has_workers(grpc_pollset *pollset); /******************************************************************************* * pollset_set definitions @@ -658,10 +659,18 @@ static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { worker->next->prev = worker->prev; } -static int pollset_has_workers(grpc_pollset *p) { +static bool pollset_has_workers(grpc_pollset *p) { return p->root_worker.next != &p->root_worker; } +static bool pollset_in_pollset_sets(grpc_pollset *p) { + return p->pollset_set_count; +} + +static bool pollset_has_observers(grpc_pollset *p) { + return pollset_has_workers(p) || pollset_in_pollset_sets(p); +} + static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { if (pollset_has_workers(p)) { grpc_pollset_worker *w = p->root_worker.next; @@ -800,6 +809,7 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->fd_count = 0; pollset->fd_capacity = 0; pollset->fds = NULL; + pollset->pollset_set_count = 0; } static void pollset_destroy(grpc_pollset *pollset) { @@ -1061,7 +1071,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (pollset->shutting_down) { if (pollset_has_workers(pollset)) { pollset_kick(pollset, NULL); - } else if (!pollset->called_shutdown) { + } else if (!pollset->called_shutdown && !pollset_has_observers(pollset)) { pollset->called_shutdown = 1; gpr_mu_unlock(&pollset->mu); finish_shutdown(exec_ctx, pollset); @@ -1093,7 +1103,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (!pollset_has_workers(pollset)) { grpc_closure_list_sched(exec_ctx, &pollset->idle_jobs); } - if (!pollset->called_shutdown && !pollset_has_workers(pollset)) { + if (!pollset->called_shutdown && !pollset_has_observers(pollset)) { pollset->called_shutdown = 1; finish_shutdown(exec_ctx, pollset); } @@ -1143,13 +1153,16 @@ static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, grpc_pollset *pollset) { size_t i, j; + gpr_mu_lock(&pollset->mu); + pollset->pollset_set_count++; + gpr_mu_unlock(&pollset->mu); gpr_mu_lock(&pollset_set->mu); if (pollset_set->pollset_count == pollset_set->pollset_capacity) { pollset_set->pollset_capacity = GPR_MAX(8, 2 * pollset_set->pollset_capacity); - pollset_set->pollsets = - gpr_realloc(pollset_set->pollsets, pollset_set->pollset_capacity * - sizeof(*pollset_set->pollsets)); + pollset_set->pollsets = gpr_realloc( + pollset_set->pollsets, + pollset_set->pollset_capacity * sizeof(*pollset_set->pollsets)); } pollset_set->pollsets[pollset_set->pollset_count++] = pollset; for (i = 0, j = 0; i < pollset_set->fd_count; i++) { @@ -1178,6 +1191,17 @@ static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, } } gpr_mu_unlock(&pollset_set->mu); + gpr_mu_lock(&pollset->mu); + pollset->pollset_set_count--; + /* check shutdown */ + if (pollset->shutting_down && !pollset->called_shutdown && + !pollset_has_observers(pollset)) { + pollset->called_shutdown = 1; + gpr_mu_unlock(&pollset->mu); + finish_shutdown(exec_ctx, pollset); + } else { + gpr_mu_unlock(&pollset->mu); + } } static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, From 716f953532a92f3a159d26a78fdb397d0838b20e Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 14 Feb 2017 10:54:20 -0800 Subject: [PATCH 131/167] Update protobuf version used for nanopb compiler --- tools/codegen/core/gen_nano_proto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/codegen/core/gen_nano_proto.sh b/tools/codegen/core/gen_nano_proto.sh index 99e49814b83..8600573e1cb 100755 --- a/tools/codegen/core/gen_nano_proto.sh +++ b/tools/codegen/core/gen_nano_proto.sh @@ -83,7 +83,7 @@ popd # this should be the same version as the submodule we compile against # ideally we'd update this as a template to ensure that -pip install protobuf==3.0.0 +pip install protobuf==3.2.0 pushd "$(dirname $INPUT_PROTO)" > /dev/null From 4d352bec3dfaee160268de94e27553f94cfa7ae7 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 14 Feb 2017 13:04:08 -0800 Subject: [PATCH 132/167] remove some temporary comments --- src/objective-c/tests/CronetUnitTests/CronetUnitTests.m | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m index b6237c950bf..1833a5be495 100644 --- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m +++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m @@ -71,9 +71,6 @@ static void drain_cq(grpc_completion_queue *cq) { + (void)setUp { [super setUp]; - /*** FILE *roots_file; - size_t roots_size = strlen(test_root_cert);*/ - char *argv[] = {"CoreCronetEnd2EndTests"}; grpc_test_init(1, argv); From b67243fd3dffadaabb661143bacf34b80560e066 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 14 Feb 2017 14:12:33 -0800 Subject: [PATCH 133/167] Polish test --- src/objective-c/tests/CronetUnitTests/CronetUnitTests.m | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m index 1833a5be495..5e3c59f8b34 100644 --- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m +++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m @@ -358,6 +358,8 @@ unsigned int parse_h2_length(const char *field) { error = grpc_call_start_batch(c, ops, (size_t)(op - ops), (void *)1, NULL); GPR_ASSERT(GRPC_CALL_OK == error); + __weak XCTestExpectation *expectation = [self expectationWithDescription:@"Coalescing"]; + dispatch_async( dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ int sl = socket(AF_INET, SOCK_STREAM, 0); @@ -420,6 +422,7 @@ unsigned int parse_h2_length(const char *field) { SSL_CTX_free(ctx); close(s); close(sl); + [expectation fulfill]; }); CQ_EXPECT_COMPLETION(cqv, (void *)1, 1); @@ -442,6 +445,8 @@ unsigned int parse_h2_length(const char *field) { grpc_completion_queue_shutdown(cq); drain_cq(cq); grpc_completion_queue_destroy(cq); + + [self waitForExpectationsWithTimeout:4 handler:nil]; } - (void)testPacketCoalescing { From 8da82bb9cf47442774c5618b856398eddf02f713 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 14 Feb 2017 14:20:13 -0800 Subject: [PATCH 134/167] Add CronetUnitTests into Jenkins tests --- .../xcshareddata/xcschemes/AllTests.xcscheme | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme b/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme index 5524a27ffde..49dc3faa3d0 100644 --- a/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme +++ b/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme @@ -59,6 +59,16 @@ ReferencedContainer = "container:Tests.xcodeproj"> + + + + + + + + From d5d9d9e6727df310596c005d1dfb38775182d05f Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Tue, 14 Feb 2017 12:58:24 -0800 Subject: [PATCH 135/167] Store cython-level call in grpc.Call This allows inspecting c-level properties of the call after completion. --- src/python/grpcio/grpc/_channel.py | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index 691e96a3228..26d93faf752 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -444,10 +444,10 @@ def _start_unary_request(request, timeout, request_serializer): return deadline, deadline_timespec, serialized_request, None -def _end_unary_response_blocking(state, with_call, deadline): +def _end_unary_response_blocking(state, call, with_call, deadline): if state.code is grpc.StatusCode.OK: if with_call: - rendezvous = _Rendezvous(state, None, None, deadline) + rendezvous = _Rendezvous(state, call, None, deadline) return state.response, rendezvous else: return state.response @@ -499,17 +499,17 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): _check_call_error(call_error, metadata) _handle_event(completion_queue.poll(), state, self._response_deserializer) - return state, deadline + return state, call, deadline def __call__(self, request, timeout=None, metadata=None, credentials=None): - state, deadline, = self._blocking(request, timeout, metadata, - credentials) - return _end_unary_response_blocking(state, False, deadline) + state, call, deadline = self._blocking(request, timeout, metadata, + credentials) + return _end_unary_response_blocking(state, call, False, deadline) def with_call(self, request, timeout=None, metadata=None, credentials=None): - state, deadline, = self._blocking(request, timeout, metadata, - credentials) - return _end_unary_response_blocking(state, True, deadline) + state, call, deadline = self._blocking(request, timeout, metadata, + credentials) + return _end_unary_response_blocking(state, call, True, deadline) def future(self, request, timeout=None, metadata=None, credentials=None): state, operations, deadline, deadline_timespec, rendezvous = self._prepare( @@ -619,25 +619,25 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): state.condition.notify_all() if not state.due: break - return state, deadline + return state, call, deadline def __call__(self, request_iterator, timeout=None, metadata=None, credentials=None): - state, deadline, = self._blocking(request_iterator, timeout, metadata, - credentials) - return _end_unary_response_blocking(state, False, deadline) + state, call, deadline = self._blocking(request_iterator, timeout, + metadata, credentials) + return _end_unary_response_blocking(state, call, False, deadline) def with_call(self, request_iterator, timeout=None, metadata=None, credentials=None): - state, deadline, = self._blocking(request_iterator, timeout, metadata, - credentials) - return _end_unary_response_blocking(state, True, deadline) + state, call, deadline = self._blocking(request_iterator, timeout, + metadata, credentials) + return _end_unary_response_blocking(state, call, True, deadline) def future(self, request_iterator, From 32df85cfab9efd8862cdc337c844de8540846399 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Tue, 14 Feb 2017 16:03:55 -0800 Subject: [PATCH 136/167] Export LICENSE.txt in //third_party/nanopb --- third_party/nanopb/BUILD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/third_party/nanopb/BUILD b/third_party/nanopb/BUILD index 570988435f1..f9fc57f50aa 100644 --- a/third_party/nanopb/BUILD +++ b/third_party/nanopb/BUILD @@ -1,4 +1,7 @@ licenses(["notice"]) + +exports_files(["LICENSE.txt"]) + package(default_visibility = ["//visibility:public"]) cc_library( From 5481a58da79977423b6ce8fe9bafbd01b19999a1 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 14 Feb 2017 16:11:42 -0800 Subject: [PATCH 137/167] Added to BUILD --- BUILD | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/BUILD b/BUILD index ba92279b8e7..9ea885a390a 100644 --- a/BUILD +++ b/BUILD @@ -526,13 +526,13 @@ grpc_cc_library( "src/core/lib/surface/server.c", "src/core/lib/surface/validate_metadata.c", "src/core/lib/surface/version.c", + "src/core/lib/transport/bdp_estimator.c", "src/core/lib/transport/byte_stream.c", "src/core/lib/transport/connectivity_state.c", "src/core/lib/transport/error_utils.c", "src/core/lib/transport/metadata.c", "src/core/lib/transport/metadata_batch.c", "src/core/lib/transport/pid_controller.c", - "src/core/lib/transport/bdp_estimator.c", "src/core/lib/transport/service_config.c", "src/core/lib/transport/static_metadata.c", "src/core/lib/transport/status_conversion.c", @@ -633,6 +633,7 @@ grpc_cc_library( "src/core/lib/surface/lame_client.h", "src/core/lib/surface/server.h", "src/core/lib/surface/validate_metadata.h", + "src/core/lib/transport/bdp_estimator.h", "src/core/lib/transport/byte_stream.h", "src/core/lib/transport/connectivity_state.h", "src/core/lib/transport/error_utils.h", @@ -640,7 +641,6 @@ grpc_cc_library( "src/core/lib/transport/metadata.h", "src/core/lib/transport/metadata_batch.h", "src/core/lib/transport/pid_controller.h", - "src/core/lib/transport/bdp_estimator.h", "src/core/lib/transport/service_config.h", "src/core/lib/transport/static_metadata.h", "src/core/lib/transport/status_conversion.h", @@ -656,6 +656,7 @@ grpc_cc_library( "include/grpc/byte_buffer.h", "include/grpc/byte_buffer_reader.h", "include/grpc/compression.h", + "include/grpc/load_reporting.h", "include/grpc/grpc.h", "include/grpc/grpc_posix.h", "include/grpc/grpc_security_constants.h", @@ -869,8 +870,8 @@ grpc_cc_library( "src/core/lib/security/credentials/plugin/plugin_credentials.c", "src/core/lib/security/credentials/ssl/ssl_credentials.c", "src/core/lib/security/transport/client_auth_filter.c", - "src/core/lib/security/transport/secure_endpoint.c", "src/core/lib/security/transport/lb_targets_info.c", + "src/core/lib/security/transport/secure_endpoint.c", "src/core/lib/security/transport/security_connector.c", "src/core/lib/security/transport/security_handshaker.c", "src/core/lib/security/transport/server_auth_filter.c", @@ -893,8 +894,8 @@ grpc_cc_library( "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/transport/auth_filters.h", - "src/core/lib/security/transport/secure_endpoint.h", "src/core/lib/security/transport/lb_targets_info.h", + "src/core/lib/security/transport/secure_endpoint.h", "src/core/lib/security/transport/security_connector.h", "src/core/lib/security/transport/security_handshaker.h", "src/core/lib/security/transport/tsi_error.h", @@ -1267,13 +1268,13 @@ grpc_cc_library( grpc_cc_library( name = "grpc++_config_proto", + external_deps = [ + "protobuf", + ], language = "c++", public_hdrs = [ "include/grpc++/impl/codegen/config_protobuf.h", ], - external_deps = [ - "protobuf", - ], ) grpc_cc_library( From bd5a4d263d841c8bed87730173d31c5febcc6906 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Feb 2017 16:45:47 -0800 Subject: [PATCH 138/167] Cleanup files during processing --- tools/run_tests/run_microbenchmark.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index ea87b6cd986..6bba29d74ba 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -132,7 +132,7 @@ def collect_perf(bm_name, args): '-g', '-c', '1000', 'bins/mutrace/%s' % bm_name, '--benchmark_filter=^%s$' % line, - '--benchmark_min_time=20']) + '--benchmark_min_time=10']) env = os.environ.copy() env.update({ 'PERF_BASE_NAME': fnize(line), @@ -141,6 +141,8 @@ def collect_perf(bm_name, args): }) subprocess.check_call(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'], env=env) + subprocess.check_call(['rm', '%s-perf.data' % fnize(line)]) + subprocess.check_call(['rm', '%s-out.perf' % fnize(line)]) def collect_summary(bm_name, args): heading('Summary: %s' % bm_name) From 7676e9ac9341759c1d71147e94de020d990783c2 Mon Sep 17 00:00:00 2001 From: Yong Ni Date: Tue, 14 Feb 2017 17:35:25 -0800 Subject: [PATCH 139/167] Tue Feb 14 17:35:25 PST 2017 --- tools/internal_ci/linux/grpc_interop.cfg | 40 ++++++++++++++++++++++ tools/internal_ci/linux/grpc_interop.sh | 42 ++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 tools/internal_ci/linux/grpc_interop.cfg create mode 100755 tools/internal_ci/linux/grpc_interop.sh diff --git a/tools/internal_ci/linux/grpc_interop.cfg b/tools/internal_ci/linux/grpc_interop.cfg new file mode 100644 index 00000000000..ecafcf62ae9 --- /dev/null +++ b/tools/internal_ci/linux/grpc_interop.cfg @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_interop.sh" +timeout_mins: 60 +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/tools/internal_ci/linux/grpc_interop.sh b/tools/internal_ci/linux/grpc_interop.sh new file mode 100755 index 00000000000..68bb4199e3d --- /dev/null +++ b/tools/internal_ci/linux/grpc_interop.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +export LANG=en_US.UTF-8 + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +git submodule update --init + +tools/run_tests/run_interop_tests.py -l all -s all --cloud_to_prod --cloud_to_prod_auth --use_docker --http2_interop -t -j 12 $@ || FAILED="true" +tools/run_tests/run_interop_tests.py -l java --use_docker --http2_badserver_interop $@ || FAILED="true" +tools/run_tests/run_interop_tests.py -l python --use_docker --http2_badserver_interop $@ || FAILED="true" From 9e5ac1bf115a182c0418f21d60d15245af6307bf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Feb 2017 22:25:50 -0800 Subject: [PATCH 140/167] Fix broken shutdown, and cascading exec_ctx usage bugs --- src/core/ext/client_channel/client_channel.c | 2 +- src/core/ext/client_channel/lb_policy.c | 2 +- src/core/ext/client_channel/subchannel.c | 2 +- .../ext/resolver/dns/native/dns_resolver.c | 2 +- src/core/lib/http/httpcli.c | 5 +-- src/core/lib/http/httpcli.h | 3 +- src/core/lib/iomgr/ev_epoll_linux.c | 7 ++-- src/core/lib/iomgr/ev_poll_posix.c | 33 ++++++++++++++----- src/core/lib/iomgr/ev_posix.c | 5 +-- src/core/lib/iomgr/ev_posix.h | 3 +- src/core/lib/iomgr/pollset_set.h | 3 +- src/core/lib/iomgr/pollset_set_uv.c | 3 +- src/core/lib/iomgr/pollset_set_windows.c | 3 +- .../google_default_credentials.c | 2 +- .../security/credentials/jwt/jwt_verifier.c | 4 +-- .../security/credentials/jwt/jwt_verifier.h | 3 +- .../credentials/oauth2/oauth2_credentials.c | 2 +- test/core/end2end/fixtures/http_proxy.c | 2 +- test/core/http/httpcli_test.c | 2 +- test/core/http/httpscli_test.c | 2 +- test/core/iomgr/pollset_set_test.c | 11 ++++--- test/core/iomgr/resolve_address_posix_test.c | 2 +- test/core/iomgr/resolve_address_test.c | 2 +- test/core/iomgr/tcp_client_posix_test.c | 2 +- test/core/security/jwt_verifier_test.c | 16 ++++----- test/core/security/verify_jwt.c | 5 +-- test/core/util/port_server_client.c | 4 +-- 27 files changed, 78 insertions(+), 54 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index bedb13d9741..62fb061bcf9 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -582,7 +582,7 @@ static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx, grpc_slice_hash_table_unref(exec_ctx, chand->method_params_table); } grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker); - grpc_pollset_set_destroy(chand->interested_parties); + grpc_pollset_set_destroy(exec_ctx, chand->interested_parties); GRPC_COMBINER_UNREF(exec_ctx, chand->combiner, "client_channel"); gpr_mu_destroy(&chand->info_mu); } diff --git a/src/core/ext/client_channel/lb_policy.c b/src/core/ext/client_channel/lb_policy.c index 45ee72e2f0c..90401b586f2 100644 --- a/src/core/ext/client_channel/lb_policy.c +++ b/src/core/ext/client_channel/lb_policy.c @@ -94,7 +94,7 @@ void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx, gpr_atm old_val = ref_mutate(policy, -(gpr_atm)1, 1 REF_MUTATE_PASS_ARGS("WEAK_UNREF")); if (old_val == 1) { - grpc_pollset_set_destroy(policy->interested_parties); + grpc_pollset_set_destroy(exec_ctx, policy->interested_parties); policy->vtable->destroy(exec_ctx, policy); } } diff --git a/src/core/ext/client_channel/subchannel.c b/src/core/ext/client_channel/subchannel.c index c37134fd5e7..09c68a91dd8 100644 --- a/src/core/ext/client_channel/subchannel.c +++ b/src/core/ext/client_channel/subchannel.c @@ -217,7 +217,7 @@ static void subchannel_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_slice_unref_internal(exec_ctx, c->initial_connect_string); grpc_connectivity_state_destroy(exec_ctx, &c->state_tracker); grpc_connector_unref(exec_ctx, c->connector); - grpc_pollset_set_destroy(c->pollset_set); + grpc_pollset_set_destroy(exec_ctx, c->pollset_set); grpc_subchannel_key_destroy(exec_ctx, c->key); gpr_free(c); } diff --git a/src/core/ext/resolver/dns/native/dns_resolver.c b/src/core/ext/resolver/dns/native/dns_resolver.c index 2c9623211b3..c08b53ea040 100644 --- a/src/core/ext/resolver/dns/native/dns_resolver.c +++ b/src/core/ext/resolver/dns/native/dns_resolver.c @@ -244,7 +244,7 @@ static void dns_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *gr) { if (r->resolved_result != NULL) { grpc_channel_args_destroy(exec_ctx, r->resolved_result); } - grpc_pollset_set_destroy(r->interested_parties); + grpc_pollset_set_destroy(exec_ctx, r->interested_parties); gpr_free(r->name_to_resolve); gpr_free(r->default_port); grpc_channel_args_destroy(exec_ctx, r->channel_args); diff --git a/src/core/lib/http/httpcli.c b/src/core/lib/http/httpcli.c index fb2108987b1..6d7aa43b815 100644 --- a/src/core/lib/http/httpcli.c +++ b/src/core/lib/http/httpcli.c @@ -93,8 +93,9 @@ void grpc_httpcli_context_init(grpc_httpcli_context *context) { context->pollset_set = grpc_pollset_set_create(); } -void grpc_httpcli_context_destroy(grpc_httpcli_context *context) { - grpc_pollset_set_destroy(context->pollset_set); +void grpc_httpcli_context_destroy(grpc_exec_ctx *exec_ctx, + grpc_httpcli_context *context) { + grpc_pollset_set_destroy(exec_ctx, context->pollset_set); } static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req, diff --git a/src/core/lib/http/httpcli.h b/src/core/lib/http/httpcli.h index 11e03b44dfd..8ae03ee78f4 100644 --- a/src/core/lib/http/httpcli.h +++ b/src/core/lib/http/httpcli.h @@ -83,7 +83,8 @@ typedef struct grpc_httpcli_request { typedef struct grpc_http_response grpc_httpcli_response; void grpc_httpcli_context_init(grpc_httpcli_context *context); -void grpc_httpcli_context_destroy(grpc_httpcli_context *context); +void grpc_httpcli_context_destroy(grpc_exec_ctx *exec_ctx, + grpc_httpcli_context *context); /* Asynchronously perform a HTTP GET. 'context' specifies the http context under which to do the get diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index fc56843128e..fac3705142b 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1842,13 +1842,12 @@ static grpc_pollset_set *pollset_set_create(void) { return pss; } -static void pollset_set_destroy(grpc_pollset_set *pss) { +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { gpr_mu_destroy(&pss->po.mu); if (pss->po.pi != NULL) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - PI_UNREF(&exec_ctx, pss->po.pi, "pss_destroy"); - grpc_exec_ctx_finish(&exec_ctx); + PI_UNREF(exec_ctx, pss->po.pi, "pss_destroy"); } gpr_free(pss); diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 6bae99a3443..c03fadaebb5 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -149,7 +149,7 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec, static bool fd_is_orphaned(grpc_fd *fd); /* Reference counting for fds */ -/*#define GRPC_FD_REF_COUNT_DEBUG*/ +//#define GRPC_FD_REF_COUNT_DEBUG #ifdef GRPC_FD_REF_COUNT_DEBUG static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); static void fd_unref(grpc_fd *fd, const char *reason, const char *file, @@ -283,8 +283,8 @@ cv_fd_table g_cvfds; static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, int line) { gpr_log(GPR_DEBUG, "FD %d %p ref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n, - gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); + (int)gpr_atm_no_barrier_load(&fd->refst), + (int)gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); #else #define REF_BY(fd, n, reason) ref_by(fd, n) #define UNREF_BY(fd, n, reason) unref_by(fd, n) @@ -298,8 +298,8 @@ static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, int line) { gpr_atm old; gpr_log(GPR_DEBUG, "FD %d %p unref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n, - gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); + (int)gpr_atm_no_barrier_load(&fd->refst), + (int)gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); #else static void unref_by(grpc_fd *fd, int n) { gpr_atm old; @@ -1137,12 +1137,27 @@ static grpc_pollset_set *pollset_set_create(void) { return pollset_set; } -static void pollset_set_destroy(grpc_pollset_set *pollset_set) { +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set) { size_t i; gpr_mu_destroy(&pollset_set->mu); for (i = 0; i < pollset_set->fd_count; i++) { GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set"); } + for (i = 0; i < pollset_set->pollset_count; i++) { + grpc_pollset *pollset = pollset_set->pollsets[i]; + gpr_mu_lock(&pollset->mu); + pollset->pollset_set_count--; + /* check shutdown */ + if (pollset->shutting_down && !pollset->called_shutdown && + !pollset_has_observers(pollset)) { + pollset->called_shutdown = 1; + gpr_mu_unlock(&pollset->mu); + finish_shutdown(exec_ctx, pollset); + } else { + gpr_mu_unlock(&pollset->mu); + } + } gpr_free(pollset_set->pollsets); gpr_free(pollset_set->pollset_sets); gpr_free(pollset_set->fds); @@ -1160,9 +1175,9 @@ static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, if (pollset_set->pollset_count == pollset_set->pollset_capacity) { pollset_set->pollset_capacity = GPR_MAX(8, 2 * pollset_set->pollset_capacity); - pollset_set->pollsets = gpr_realloc( - pollset_set->pollsets, - pollset_set->pollset_capacity * sizeof(*pollset_set->pollsets)); + pollset_set->pollsets = + gpr_realloc(pollset_set->pollsets, pollset_set->pollset_capacity * + sizeof(*pollset_set->pollsets)); } pollset_set->pollsets[pollset_set->pollset_count++] = pollset; for (i = 0, j = 0; i < pollset_set->fd_count; i++) { diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 95b1d99d757..b5be5504b9d 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -215,8 +215,9 @@ grpc_pollset_set *grpc_pollset_set_create(void) { return g_event_engine->pollset_set_create(); } -void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) { - g_event_engine->pollset_set_destroy(pollset_set); +void grpc_pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set) { + g_event_engine->pollset_set_destroy(exec_ctx, pollset_set); } void grpc_pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index 1aea7d61f32..1a9e5c115ae 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -74,7 +74,8 @@ typedef struct grpc_event_engine_vtable { struct grpc_fd *fd); grpc_pollset_set *(*pollset_set_create)(void); - void (*pollset_set_destroy)(grpc_pollset_set *pollset_set); + void (*pollset_set_destroy)(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set); void (*pollset_set_add_pollset)(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, grpc_pollset *pollset); diff --git a/src/core/lib/iomgr/pollset_set.h b/src/core/lib/iomgr/pollset_set.h index 34bb728c410..d11801d63b0 100644 --- a/src/core/lib/iomgr/pollset_set.h +++ b/src/core/lib/iomgr/pollset_set.h @@ -44,7 +44,8 @@ typedef struct grpc_pollset_set grpc_pollset_set; grpc_pollset_set *grpc_pollset_set_create(void); -void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set); +void grpc_pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set); void grpc_pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, grpc_pollset *pollset); diff --git a/src/core/lib/iomgr/pollset_set_uv.c b/src/core/lib/iomgr/pollset_set_uv.c index e5ef8b29e03..836cfee4efc 100644 --- a/src/core/lib/iomgr/pollset_set_uv.c +++ b/src/core/lib/iomgr/pollset_set_uv.c @@ -41,7 +41,8 @@ grpc_pollset_set* grpc_pollset_set_create(void) { return (grpc_pollset_set*)((intptr_t)0xdeafbeef); } -void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) {} +void grpc_pollset_set_destroy(grpc_exec_ctx* exec_ctx, + grpc_pollset_set* pollset_set) {} void grpc_pollset_set_add_pollset(grpc_exec_ctx* exec_ctx, grpc_pollset_set* pollset_set, diff --git a/src/core/lib/iomgr/pollset_set_windows.c b/src/core/lib/iomgr/pollset_set_windows.c index 645650db9be..ae18c8a3ce1 100644 --- a/src/core/lib/iomgr/pollset_set_windows.c +++ b/src/core/lib/iomgr/pollset_set_windows.c @@ -42,7 +42,8 @@ grpc_pollset_set* grpc_pollset_set_create(void) { return (grpc_pollset_set*)((intptr_t)0xdeafbeef); } -void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) {} +void grpc_pollset_set_destroy(grpc_exec_ctx* exec_ctx, + grpc_pollset_set* pollset_set) {} void grpc_pollset_set_add_pollset(grpc_exec_ctx* exec_ctx, grpc_pollset_set* pollset_set, diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c index a098741b707..ecd26de9faa 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.c +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c @@ -154,7 +154,7 @@ static int is_stack_running_on_compute_engine(grpc_exec_ctx *exec_ctx) { } gpr_mu_unlock(g_polling_mu); - grpc_httpcli_context_destroy(&context); + grpc_httpcli_context_destroy(exec_ctx, &context); grpc_closure_init(&destroy_closure, destroy_pollset, grpc_polling_entity_pollset(&detector.pollent), grpc_schedule_on_exec_ctx); diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c index 2270be8f449..f128177e8c4 100644 --- a/src/core/lib/security/credentials/jwt/jwt_verifier.c +++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c @@ -898,10 +898,10 @@ grpc_jwt_verifier *grpc_jwt_verifier_create( return v; } -void grpc_jwt_verifier_destroy(grpc_jwt_verifier *v) { +void grpc_jwt_verifier_destroy(grpc_exec_ctx *exec_ctx, grpc_jwt_verifier *v) { size_t i; if (v == NULL) return; - grpc_httpcli_context_destroy(&v->http_ctx); + grpc_httpcli_context_destroy(exec_ctx, &v->http_ctx); if (v->mappings != NULL) { for (i = 0; i < v->num_mappings; i++) { gpr_free(v->mappings[i].email_domain); diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.h b/src/core/lib/security/credentials/jwt/jwt_verifier.h index 4fa320a4159..5c3d2a77887 100644 --- a/src/core/lib/security/credentials/jwt/jwt_verifier.h +++ b/src/core/lib/security/credentials/jwt/jwt_verifier.h @@ -109,7 +109,8 @@ grpc_jwt_verifier *grpc_jwt_verifier_create( size_t num_mappings); /*The verifier must not be destroyed if there are still outstanding callbacks.*/ -void grpc_jwt_verifier_destroy(grpc_jwt_verifier *verifier); +void grpc_jwt_verifier_destroy(grpc_exec_ctx *exec_ctx, + grpc_jwt_verifier *verifier); /* User provided callback that will be called when the verification of the JWT is done (maybe in another thread). diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c index 1b0e43a1e46..c0f260f9380 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c @@ -124,7 +124,7 @@ static void oauth2_token_fetcher_destruct(grpc_exec_ctx *exec_ctx, (grpc_oauth2_token_fetcher_credentials *)creds; grpc_credentials_md_store_unref(exec_ctx, c->access_token_md); gpr_mu_destroy(&c->mu); - grpc_httpcli_context_destroy(&c->httpcli_context); + grpc_httpcli_context_destroy(exec_ctx, &c->httpcli_context); } grpc_credentials_status diff --git a/test/core/end2end/fixtures/http_proxy.c b/test/core/end2end/fixtures/http_proxy.c index 6fdc86fc12f..2682ea0e7b0 100644 --- a/test/core/end2end/fixtures/http_proxy.c +++ b/test/core/end2end/fixtures/http_proxy.c @@ -110,7 +110,7 @@ static void proxy_connection_unref(grpc_exec_ctx* exec_ctx, grpc_endpoint_destroy(exec_ctx, conn->client_endpoint); if (conn->server_endpoint != NULL) grpc_endpoint_destroy(exec_ctx, conn->server_endpoint); - grpc_pollset_set_destroy(conn->pollset_set); + grpc_pollset_set_destroy(exec_ctx, conn->pollset_set); grpc_slice_buffer_destroy_internal(exec_ctx, &conn->client_read_buffer); grpc_slice_buffer_destroy_internal(exec_ctx, &conn->client_deferred_write_buffer); diff --git a/test/core/http/httpcli_test.c b/test/core/http/httpcli_test.c index 6cc00f871d8..be8301c5e33 100644 --- a/test/core/http/httpcli_test.c +++ b/test/core/http/httpcli_test.c @@ -209,7 +209,7 @@ int main(int argc, char **argv) { test_get(port); test_post(port); - grpc_httpcli_context_destroy(&g_context); + grpc_httpcli_context_destroy(&exec_ctx, &g_context); grpc_closure_init(&destroyed, destroy_pops, &g_pops, grpc_schedule_on_exec_ctx); grpc_pollset_shutdown(&exec_ctx, grpc_polling_entity_pollset(&g_pops), diff --git a/test/core/http/httpscli_test.c b/test/core/http/httpscli_test.c index e1a26d91e9e..5a6f07bec25 100644 --- a/test/core/http/httpscli_test.c +++ b/test/core/http/httpscli_test.c @@ -212,7 +212,7 @@ int main(int argc, char **argv) { test_get(port); test_post(port); - grpc_httpcli_context_destroy(&g_context); + grpc_httpcli_context_destroy(&exec_ctx, &g_context); grpc_closure_init(&destroyed, destroy_pops, &g_pops, grpc_schedule_on_exec_ctx); grpc_pollset_shutdown(&exec_ctx, grpc_polling_entity_pollset(&g_pops), diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index 40fa8586020..e7777acce1e 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -59,10 +59,11 @@ void init_test_pollset_sets(test_pollset_set *pollset_sets, const int num_pss) { } } -void cleanup_test_pollset_sets(test_pollset_set *pollset_sets, +void cleanup_test_pollset_sets(grpc_exec_ctx *exec_ctx, + test_pollset_set *pollset_sets, const int num_pss) { for (int i = 0; i < num_pss; i++) { - grpc_pollset_set_destroy(pollset_sets[i].pss); + grpc_pollset_set_destroy(exec_ctx, pollset_sets[i].pss); pollset_sets[i].pss = NULL; } } @@ -297,7 +298,7 @@ static void pollset_set_test_basic() { cleanup_test_fds(&exec_ctx, tfds, num_fds); cleanup_test_pollsets(&exec_ctx, pollsets, num_ps); - cleanup_test_pollset_sets(pollset_sets, num_pss); + cleanup_test_pollset_sets(&exec_ctx, pollset_sets, num_pss); grpc_exec_ctx_finish(&exec_ctx); } @@ -372,7 +373,7 @@ void pollset_set_test_dup_fds() { cleanup_test_fds(&exec_ctx, tfds, num_fds); cleanup_test_pollsets(&exec_ctx, &pollset, num_ps); - cleanup_test_pollset_sets(pollset_sets, num_pss); + cleanup_test_pollset_sets(&exec_ctx, pollset_sets, num_pss); grpc_exec_ctx_finish(&exec_ctx); } @@ -437,7 +438,7 @@ void pollset_set_test_empty_pollset() { cleanup_test_fds(&exec_ctx, tfds, num_fds); cleanup_test_pollsets(&exec_ctx, pollsets, num_ps); - cleanup_test_pollset_sets(&pollset_set, num_pss); + cleanup_test_pollset_sets(&exec_ctx, &pollset_set, num_pss); grpc_exec_ctx_finish(&exec_ctx); } diff --git a/test/core/iomgr/resolve_address_posix_test.c b/test/core/iomgr/resolve_address_posix_test.c index a4feff8b009..ef4cfdf06f8 100644 --- a/test/core/iomgr/resolve_address_posix_test.c +++ b/test/core/iomgr/resolve_address_posix_test.c @@ -74,7 +74,7 @@ void args_finish(grpc_exec_ctx *exec_ctx, args_struct *args) { GPR_ASSERT(gpr_event_wait(&args->ev, test_deadline())); grpc_resolved_addresses_destroy(args->addrs); grpc_pollset_set_del_pollset(exec_ctx, args->pollset_set, args->pollset); - grpc_pollset_set_destroy(args->pollset_set); + grpc_pollset_set_destroy(exec_ctx, args->pollset_set); grpc_closure do_nothing_cb; grpc_closure_init(&do_nothing_cb, do_nothing, NULL, grpc_schedule_on_exec_ctx); diff --git a/test/core/iomgr/resolve_address_test.c b/test/core/iomgr/resolve_address_test.c index 54de9a20e1c..6a9bb5ae6f3 100644 --- a/test/core/iomgr/resolve_address_test.c +++ b/test/core/iomgr/resolve_address_test.c @@ -69,7 +69,7 @@ void args_finish(grpc_exec_ctx *exec_ctx, args_struct *args) { GPR_ASSERT(gpr_event_wait(&args->ev, test_deadline())); grpc_resolved_addresses_destroy(args->addrs); grpc_pollset_set_del_pollset(exec_ctx, args->pollset_set, args->pollset); - grpc_pollset_set_destroy(args->pollset_set); + grpc_pollset_set_destroy(exec_ctx, args->pollset_set); grpc_closure do_nothing_cb; grpc_closure_init(&do_nothing_cb, do_nothing, NULL, grpc_schedule_on_exec_ctx); diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index dcdff8efb13..c9b514a024a 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -207,7 +207,7 @@ int main(int argc, char **argv) { test_succeeds(); gpr_log(GPR_ERROR, "End of first test"); test_fails(); - grpc_pollset_set_destroy(g_pollset_set); + grpc_pollset_set_destroy(&exec_ctx, g_pollset_set); grpc_closure_init(&destroyed, destroy_pollset, g_pollset, grpc_schedule_on_exec_ctx); grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); diff --git a/test/core/security/jwt_verifier_test.c b/test/core/security/jwt_verifier_test.c index a9bd976a39d..0a73f675280 100644 --- a/test/core/security/jwt_verifier_test.c +++ b/test/core/security/jwt_verifier_test.c @@ -386,9 +386,9 @@ static void test_jwt_verifier_google_email_issuer_success(void) { GPR_ASSERT(jwt != NULL); grpc_jwt_verifier_verify(&exec_ctx, verifier, NULL, jwt, expected_audience, on_verification_success, (void *)expected_user_data); + grpc_jwt_verifier_destroy(&exec_ctx, verifier); grpc_exec_ctx_finish(&exec_ctx); gpr_free(jwt); - grpc_jwt_verifier_destroy(verifier); grpc_httpcli_set_override(NULL, NULL); } @@ -420,9 +420,9 @@ static void test_jwt_verifier_custom_email_issuer_success(void) { GPR_ASSERT(jwt != NULL); grpc_jwt_verifier_verify(&exec_ctx, verifier, NULL, jwt, expected_audience, on_verification_success, (void *)expected_user_data); + grpc_jwt_verifier_destroy(&exec_ctx, verifier); grpc_exec_ctx_finish(&exec_ctx); gpr_free(jwt); - grpc_jwt_verifier_destroy(verifier); grpc_httpcli_set_override(NULL, NULL); } @@ -469,9 +469,9 @@ static void test_jwt_verifier_url_issuer_success(void) { GPR_ASSERT(jwt != NULL); grpc_jwt_verifier_verify(&exec_ctx, verifier, NULL, jwt, expected_audience, on_verification_success, (void *)expected_user_data); + grpc_jwt_verifier_destroy(&exec_ctx, verifier); grpc_exec_ctx_finish(&exec_ctx); gpr_free(jwt); - grpc_jwt_verifier_destroy(verifier); grpc_httpcli_set_override(NULL, NULL); } @@ -511,9 +511,9 @@ static void test_jwt_verifier_url_issuer_bad_config(void) { grpc_jwt_verifier_verify(&exec_ctx, verifier, NULL, jwt, expected_audience, on_verification_key_retrieval_error, (void *)expected_user_data); + grpc_jwt_verifier_destroy(&exec_ctx, verifier); grpc_exec_ctx_finish(&exec_ctx); gpr_free(jwt); - grpc_jwt_verifier_destroy(verifier); grpc_httpcli_set_override(NULL, NULL); } @@ -534,9 +534,9 @@ static void test_jwt_verifier_bad_json_key(void) { grpc_jwt_verifier_verify(&exec_ctx, verifier, NULL, jwt, expected_audience, on_verification_key_retrieval_error, (void *)expected_user_data); + grpc_jwt_verifier_destroy(&exec_ctx, verifier); grpc_exec_ctx_finish(&exec_ctx); gpr_free(jwt); - grpc_jwt_verifier_destroy(verifier); grpc_httpcli_set_override(NULL, NULL); } @@ -588,9 +588,9 @@ static void test_jwt_verifier_bad_signature(void) { grpc_jwt_verifier_verify(&exec_ctx, verifier, NULL, jwt, expected_audience, on_verification_bad_signature, (void *)expected_user_data); - grpc_exec_ctx_finish(&exec_ctx); gpr_free(jwt); - grpc_jwt_verifier_destroy(verifier); + grpc_jwt_verifier_destroy(&exec_ctx, verifier); + grpc_exec_ctx_finish(&exec_ctx); grpc_httpcli_set_override(NULL, NULL); } @@ -619,8 +619,8 @@ static void test_jwt_verifier_bad_format(void) { grpc_jwt_verifier_verify(&exec_ctx, verifier, NULL, "bad jwt", expected_audience, on_verification_bad_format, (void *)expected_user_data); + grpc_jwt_verifier_destroy(&exec_ctx, verifier); grpc_exec_ctx_finish(&exec_ctx); - grpc_jwt_verifier_destroy(verifier); grpc_httpcli_set_override(NULL, NULL); } diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index bbd4a67ac1c..aaf0e7f6b18 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -123,14 +123,15 @@ int main(int argc, char **argv) { gpr_inf_future(GPR_CLOCK_MONOTONIC)))) sync.is_done = true; gpr_mu_unlock(sync.mu); - grpc_exec_ctx_finish(&exec_ctx); + grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(sync.mu); } gpr_mu_unlock(sync.mu); gpr_free(sync.pollset); - grpc_jwt_verifier_destroy(verifier); + grpc_jwt_verifier_destroy(&exec_ctx, verifier); + grpc_exec_ctx_finish(&exec_ctx); gpr_cmdline_destroy(cl); grpc_shutdown(); return !sync.success; diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index 6d722ffc882..7b733ab9c7d 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -121,7 +121,7 @@ void grpc_free_port_using_server(char *server, int port) { } gpr_mu_unlock(pr.mu); - grpc_httpcli_context_destroy(&context); + grpc_httpcli_context_destroy(&exec_ctx, &context); grpc_exec_ctx_finish(&exec_ctx); grpc_pollset_shutdown(&exec_ctx, grpc_polling_entity_pollset(&pr.pops), shutdown_closure); @@ -245,7 +245,7 @@ int grpc_pick_port_using_server(char *server) { gpr_mu_unlock(pr.mu); grpc_http_response_destroy(&pr.response); - grpc_httpcli_context_destroy(&context); + grpc_httpcli_context_destroy(&exec_ctx, &context); grpc_pollset_shutdown(&exec_ctx, grpc_polling_entity_pollset(&pr.pops), shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); From 37bfff35eeee1805a88b6ab5bbf91c434c52336e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Feb 2017 22:35:39 -0800 Subject: [PATCH 141/167] Fix port server --- tools/run_tests/python_utils/start_port_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/python_utils/start_port_server.py b/tools/run_tests/python_utils/start_port_server.py index d521aa6a9d9..8ee7080168a 100644 --- a/tools/run_tests/python_utils/start_port_server.py +++ b/tools/run_tests/python_utils/start_port_server.py @@ -36,6 +36,7 @@ import tempfile import sys import time import jobset +import socket def start_port_server(port_server_port): # check if a compatible port server is running From 7f9184f31450a0cff738b2d92a5e08535251348c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Feb 2017 22:39:47 -0800 Subject: [PATCH 142/167] Review feedback --- test/core/util/trickle_endpoint.c | 4 +++- test/cpp/microbenchmarks/bm_fullstack.cc | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/core/util/trickle_endpoint.c b/test/core/util/trickle_endpoint.c index 5b6c666950a..7ab0488a660 100644 --- a/test/core/util/trickle_endpoint.c +++ b/test/core/util/trickle_endpoint.c @@ -166,7 +166,9 @@ grpc_endpoint *grpc_trickle_endpoint_create(grpc_endpoint *wrap, return &te->base; } -static double ts2dbl(gpr_timespec s) { return s.tv_sec + 1e-9 * s.tv_nsec; } +static double ts2dbl(gpr_timespec s) { + return (double)s.tv_sec + 1e-9 * (double)s.tv_nsec; +} size_t grpc_trickle_endpoint_trickle(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index 619a03d4740..d449acd7444 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -355,7 +355,7 @@ class TrickledCHTTP2 : public EndpointPairFixture { grpc_endpoint_pair p; grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(), &stats_); - double bytes_per_second = 125 * kilobits; + double bytes_per_second = 125.0 * kilobits; p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second); p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second); return p; @@ -1022,7 +1022,8 @@ BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) static void TrickleArgs(benchmark::internal::Benchmark* b) { for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) { - double expected_time = (double)(14 + i) / (125 * (double)j); + double expected_time = + static_cast(14 + i) / (125.0 * static_cast(j)); if (expected_time > 0.01) continue; b->Args({i, j}); } From fd2cd7856b55f391d6bb43d2fd39904b0db3b397 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Feb 2017 06:53:51 -0800 Subject: [PATCH 143/167] Possible connection window flow control improvements Instead of incrementally doling out connection level flow control, use the fact that if a stream's initial_window_delta >= 0 then the application has made some commitment to read those bytes. That means that we should target connection level flow control to be: target_incoming_window = sum(stream_incoming_window_delta if stream_incoming_window_delta > 0) + max(0, k * bdp - sum(abs(stream_incoming_window_delta) if stream_incoming_window_delta < 0)) --- .../chttp2/transport/chttp2_transport.c | 12 ++++- .../ext/transport/chttp2/transport/internal.h | 54 +++++++++++++++++-- .../ext/transport/chttp2/transport/parsing.c | 20 ++++--- .../ext/transport/chttp2/transport/writing.c | 18 +++++-- 4 files changed, 83 insertions(+), 21 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 3ee5e976f84..93bae9f7bda 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -569,6 +569,14 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, GRPC_ERROR_UNREF(s->read_closed_error); GRPC_ERROR_UNREF(s->write_closed_error); + if (s->incoming_window_delta > 0) { + GRPC_CHTTP2_FLOW_DEBIT_STREAM_INCOMING_WINDOW_DELTA( + "destroy", t, s, s->incoming_window_delta); + } else if (s->incoming_window_delta < 0) { + GRPC_CHTTP2_FLOW_CREDIT_STREAM_INCOMING_WINDOW_DELTA( + "destroy", t, s, -s->incoming_window_delta); + } + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "stream"); GPR_TIMER_END("destroy_stream", 0); @@ -2054,8 +2062,8 @@ static void incoming_byte_stream_update_flow_control(grpc_exec_ctx *exec_ctx, (int64_t)have_already) { write_type = GRPC_CHTTP2_STREAM_WRITE_INITIATE_COVERED; } - GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", t, s, incoming_window_delta, - add_max_recv_bytes); + GRPC_CHTTP2_FLOW_CREDIT_STREAM_INCOMING_WINDOW_DELTA("op", t, s, + add_max_recv_bytes); GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", t, s, announce_window, add_max_recv_bytes); if ((int64_t)s->incoming_window_delta + (int64_t)initial_window_size - diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 075d421dd46..5d41f4bfda6 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -271,10 +271,6 @@ struct grpc_chttp2_transport { /** data to write next write */ grpc_slice_buffer qbuf; - /** window available to announce to peer */ - int64_t announce_incoming_window; - /** how much window would we like to have for incoming_window */ - uint32_t connection_window_target; /** how much data are we willing to buffer when the WRITE_BUFFER_HINT is set? */ uint32_t write_buffer_size; @@ -328,6 +324,16 @@ struct grpc_chttp2_transport { /** window available for peer to send to us */ int64_t incoming_window; + /** calculating what we should give for incoming window: + we track the total amount of flow control over initial window size + across all streams: this is data that we want to receive right now (it + has an outstanding read) + and the total amount of flow control under initial window size across all + streams: this is data we've read early + we want to adjust incoming_window such that: + incoming_window = total_over - max(bdp - total_under, 0) */ + int64_t stream_total_over_incoming_window; + int64_t stream_total_under_incoming_window; /* deframing */ grpc_chttp2_deframe_transport_state deframe_state; @@ -634,6 +640,44 @@ typedef enum { GRPC_CHTTP2_FLOW_CREDIT_COMMON(phase, dst_context, 0, dst_context, dst_var, \ amount) +#define GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_PREUPDATE( \ + phase, transport, dst_context) \ + if (dst_context->incoming_window_delta < 0) { \ + transport->stream_total_under_incoming_window += \ + dst_context->incoming_window_delta; \ + } else if (dst_context->incoming_window_delta > 0) { \ + transport->stream_total_over_incoming_window -= \ + dst_context->incoming_window_delta; \ + } + +#define GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_POSTUPDATE( \ + phase, transport, dst_context) \ + if (dst_context->incoming_window_delta < 0) { \ + transport->stream_total_under_incoming_window -= \ + dst_context->incoming_window_delta; \ + } else if (dst_context->incoming_window_delta > 0) { \ + transport->stream_total_over_incoming_window += \ + dst_context->incoming_window_delta; \ + } + +#define GRPC_CHTTP2_FLOW_DEBIT_STREAM_INCOMING_WINDOW_DELTA( \ + phase, transport, dst_context, amount) \ + GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_PREUPDATE(phase, transport, \ + dst_context); \ + GRPC_CHTTP2_FLOW_DEBIT_STREAM(phase, transport, dst_context, \ + incoming_window_delta, amount); \ + GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_POSTUPDATE(phase, transport, \ + dst_context); + +#define GRPC_CHTTP2_FLOW_CREDIT_STREAM_INCOMING_WINDOW_DELTA( \ + phase, transport, dst_context, amount) \ + GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_PREUPDATE(phase, transport, \ + dst_context); \ + GRPC_CHTTP2_FLOW_CREDIT_STREAM(phase, transport, dst_context, \ + incoming_window_delta, amount); \ + GRPC_CHTTP2_FLOW_STREAM_INCOMING_WINDOW_DELTA_POSTUPDATE(phase, transport, \ + dst_context); + #define GRPC_CHTTP2_FLOW_DEBIT_COMMON(phase, transport, id, dst_context, \ dst_var, amount) \ do { \ @@ -752,4 +796,6 @@ void grpc_chttp2_fail_pending_writes(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, grpc_error *error); +uint32_t grpc_chttp2_target_incoming_window(grpc_chttp2_transport *t); + #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_INTERNAL_H */ diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 24bd93067b3..7ed00522c3e 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -376,15 +376,6 @@ static grpc_error *update_incoming_window(grpc_exec_ctx *exec_ctx, return err; } - uint32_t target_incoming_window = GPR_MAX( - t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], - 1024); - GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("parse", t, incoming_window, - incoming_frame_size); - if (t->incoming_window <= target_incoming_window / 2) { - grpc_chttp2_initiate_write(exec_ctx, t, false, "flow_control"); - } - if (s != NULL) { if (incoming_frame_size > s->incoming_window_delta + @@ -402,8 +393,8 @@ static grpc_error *update_incoming_window(grpc_exec_ctx *exec_ctx, return err; } - GRPC_CHTTP2_FLOW_DEBIT_STREAM("parse", t, s, incoming_window_delta, - incoming_frame_size); + GRPC_CHTTP2_FLOW_DEBIT_STREAM_INCOMING_WINDOW_DELTA("parse", t, s, + incoming_frame_size); if ((int64_t)t->settings[GRPC_SENT_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] + (int64_t)s->incoming_window_delta - (int64_t)s->announce_window <= @@ -417,6 +408,13 @@ static grpc_error *update_incoming_window(grpc_exec_ctx *exec_ctx, s->received_bytes += incoming_frame_size; } + uint32_t target_incoming_window = grpc_chttp2_target_incoming_window(t); + GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("parse", t, incoming_window, + incoming_frame_size); + if (t->incoming_window <= target_incoming_window / 2) { + grpc_chttp2_initiate_write(exec_ctx, t, false, "flow_control"); + } + return GRPC_ERROR_NONE; } diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 05e6f59947a..2b9d93cae7c 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -152,6 +152,17 @@ static bool stream_ref_if_not_destroyed(gpr_refcount *r) { return true; } +uint32_t grpc_chttp2_target_incoming_window(grpc_chttp2_transport *t) { + return (uint32_t)GPR_MAX( + (int64_t)((1u << 31) - 1), + t->stream_total_over_incoming_window + + (int64_t)GPR_MAX( + t->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] - + t->stream_total_under_incoming_window, + 0)); +} + bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { grpc_chttp2_stream *s; @@ -310,13 +321,12 @@ bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, /* if the grpc_chttp2_transport is ready to send a window update, do so here also; 3/4 is a magic number that will likely get tuned soon */ - uint32_t target_incoming_window = GPR_MAX( - t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], - 1024); + uint32_t target_incoming_window = grpc_chttp2_target_incoming_window(t); uint32_t threshold_to_send_transport_window_update = t->outbuf.count > 0 ? 3 * target_incoming_window / 4 : target_incoming_window / 2; - if (t->incoming_window <= threshold_to_send_transport_window_update) { + if (t->incoming_window <= threshold_to_send_transport_window_update && + t->incoming_window != target_incoming_window) { maybe_initiate_ping(exec_ctx, t, GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE); uint32_t announced = (uint32_t)GPR_CLAMP( From 5a8c586c40ba88632efb75c0762fba9db0d278d6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Feb 2017 07:59:05 -0800 Subject: [PATCH 144/167] Add flamegraphs to index.html --- tools/run_tests/run_microbenchmark.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 6bba29d74ba..92685de5ee8 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -128,6 +128,7 @@ def collect_perf(bm_name, args): 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()]) for line in subprocess.check_output(['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines(): + link(line, '%s.svg' % fnize(line)) subprocess.check_call(['perf', 'record', '-o', '%s-perf.data' % fnize(line), '-g', '-c', '1000', 'bins/mutrace/%s' % bm_name, From e2894b42d73e56c85bda418b163beb49756f7140 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Feb 2017 08:12:55 -0800 Subject: [PATCH 145/167] Add some tracing --- .../ext/transport/chttp2/transport/chttp2_transport.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 93bae9f7bda..ddbc656b96d 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -265,7 +265,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, .gain_d = 0, .initial_control_value = log2(DEFAULT_WINDOW), .min_control_value = -1, - .max_control_value = 22, + .max_control_value = 25, .integral_range = 10}); grpc_chttp2_goaway_parser_init(&t->goaway_parser); @@ -1809,13 +1809,13 @@ static void update_bdp(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, if (delta == 0 || (bdp != 0 && delta > -1024 && delta < 1024)) { return; } + if (grpc_bdp_estimator_trace) { + gpr_log(GPR_DEBUG, "%s: update initial window size to %d", t->peer_string, + (int)bdp); + } push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, bdp); } -/******************************************************************************* - * INPUT PROCESSING - PARSING - */ - static grpc_error *try_http_parsing(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { grpc_http_parser parser; From d9bc2107e5c69e8e2f47c3c5487aadaef42c0fb0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Feb 2017 08:24:55 -0800 Subject: [PATCH 146/167] Update microbenchmarking framework for new benchmark --- test/cpp/microbenchmarks/bm_fullstack.cc | 8 ++++---- tools/profiling/microbenchmarks/bm2bq.py | 11 ++++++++++- tools/run_tests/run_microbenchmark.py | 13 ++++++++++--- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index f83c3599616..9496a3ac737 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -312,18 +312,18 @@ class TrickledCHTTP2 : public EndpointPairFixture { void AddToLabel(std::ostream& out, benchmark::State& state) { out << " writes/iter:" << ((double)stats_.num_writes / (double)state.iterations()) - << " cli-transport-stalls/iter:" + << " cli_transport_stalls/iter:" << ((double) client_stats_.streams_stalled_due_to_transport_flow_control / (double)state.iterations()) - << " cli-stream-stalls/iter:" + << " cli_stream_stalls/iter:" << ((double)client_stats_.streams_stalled_due_to_stream_flow_control / (double)state.iterations()) - << " svr-transport-stalls/iter:" + << " svr_transport_stalls/iter:" << ((double) server_stats_.streams_stalled_due_to_transport_flow_control / (double)state.iterations()) - << " svr-stream-stalls/iter:" + << " svr_stream_stalls/iter:" << ((double)server_stats_.streams_stalled_due_to_stream_flow_control / (double)state.iterations()); } diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 124dbdfec52..fd6207f42bb 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -61,6 +61,11 @@ columns = [ ('allocs_per_iteration', 'float'), ('locks_per_iteration', 'float'), ('writes_per_iteration', 'float'), + ('bandwidth_kilobits', 'integer'), + ('cli_transport_stalls_per_iteration', 'float'), + ('cli_stream_stalls_per_iteration', 'float'), + ('svr_transport_stalls_per_iteration', 'float'), + ('svr_stream_stalls_per_iteration', 'float'), ] if sys.argv[1] == '--schema': @@ -92,7 +97,11 @@ bm_specs = { 'BM_StreamingPingPongMsgs': { 'tpl': ['fixture', 'client_mutator', 'server_mutator'], 'dyn': ['request_size'], - } + }, + 'BM_PumpStreamServerToClient_Trickle': { + 'tpl': [], + 'dyn': ['request_size', 'bandwidth_kilobits'], + }, } def numericalize(s): diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 6bba29d74ba..3bb4a9547c8 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -149,9 +149,12 @@ def collect_summary(bm_name, args): subprocess.check_call( ['make', bm_name, 'CONFIG=counters', '-j', '%d' % multiprocessing.cpu_count()]) - text(subprocess.check_output(['bins/counters/%s' % bm_name, - '--benchmark_out=out.json', - '--benchmark_out_format=json'])) + cmd = ['bins/counters/%s' % bm_name, + '--benchmark_out=out.json', + '--benchmark_out_format=json'] + if args.summary_time is not None: + cmd += ['--benchmark_min_time=%d' % args.summary_time] + text(subprocess.check_output(cmd)) if args.bigquery_upload: with open('out.csv', 'w') as f: f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.json'])) @@ -179,6 +182,10 @@ argp.add_argument('--bigquery_upload', action='store_const', const=True, help='Upload results from summary collection to bigquery') +argp.add_argument('--summary_time', + default=None, + type=int, + help='Minimum time to run benchmarks for the summary collection') args = argp.parse_args() for bm_name in args.benchmarks: From 3f878bc78493bc3f0d8a826740355be445ffab7a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Feb 2017 08:33:09 -0800 Subject: [PATCH 147/167] Update microbenchmarking framework for new benchmark --- tools/profiling/microbenchmarks/bm2bq.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index fd6207f42bb..a7d82269f55 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -115,6 +115,8 @@ def numericalize(s): assert 'not a number: %s' % s def parse_name(name): + if '<' not in name and '/' not in name and name not in bm_specs: + return {'name': name} rest = name out = {} tpl_args = [] @@ -145,7 +147,7 @@ def parse_name(name): rest = s[0] dyn_args = s[1:] name = rest - assert name in bm_specs + assert name in bm_specs, 'bm_specs needs to be expanded for %s' % name assert len(dyn_args) == len(bm_specs[name]['dyn']) assert len(tpl_args) == len(bm_specs[name]['tpl']) out['name'] = name @@ -155,10 +157,13 @@ def parse_name(name): for bm in js['benchmarks']: context = js['context'] - labels_list = [s.split(':') for s in bm.get('label', '').split(' ')] - for el in labels_list: - el[0] = el[0].replace('/iter', '_per_iteration') - labels = dict(labels_list) + if 'label' in bm: + labels_list = [s.split(':') for s in bm['label'].split(' ')] + for el in labels_list: + el[0] = el[0].replace('/iter', '_per_iteration') + labels = dict(labels_list) + else: + labels = {} row = { 'jenkins_build': os.environ.get('BUILD_NUMBER', ''), 'jenkins_job': os.environ.get('JOB_NAME', ''), @@ -167,5 +172,6 @@ for bm in js['benchmarks']: row.update(bm) row.update(parse_name(row['name'])) row.update(labels) - del row['label'] + if 'label' in row: + del row['label'] writer.writerow(row) From 86e369165a69a2c87ea60e6702272d0f8c8d3d24 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Feb 2017 08:35:55 -0800 Subject: [PATCH 148/167] Update microbenchmarking framework for new benchmark --- tools/run_tests/run_microbenchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 3bb4a9547c8..d748e5a0ce4 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -173,7 +173,7 @@ argp.add_argument('-c', '--collect', default=sorted(collectors.keys()), help='Which collectors should be run against each benchmark') argp.add_argument('-b', '--benchmarks', - default=['bm_fullstack'], + default=['bm_fullstack', 'bm_closure'], nargs='+', type=str, help='Which microbenchmarks should be run') From 6ad0072f663f8b6c3816d267d0b71f5520a07157 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Feb 2017 09:14:24 -0800 Subject: [PATCH 149/167] Run perf benchmarks in parallel --- tools/run_tests/run_microbenchmark.py | 51 +++++++++++++++++++-------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 92685de5ee8..bda31f6b3d5 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -126,24 +126,45 @@ def collect_perf(bm_name, args): subprocess.check_call( ['make', bm_name, 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()]) + benchmarks = [] + profile_analysis = [] + cleanup = [] for line in subprocess.check_output(['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines(): link(line, '%s.svg' % fnize(line)) - subprocess.check_call(['perf', 'record', '-o', '%s-perf.data' % fnize(line), - '-g', '-c', '1000', - 'bins/mutrace/%s' % bm_name, - '--benchmark_filter=^%s$' % line, - '--benchmark_min_time=10']) - env = os.environ.copy() - env.update({ - 'PERF_BASE_NAME': fnize(line), - 'OUTPUT_DIR': 'reports', - 'OUTPUT_FILENAME': fnize(line), - }) - subprocess.check_call(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'], - env=env) - subprocess.check_call(['rm', '%s-perf.data' % fnize(line)]) - subprocess.check_call(['rm', '%s-out.perf' % fnize(line)]) + benchmarks.append( + jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line), + '-g', '-c', '1000', + 'bins/mutrace/%s' % bm_name, + '--benchmark_filter=^%s$' % line, + '--benchmark_min_time=10'])) + profile_analysis.append( + jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'], + environ = { + 'PERF_BASE_NAME': fnize(line), + 'OUTPUT_DIR': 'reports', + 'OUTPUT_FILENAME': fnize(line), + })) + cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)])) + cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)])) + # periodically flush out the list of jobs: temporary space required for this + # processing is large + if len(benchmarks) >= 20: + # run up to half the cpu count: each benchmark can use up to two cores + # (one for the microbenchmark, one for the data flush) + jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2), + add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) + jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) + benchmarks = [] + profile_analysis = [] + cleanup = [] + # run the remaining benchmarks that weren't flushed + if len(benchmarks): + jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2), + add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) + jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) def collect_summary(bm_name, args): heading('Summary: %s' % bm_name) From fafe99590027b882a5cf536f97fccb9ffffdd19a Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 15 Feb 2017 10:53:25 -0800 Subject: [PATCH 150/167] comments --- include/grpc++/test/server_context_test_spouse.h | 5 +++-- src/core/lib/surface/call.h | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/grpc++/test/server_context_test_spouse.h b/include/grpc++/test/server_context_test_spouse.h index 4cb84e77c19..bac0db7bdc5 100644 --- a/include/grpc++/test/server_context_test_spouse.h +++ b/include/grpc++/test/server_context_test_spouse.h @@ -55,8 +55,9 @@ class ServerContextTestSpouse { for (auto iter = client_metadata_storage_.begin(); iter != client_metadata_storage_.end(); ++iter) { ctx_->client_metadata_.map()->insert( - std::pair(iter->first.c_str(), - iter->second.c_str())); + std::pair( + iter->first.c_str(), + grpc::string_ref(iter->second.data(), iter->second.size()))); } } diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index b70343ddf18..7d4d0db28d9 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -110,6 +110,7 @@ void grpc_call_log_batch(char *file, int line, gpr_log_severity severity, /* Set a context pointer. No thread safety guarantees are made wrt this value. */ +/* TODO(#9731): add exec_ctx to destroy */ void grpc_call_context_set(grpc_call *call, grpc_context_index elem, void *value, void (*destroy)(void *value)); /* Get a context pointer. */ From d5b389d401630ffef65886291f421ef074525f9e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Feb 2017 11:25:50 -0800 Subject: [PATCH 151/167] Serialize running of perf, switch back to -f for consistent event counts --- tools/run_tests/run_microbenchmark.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index bda31f6b3d5..a2852546b37 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -134,7 +134,7 @@ def collect_perf(bm_name, args): link(line, '%s.svg' % fnize(line)) benchmarks.append( jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line), - '-g', '-c', '1000', + '-g', '-f', '997', 'bins/mutrace/%s' % bm_name, '--benchmark_filter=^%s$' % line, '--benchmark_min_time=10'])) @@ -152,7 +152,7 @@ def collect_perf(bm_name, args): if len(benchmarks) >= 20: # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) - jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2), + jobset.run(benchmarks, maxjobs=1), add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) @@ -161,7 +161,7 @@ def collect_perf(bm_name, args): cleanup = [] # run the remaining benchmarks that weren't flushed if len(benchmarks): - jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2), + jobset.run(benchmarks, maxjobs=1), add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) From ab951947ce07336769682152880dd9eab2e03c80 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Feb 2017 11:26:54 -0800 Subject: [PATCH 152/167] Syntax fix --- tools/run_tests/run_microbenchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index a2852546b37..05a12332902 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -152,7 +152,7 @@ def collect_perf(bm_name, args): if len(benchmarks) >= 20: # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) - jobset.run(benchmarks, maxjobs=1), + jobset.run(benchmarks, maxjobs=1, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) @@ -161,7 +161,7 @@ def collect_perf(bm_name, args): cleanup = [] # run the remaining benchmarks that weren't flushed if len(benchmarks): - jobset.run(benchmarks, maxjobs=1), + jobset.run(benchmarks, maxjobs=1, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) From c2c0c6f549df0428f817a19c9366919f88f5b32b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Feb 2017 11:27:37 -0800 Subject: [PATCH 153/167] capitALization --- tools/run_tests/run_microbenchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 05a12332902..4e60cce8688 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -134,7 +134,7 @@ def collect_perf(bm_name, args): link(line, '%s.svg' % fnize(line)) benchmarks.append( jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line), - '-g', '-f', '997', + '-g', '-F', '997', 'bins/mutrace/%s' % bm_name, '--benchmark_filter=^%s$' % line, '--benchmark_min_time=10'])) From 18570ec948bc45c03600fe5b41b79afc53f66bb3 Mon Sep 17 00:00:00 2001 From: Yong Ni Date: Wed, 15 Feb 2017 12:49:32 -0800 Subject: [PATCH 154/167] Wed Feb 15 12:49:32 PST 2017 --- tools/internal_ci/linux/grpc_interop.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_interop.cfg b/tools/internal_ci/linux/grpc_interop.cfg index ecafcf62ae9..23b83d2f91e 100644 --- a/tools/internal_ci/linux/grpc_interop.cfg +++ b/tools/internal_ci/linux/grpc_interop.cfg @@ -32,7 +32,8 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_interop.sh" -timeout_mins: 60 +# grpc_interop tests can take MANY hours to complete. +timeout_mins: 480 action { define_artifacts { regex: "**/sponge_log.xml" From 0392025bff4f845607af1c8120117c70a629af67 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 15 Feb 2017 12:51:21 -0800 Subject: [PATCH 155/167] Better error reporting in run_tests.py --- tools/run_tests/run_tests.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 8b76193cbe1..3d9b538ca77 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -80,6 +80,13 @@ def platform_string(): _DEFAULT_TIMEOUT_SECONDS = 5 * 60 +def run_shell_command(cmd, env=None, cwd=None): + try: + subprocess.check_output(cmd, shell=True, env=env, cwd=cwd) + except subprocess.CalledProcessError as e: + print("Error while running command '%s'. Exit status %d. Output:\n%s", + e.cmd, e.returncode, e.output) + raise # SimpleConfig: just compile with CONFIG=config, and run the binary to test class Config(object): @@ -1199,7 +1206,7 @@ for spec in args.update_submodules: cwd = 'third_party/%s' % submodule def git(cmd, cwd=cwd): print('in %s: git %s' % (cwd, cmd)) - subprocess.check_call('git %s' % cmd, cwd=cwd, shell=True) + run_shell_command('git %s' % cmd, cwd=cwd) git('fetch') git('checkout %s' % branch) git('pull origin %s' % branch) @@ -1207,7 +1214,7 @@ for spec in args.update_submodules: need_to_regenerate_projects = True if need_to_regenerate_projects: if jobset.platform_string() == 'linux': - subprocess.check_call('tools/buildgen/generate_projects.sh', shell=True) + run_shell_command('tools/buildgen/generate_projects.sh') else: print('WARNING: may need to regenerate projects, but since we are not on') print(' Linux this step is being skipped. Compilation MAY fail.') @@ -1276,9 +1283,7 @@ if args.use_docker: if not args.travis: env['TTY_FLAG'] = '-t' # enables Ctrl-C when not on Jenkins. - subprocess.check_call(['tools/run_tests/dockerize/build_docker_and_run_tests.sh'], - shell=True, - env=env) + run_shell_command('tools/run_tests/dockerize/build_docker_and_run_tests.sh', env=env) sys.exit(0) _check_arch_option(args.arch) From 17342333a8d36bed87fb01cabb5b685e20c31edf Mon Sep 17 00:00:00 2001 From: yongni Date: Wed, 15 Feb 2017 13:19:41 -0800 Subject: [PATCH 156/167] Update grpc_interop.cfg --- tools/internal_ci/linux/grpc_interop.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_interop.cfg b/tools/internal_ci/linux/grpc_interop.cfg index 23b83d2f91e..9259faf34d5 100644 --- a/tools/internal_ci/linux/grpc_interop.cfg +++ b/tools/internal_ci/linux/grpc_interop.cfg @@ -32,7 +32,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_interop.sh" -# grpc_interop tests can take MANY hours to complete. +# grpc_interop tests can take 6+ hours to complete. timeout_mins: 480 action { define_artifacts { From 3577f63136e1f1d6e26a00176a0cff6d9b2d9934 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Wed, 15 Feb 2017 22:09:16 +0000 Subject: [PATCH 157/167] Fix and enable logging-format-interpolation lint --- .pylintrc | 3 +-- src/python/grpcio/grpc/_common.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.pylintrc b/.pylintrc index a4f757f9bdd..23375149c7a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -32,7 +32,6 @@ #TODO: Enable simplifiable-if-statement #TODO: Enable no-self-use #TODO: Enable no-member -#TODO: Enable logging-format-interpolation #TODO: Enable dangerous-default-value -disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,suppressed-message,locally-disabled,protected-access,no-name-in-module,unused-argument,fixme,wrong-import-order,no-value-for-parameter,cyclic-import,unused-variable,redefined-outer-name,unused-import,too-many-instance-attributes,broad-except,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called,simplifiable-if-statement,no-self-use,no-member,logging-format-interpolation,dangerous-default-value +disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,suppressed-message,locally-disabled,protected-access,no-name-in-module,unused-argument,fixme,wrong-import-order,no-value-for-parameter,cyclic-import,unused-variable,redefined-outer-name,unused-import,too-many-instance-attributes,broad-except,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called,simplifiable-if-statement,no-self-use,no-member,dangerous-default-value diff --git a/src/python/grpcio/grpc/_common.py b/src/python/grpcio/grpc/_common.py index a5f1f38ea97..dd74ba3ab61 100644 --- a/src/python/grpcio/grpc/_common.py +++ b/src/python/grpcio/grpc/_common.py @@ -92,7 +92,7 @@ def decode(b): try: return b.decode('utf8') except UnicodeDecodeError: - logging.exception('Invalid encoding on {}'.format(b)) + logging.exception('Invalid encoding on %s', b) return b.decode('latin1') From 402aca6621e2880b39de68ee98acdf43f6a211da Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Wed, 15 Feb 2017 22:43:52 +0000 Subject: [PATCH 158/167] Fix and enable dangerous-default-value lint The fix is more of an elision and the code in the changed area is reflowed. --- .pylintrc | 3 +-- src/python/grpcio/grpc/_common.py | 43 +++++++++++-------------------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/.pylintrc b/.pylintrc index 23375149c7a..e887b1cdbd2 100644 --- a/.pylintrc +++ b/.pylintrc @@ -32,6 +32,5 @@ #TODO: Enable simplifiable-if-statement #TODO: Enable no-self-use #TODO: Enable no-member -#TODO: Enable dangerous-default-value -disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,suppressed-message,locally-disabled,protected-access,no-name-in-module,unused-argument,fixme,wrong-import-order,no-value-for-parameter,cyclic-import,unused-variable,redefined-outer-name,unused-import,too-many-instance-attributes,broad-except,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called,simplifiable-if-statement,no-self-use,no-member,dangerous-default-value +disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,suppressed-message,locally-disabled,protected-access,no-name-in-module,unused-argument,fixme,wrong-import-order,no-value-for-parameter,cyclic-import,unused-variable,redefined-outer-name,unused-import,too-many-instance-attributes,broad-except,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called,simplifiable-if-statement,no-self-use,no-member diff --git a/src/python/grpcio/grpc/_common.py b/src/python/grpcio/grpc/_common.py index dd74ba3ab61..f9accd75a9f 100644 --- a/src/python/grpcio/grpc/_common.py +++ b/src/python/grpcio/grpc/_common.py @@ -148,36 +148,23 @@ def fully_qualified_method(group, method): class CleanupThread(threading.Thread): """A threading.Thread subclass supporting custom behavior on join(). - On Python Interpreter exit, Python will attempt to join outstanding threads - prior to garbage collection. We may need to do additional cleanup, and - we accomplish this by overriding the join() method. - """ - - def __init__(self, - behavior, - group=None, - target=None, - name=None, - args=(), - kwargs={}): + On Python Interpreter exit, Python will attempt to join outstanding threads + prior to garbage collection. We may need to do additional cleanup, and + we accomplish this by overriding the join() method. + """ + + def __init__(self, behavior, *args, **kwargs): """Constructor. - Args: - behavior (function): Function called on join() with a single - argument, timeout, indicating the maximum duration of - `behavior`, or None indicating `behavior` has no deadline. - `behavior` must be idempotent. - group (None): should be None. Reseved for future extensions - when ThreadGroup is implemented. - target (function): The function to invoke when this thread is - run. Defaults to None. - name (str): The name of this thread. Defaults to None. - args (tuple[object]): A tuple of arguments to pass to `target`. - kwargs (dict[str,object]): A dictionary of keyword arguments to - pass to `target`. - """ - super(CleanupThread, self).__init__( - group=group, target=target, name=name, args=args, kwargs=kwargs) + Args: + behavior (function): Function called on join() with a single + argument, timeout, indicating the maximum duration of + `behavior`, or None indicating `behavior` has no deadline. + `behavior` must be idempotent. + args: Positional arguments passed to threading.Thread constructor. + kwargs: Keyword arguments passed to threading.Thread constructor. + """ + super(CleanupThread, self).__init__(*args, **kwargs) self._behavior = behavior def join(self, timeout=None): From a52d3bb39b79039ebc66725773289035f0c002d7 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Wed, 15 Feb 2017 23:00:05 +0000 Subject: [PATCH 159/167] Fix and enable simplifiable-if-statement lint --- .pylintrc | 3 +-- src/python/grpcio/grpc/_auth.py | 7 ++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.pylintrc b/.pylintrc index e887b1cdbd2..1682488cb4a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -29,8 +29,7 @@ #TODO: Enable too-many-return-statements #TODO: Enable too-many-nested-blocks #TODO: Enable super-init-not-called -#TODO: Enable simplifiable-if-statement #TODO: Enable no-self-use #TODO: Enable no-member -disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,suppressed-message,locally-disabled,protected-access,no-name-in-module,unused-argument,fixme,wrong-import-order,no-value-for-parameter,cyclic-import,unused-variable,redefined-outer-name,unused-import,too-many-instance-attributes,broad-except,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called,simplifiable-if-statement,no-self-use,no-member +disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,suppressed-message,locally-disabled,protected-access,no-name-in-module,unused-argument,fixme,wrong-import-order,no-value-for-parameter,cyclic-import,unused-variable,redefined-outer-name,unused-import,too-many-instance-attributes,broad-except,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called,no-self-use,no-member diff --git a/src/python/grpcio/grpc/_auth.py b/src/python/grpcio/grpc/_auth.py index e8a90cf5047..21131f85f17 100644 --- a/src/python/grpcio/grpc/_auth.py +++ b/src/python/grpcio/grpc/_auth.py @@ -48,11 +48,8 @@ class GoogleCallCredentials(grpc.AuthMetadataPlugin): # Hack to determine if these are JWT creds and we need to pass # additional_claims when getting a token - if 'additional_claims' in inspect.getargspec( - credentials.get_access_token).args: - self._is_jwt = True - else: - self._is_jwt = False + self._is_jwt = 'additional_claims' in inspect.getargspec( + credentials.get_access_token).args def __call__(self, context, callback): # MetadataPlugins cannot block (see grpc.beta.interfaces.py) From 73495e3b3507438828d34376fa06fd68db18250e Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Wed, 15 Feb 2017 23:06:30 +0000 Subject: [PATCH 160/167] Ignore python_pylint_venv/. This should have been included in 1a967c31c3a7fbd2d6d5cb6c1e575fdb2d. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4098106e760..56e4b6d4b43 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ objs cython_debug/ python_build/ python_format_venv/ +python_pylint_venv/ .coverage* .eggs htmlcov/ From b3a1ec2111676d1504e1fdef41fdf8e046c1e283 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Wed, 15 Feb 2017 23:07:40 +0000 Subject: [PATCH 161/167] Output diff on failed yapf sanity --- tools/distrib/yapf_code.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/distrib/yapf_code.sh b/tools/distrib/yapf_code.sh index 030412ebeb9..007b14810eb 100755 --- a/tools/distrib/yapf_code.sh +++ b/tools/distrib/yapf_code.sh @@ -53,7 +53,7 @@ for dir in $DIRS; do tempdir=`mktemp -d` cp -RT $dir $tempdir $PYTHON -m yapf -i -r -p $exclusion_args $dir - if ! diff -rq $dir $tempdir; then + if ! diff -r $dir $tempdir; then script_result=1 fi rm -rf $tempdir From 95b37b7dfd7e11a50cec015f248f26a6485e4373 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 15 Feb 2017 16:49:49 -0800 Subject: [PATCH 162/167] Add random samping to run_tests.py --- tools/run_tests/run_tests.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 8b76193cbe1..54803de6804 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1092,6 +1092,18 @@ def runs_per_test_type(arg_str): raise argparse.ArgumentTypeError(msg) +def percent_type(arg_str): + pct = float(arg_str) + if pct > 100 or pct < 0: + raise argparse.ArgumentTypeError( + "'%f' is not a valid percentage in the [0, 100] range" % pct) + return pct + +# This is math.isclose in python >= 3.5 +def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) + + # parse command line argp = argparse.ArgumentParser(description='Run grpc tests.') argp.add_argument('-c', '--config', @@ -1104,6 +1116,8 @@ argp.add_argument('-r', '--regex', default='.*', type=str) argp.add_argument('--regex_exclude', default='', type=str) argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) argp.add_argument('-s', '--slowdown', default=1.0, type=float) +argp.add_argument('-p', '--sample_percent', default=100.0, type=percent_type, + help='Run a random sample with that percentage of tests') argp.add_argument('-f', '--forever', default=False, action='store_const', @@ -1438,8 +1452,16 @@ def _build_and_run( else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. - massaged_one_run = list(one_run) # random.shuffle needs an indexable seq. - random.shuffle(massaged_one_run) # which it modifies in-place. + massaged_one_run = list(one_run) # random.sample needs an indexable seq. + num_jobs = len(massaged_one_run) + # for a random sample, get as many as indicated by the 'sample_percent' + # argument. By default this arg is 100, resulting in a shuffle of all + # jobs. + sample_size = int(num_jobs * args.sample_percent/100.0) + massaged_one_run = random.sample(massaged_one_run, sample_size) + if not isclose(args.sample_percent, 100.0): + print("Running %d tests out of %d (~%d%%)" % + (sample_size, num_jobs, args.sample_percent)) if infinite_runs: assert len(massaged_one_run) > 0, 'Must have at least one test for a -n inf run' runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs @@ -1450,7 +1472,7 @@ def _build_and_run( jobset.message('START', 'Running tests quietly, only failing tests will be reported', do_newline=True) num_test_failures, resultset = jobset.run( all_runs, check_cancelled, newline_on_success=newline_on_success, - travis=args.travis, infinite_runs=infinite_runs, maxjobs=args.jobs, + travis=args.travis, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}, quiet_success=args.quiet_success) From 891e8162e409b7048297c5b820a399f19f2a020b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Feb 2017 23:30:27 -0800 Subject: [PATCH 163/167] Fix escaping in index.html --- tools/run_tests/run_microbenchmark.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 4e60cce8688..9bcba598b83 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -28,6 +28,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import cgi import multiprocessing import os import subprocess @@ -71,11 +72,12 @@ def heading(name): def link(txt, tgt): global index_html - index_html += "

%s

\n" % (tgt, txt) + index_html += "

%s

\n" % ( + cgi.escape(tgt, quote=True), cgi.escape(txt)) def text(txt): global index_html - index_html += "

%s

\n" % txt + index_html += "

%s

\n" % cgi.escape(txt) def collect_latency(bm_name, args): """generate latency profiles""" From 6955c5e8d2c881169dfa4ffd80a38182d6e2e14e Mon Sep 17 00:00:00 2001 From: yang-g Date: Mon, 13 Feb 2017 15:49:27 -0800 Subject: [PATCH 164/167] Remove code from network_status_tracker.c Pull the hash function to useful.h --- include/grpc/support/useful.h | 3 + src/core/lib/iomgr/network_status_tracker.c | 87 ++----------------- src/core/lib/iomgr/timer_generic.c | 10 +-- src/core/lib/support/cpu_posix.c | 9 +- .../end2end/tests/network_status_change.c | 5 +- 5 files changed, 16 insertions(+), 98 deletions(-) diff --git a/include/grpc/support/useful.h b/include/grpc/support/useful.h index 003e096cf9a..9d8314e4bef 100644 --- a/include/grpc/support/useful.h +++ b/include/grpc/support/useful.h @@ -74,4 +74,7 @@ #define GPR_ICMP(a, b) ((a) < (b) ? -1 : ((a) > (b) ? 1 : 0)) +#define GPR_HASH_POINTER(x, range) \ + ((((size_t)x) >> 4) ^ (((size_t)x) >> 9) ^ (((size_t)x) >> 14)) % (range) + #endif /* GRPC_SUPPORT_USEFUL_H */ diff --git a/src/core/lib/iomgr/network_status_tracker.c b/src/core/lib/iomgr/network_status_tracker.c index 1601a390028..4104bf927a1 100644 --- a/src/core/lib/iomgr/network_status_tracker.c +++ b/src/core/lib/iomgr/network_status_tracker.c @@ -31,95 +31,18 @@ * */ -#include -#include #include "src/core/lib/iomgr/endpoint.h" -typedef struct endpoint_ll_node { - grpc_endpoint *ep; - struct endpoint_ll_node *next; -} endpoint_ll_node; - -static endpoint_ll_node *head = NULL; -static gpr_mu g_endpoint_mutex; - -void grpc_network_status_shutdown(void) { - if (head != NULL) { - gpr_log(GPR_ERROR, - "Memory leaked as not all network endpoints were shut down"); - } - gpr_mu_destroy(&g_endpoint_mutex); -} +void grpc_network_status_shutdown(void) {} void grpc_network_status_init(void) { - gpr_mu_init(&g_endpoint_mutex); // TODO(makarandd): Install callback with OS to monitor network status. } -void grpc_destroy_network_status_monitor() { - for (endpoint_ll_node *curr = head; curr != NULL;) { - endpoint_ll_node *next = curr->next; - gpr_free(curr); - curr = next; - } - gpr_mu_destroy(&g_endpoint_mutex); -} - -void grpc_network_status_register_endpoint(grpc_endpoint *ep) { - gpr_mu_lock(&g_endpoint_mutex); - if (head == NULL) { - head = (endpoint_ll_node *)gpr_malloc(sizeof(endpoint_ll_node)); - head->ep = ep; - head->next = NULL; - } else { - endpoint_ll_node *prev_head = head; - head = (endpoint_ll_node *)gpr_malloc(sizeof(endpoint_ll_node)); - head->ep = ep; - head->next = prev_head; - } - gpr_mu_unlock(&g_endpoint_mutex); -} +void grpc_destroy_network_status_monitor() {} -void grpc_network_status_unregister_endpoint(grpc_endpoint *ep) { - gpr_mu_lock(&g_endpoint_mutex); - GPR_ASSERT(head); - bool found = false; - endpoint_ll_node *prev = head; - // if we're unregistering the head, just move head to the next - if (ep == head->ep) { - head = head->next; - gpr_free(prev); - found = true; - } else { - for (endpoint_ll_node *curr = head->next; curr != NULL; curr = curr->next) { - if (ep == curr->ep) { - prev->next = curr->next; - gpr_free(curr); - found = true; - break; - } - prev = curr; - } - } - gpr_mu_unlock(&g_endpoint_mutex); - GPR_ASSERT(found); -} +void grpc_network_status_register_endpoint(grpc_endpoint *ep) { (void)ep; } -// Walk the linked-list from head and execute shutdown. It is possible that -// other threads might be in the process of shutdown as well, but that has -// no side effect since endpoint shutdown is idempotent. -void grpc_network_status_shutdown_all_endpoints() { - gpr_mu_lock(&g_endpoint_mutex); - if (head == NULL) { - gpr_mu_unlock(&g_endpoint_mutex); - return; - } - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; +void grpc_network_status_unregister_endpoint(grpc_endpoint *ep) { (void)ep; } - for (endpoint_ll_node *curr = head; curr != NULL; curr = curr->next) { - curr->ep->vtable->shutdown(&exec_ctx, curr->ep, - GRPC_ERROR_CREATE("Network unavailable")); - } - gpr_mu_unlock(&g_endpoint_mutex); - grpc_exec_ctx_finish(&exec_ctx); -} +void grpc_network_status_shutdown_all_endpoints() {} diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index 40c83514724..8a5617e7c13 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -121,12 +121,6 @@ void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx) { g_initialized = false; } -/* This is a cheap, but good enough, pointer hash for sharding the tasks: */ -static size_t shard_idx(const grpc_timer *info) { - size_t x = (size_t)info; - return ((x >> 4) ^ (x >> 9) ^ (x >> 14)) & (NUM_SHARDS - 1); -} - static double ts_to_dbl(gpr_timespec ts) { return (double)ts.tv_sec + 1e-9 * ts.tv_nsec; } @@ -181,7 +175,7 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, gpr_timespec deadline, grpc_closure *closure, gpr_timespec now) { int is_first_timer = 0; - shard_type *shard = &g_shards[shard_idx(timer)]; + shard_type *shard = &g_shards[GPR_HASH_POINTER(timer, NUM_SHARDS)]; GPR_ASSERT(deadline.clock_type == g_clock_type); GPR_ASSERT(now.clock_type == g_clock_type); timer->closure = closure; @@ -247,7 +241,7 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) { return; } - shard_type *shard = &g_shards[shard_idx(timer)]; + shard_type *shard = &g_shards[GPR_HASH_POINTER(timer, NUM_SHARDS)]; gpr_mu_lock(&shard->mu); if (!timer->triggered) { grpc_closure_sched(exec_ctx, timer->closure, GRPC_ERROR_CANCELLED); diff --git a/src/core/lib/support/cpu_posix.c b/src/core/lib/support/cpu_posix.c index 667bde7cad4..245f12f06d7 100644 --- a/src/core/lib/support/cpu_posix.c +++ b/src/core/lib/support/cpu_posix.c @@ -41,6 +41,7 @@ #include #include +#include static __thread char magic_thread_local; @@ -60,18 +61,12 @@ unsigned gpr_cpu_num_cores(void) { return (unsigned)ncpus; } -/* This is a cheap, but good enough, pointer hash for sharding things: */ -static size_t shard_ptr(const void *info) { - size_t x = (size_t)info; - return ((x >> 4) ^ (x >> 9) ^ (x >> 14)) % gpr_cpu_num_cores(); -} - unsigned gpr_cpu_current_cpu(void) { /* NOTE: there's no way I know to return the actual cpu index portably... most code that's using this is using it to shard across work queues though, so here we use thread identity instead to achieve a similar though not identical effect */ - return (unsigned)shard_ptr(&magic_thread_local); + return (unsigned)GPR_HASH_POINTER(&magic_thread_local, gpr_cpu_num_cores()); } #endif /* GPR_CPU_POSIX */ diff --git a/test/core/end2end/tests/network_status_change.c b/test/core/end2end/tests/network_status_change.c index 9cef02b2b3b..7540ce93a1b 100644 --- a/test/core/end2end/tests/network_status_change.c +++ b/test/core/end2end/tests/network_status_change.c @@ -212,8 +212,11 @@ static void test_invoke_network_status_change(grpc_end2end_test_config config) { CQ_EXPECT_COMPLETION(cqv, tag(1), 1); cq_verify(cqv); + // TODO(makdharma) Update this when the shutdown_all_endpoints is implemented. // Expected behavior of a RPC when network is lost. - GPR_ASSERT(status == GRPC_STATUS_UNAVAILABLE); + // GPR_ASSERT(status == GRPC_STATUS_UNAVAILABLE); + GPR_ASSERT(status == GRPC_STATUS_OK); + GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); validate_host_override_string("foo.test.google.fr:1234", call_details.host, config); From b5359b5fac0513ad60124259c37df64759edb541 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 16 Feb 2017 14:06:34 -0800 Subject: [PATCH 165/167] added assert against combining -p and -n --- tools/run_tests/run_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 54803de6804..1de20ae1b60 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1462,6 +1462,8 @@ def _build_and_run( if not isclose(args.sample_percent, 100.0): print("Running %d tests out of %d (~%d%%)" % (sample_size, num_jobs, args.sample_percent)) + else: + assert args.runs_per_test == 1, "Can't do sampling (-p) over multiple runs (-n)." if infinite_runs: assert len(massaged_one_run) > 0, 'Must have at least one test for a -n inf run' runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs From 37cbc3f5b30451254c99a85be36364bae74f2c70 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 16 Feb 2017 14:54:55 -0800 Subject: [PATCH 166/167] Use special errors where appropriate in call cancellation: avoids many allocations in a semi-common case --- src/core/lib/surface/call.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 3352e427cd2..48a1e586e17 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -481,7 +481,10 @@ void grpc_call_destroy(grpc_call *c) { c->destroy_called = 1; cancel = !c->received_final_op; gpr_mu_unlock(&c->mu); - if (cancel) grpc_call_cancel(c, NULL); + if (cancel) { + cancel_with_error(&exec_ctx, c, STATUS_FROM_API_OVERRIDE, + GRPC_ERROR_CANCELLED); + } GRPC_CALL_INTERNAL_UNREF(&exec_ctx, c, "destroy"); grpc_exec_ctx_finish(&exec_ctx); GPR_TIMER_END("grpc_call_destroy", 0); @@ -490,8 +493,11 @@ void grpc_call_destroy(grpc_call *c) { grpc_call_error grpc_call_cancel(grpc_call *call, void *reserved) { GRPC_API_TRACE("grpc_call_cancel(call=%p, reserved=%p)", 2, (call, reserved)); GPR_ASSERT(!reserved); - return grpc_call_cancel_with_status(call, GRPC_STATUS_CANCELLED, "Cancelled", - NULL); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + cancel_with_error(&exec_ctx, call, STATUS_FROM_API_OVERRIDE, + GRPC_ERROR_CANCELLED); + grpc_exec_ctx_finish(&exec_ctx); + return GRPC_CALL_OK; } static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call, From f07ed45439110033c8460295a71e4a5b4c42a0a1 Mon Sep 17 00:00:00 2001 From: yang-g Date: Thu, 16 Feb 2017 23:01:28 -0800 Subject: [PATCH 167/167] Restore the NextMessageSize for streamed calls --- include/grpc++/impl/codegen/call.h | 15 ++++++++++++++- include/grpc++/impl/codegen/sync_stream.h | 8 ++++---- src/cpp/server/server_cc.cc | 4 ++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index a17cdf9236f..19a5ca2b2ee 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -618,7 +618,17 @@ class Call final { public: /* call is owned by the caller */ Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq) - : call_hook_(call_hook), cq_(cq), call_(call) {} + : call_hook_(call_hook), + cq_(cq), + call_(call), + max_receive_message_size_(-1) {} + + Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq, + int max_receive_message_size) + : call_hook_(call_hook), + cq_(cq), + call_(call), + max_receive_message_size_(max_receive_message_size) {} void PerformOps(CallOpSetInterface* ops) { call_hook_->PerformOpsOnCall(ops, this); @@ -627,10 +637,13 @@ class Call final { grpc_call* call() const { return call_; } CompletionQueue* cq() const { return cq_; } + int max_receive_message_size() const { return max_receive_message_size_; } + private: CallHook* call_hook_; CompletionQueue* cq_; grpc_call* call_; + int max_receive_message_size_; }; } // namespace grpc diff --git a/include/grpc++/impl/codegen/sync_stream.h b/include/grpc++/impl/codegen/sync_stream.h index 1f7708bab94..4d9b074e95f 100644 --- a/include/grpc++/impl/codegen/sync_stream.h +++ b/include/grpc++/impl/codegen/sync_stream.h @@ -160,7 +160,7 @@ class ClientReader final : public ClientReaderInterface { } bool NextMessageSize(uint32_t* sz) override { - *sz = INT_MAX; + *sz = call_.max_receive_message_size(); return true; } @@ -310,7 +310,7 @@ class ClientReaderWriter final : public ClientReaderWriterInterface { } bool NextMessageSize(uint32_t* sz) override { - *sz = INT_MAX; + *sz = call_.max_receive_message_size(); return true; } @@ -382,7 +382,7 @@ class ServerReader final : public ServerReaderInterface { } bool NextMessageSize(uint32_t* sz) override { - *sz = INT_MAX; + *sz = call_->max_receive_message_size(); return true; } @@ -474,7 +474,7 @@ class ServerReaderWriterBody final { } bool NextMessageSize(uint32_t* sz) { - *sz = INT_MAX; + *sz = call_->max_receive_message_size(); return true; } diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index 29898a4209e..5ed9f41d4f4 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -186,7 +186,7 @@ class Server::SyncRequest final : public CompletionQueueTag { public: explicit CallData(Server* server, SyncRequest* mrd) : cq_(mrd->cq_), - call_(mrd->call_, server, &cq_), + call_(mrd->call_, server, &cq_, server->max_receive_message_size()), ctx_(mrd->deadline_, mrd->request_metadata_.metadata, mrd->request_metadata_.count), has_request_payload_(mrd->has_request_payload_), @@ -590,7 +590,7 @@ bool ServerInterface::BaseAsyncRequest::FinalizeResult(void** tag, } context_->set_call(call_); context_->cq_ = call_cq_; - Call call(call_, server_, call_cq_); + Call call(call_, server_, call_cq_, server_->max_receive_message_size()); if (*status && call_) { context_->BeginCompletionOp(&call); }