From 2f15e9b456d2490dabec55b1bd11a34cb1f4181d Mon Sep 17 00:00:00 2001 From: ncteisen Date: Thu, 16 Feb 2017 15:00:31 -0800 Subject: [PATCH 001/188] Add new bm_error test to exercise error_set_xxx --- test/cpp/microbenchmarks/bm_error.cc | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc index 8a4b86f281a..4a5af6884df 100644 --- a/test/cpp/microbenchmarks/bm_error.cc +++ b/test/cpp/microbenchmarks/bm_error.cc @@ -64,6 +64,36 @@ static void BM_ErrorCreateAndSetStatus(benchmark::State& state) { } BENCHMARK(BM_ErrorCreateAndSetStatus); +static void BM_ErrorCreateAndSetIntAndStr(benchmark::State& state) { + while (state.KeepRunning()) { + GRPC_ERROR_UNREF(grpc_error_set_str( + grpc_error_set_int(GRPC_ERROR_CREATE("GOAWAY received"), + GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)0), + GRPC_ERROR_STR_RAW_BYTES, "raw bytes")); + } +} +BENCHMARK(BM_ErrorCreateAndSetIntAndStr); + +static void BM_ErrorCreateAndSetIntLoop(benchmark::State& state) { + grpc_error* error = GRPC_ERROR_CREATE("Error"); + int n = 0; + while (state.KeepRunning()) { + error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, n++); + } + GRPC_ERROR_UNREF(error); +} +BENCHMARK(BM_ErrorCreateAndSetIntLoop); + +static void BM_ErrorCreateAndSetStrLoop(benchmark::State& state) { + grpc_error* error = GRPC_ERROR_CREATE("Error"); + const char* str = "hello"; + while (state.KeepRunning()) { + error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, str); + } + GRPC_ERROR_UNREF(error); +} +BENCHMARK(BM_ErrorCreateAndSetStrLoop); + static void BM_ErrorRefUnref(benchmark::State& state) { grpc_error* error = GRPC_ERROR_CREATE("Error"); while (state.KeepRunning()) { From 2c94aaeaba4abd75232ddd12f64f02c7740d6b88 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Thu, 16 Feb 2017 15:01:17 -0800 Subject: [PATCH 002/188] Add gpr_ref_is_unique --- include/grpc/support/sync.h | 4 ++++ src/core/lib/support/sync.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/include/grpc/support/sync.h b/include/grpc/support/sync.h index a7bbb38c27e..5cfeecb6017 100644 --- a/include/grpc/support/sync.h +++ b/include/grpc/support/sync.h @@ -164,6 +164,10 @@ GPRAPI void gpr_refn(gpr_refcount *r, int n); zero. . Requires *r initialized. */ GPRAPI int gpr_unref(gpr_refcount *r); +/* Return non-zero iff the reference count of *r is one, and thus is owned + by exactly one object. */ +GPRAPI int gpr_ref_is_unique(gpr_refcount *r); + /* --- Stats counters --- These calls act on the integral type gpr_stats_counter. It requires no diff --git a/src/core/lib/support/sync.c b/src/core/lib/support/sync.c index 44b83f8175a..9b5a4ac7087 100644 --- a/src/core/lib/support/sync.c +++ b/src/core/lib/support/sync.c @@ -113,6 +113,10 @@ int gpr_unref(gpr_refcount *r) { return prior == 1; } +int gpr_ref_is_unique(gpr_refcount *r) { + return gpr_atm_acq_load(&r->count) == 1; +} + void gpr_stats_init(gpr_stats_counter *c, intptr_t n) { gpr_atm_rel_store(&c->value, n); } From 311f7f1eaf2a11b09fa15fea3d26eab07fe23239 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Thu, 16 Feb 2017 15:01:55 -0800 Subject: [PATCH 003/188] Speed up copy_error_and_unref --- src/core/lib/iomgr/error.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index dbe5b139f91..6c4d0157c20 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -262,6 +262,8 @@ static grpc_error *copy_error_and_unref(grpc_error *in) { GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_CANCELLED); else out = GRPC_ERROR_CREATE("unknown"); + } else if (gpr_ref_is_unique(&in->refs)) { + return in; } else { out = gpr_malloc(sizeof(*out)); #ifdef GRPC_ERROR_REFCOUNT_DEBUG From 3da3ce3c29ab169faa7670f471eeaebcdbc19d6e Mon Sep 17 00:00:00 2001 From: ncteisen Date: Fri, 17 Feb 2017 09:51:25 -0800 Subject: [PATCH 004/188] Add error test --- CMakeLists.txt | 28 +++ Makefile | 36 ++++ build.yaml | 11 + grpc.def | 1 + src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 + src/ruby/ext/grpc/rb_grpc_imports.generated.h | 3 + test/core/iomgr/error_test.c | 140 ++++++++++++ .../generated/sources_and_headers.json | 17 ++ tools/run_tests/generated/tests.json | 22 ++ vsprojects/buildtests_c.sln | 27 +++ .../test/error_test/error_test.vcxproj | 199 ++++++++++++++++++ .../error_test/error_test.vcxproj.filters | 21 ++ 12 files changed, 507 insertions(+) create mode 100644 test/core/iomgr/error_test.c create mode 100644 vsprojects/vcxproj/test/error_test/error_test.vcxproj create mode 100644 vsprojects/vcxproj/test/error_test/error_test.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index f2a2380f2e4..6000e1021d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -353,6 +353,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c dualstack_socket_test) endif() add_dependencies(buildtests_c endpoint_pair_test) +add_dependencies(buildtests_c error_test) if(_gRPC_PLATFORM_LINUX) add_dependencies(buildtests_c ev_epoll_linux_test) endif() @@ -4548,6 +4549,33 @@ target_link_libraries(endpoint_pair_test gpr ) +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + +add_executable(error_test + test/core/iomgr/error_test.c +) + + +target_include_directories(error_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include +) + +target_link_libraries(error_test + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc + gpr_test_util + gpr +) + endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX) diff --git a/Makefile b/Makefile index 0d8b9608f2e..e23ab970e3b 100644 --- a/Makefile +++ b/Makefile @@ -926,6 +926,7 @@ dns_resolver_connectivity_test: $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_te dns_resolver_test: $(BINDIR)/$(CONFIG)/dns_resolver_test dualstack_socket_test: $(BINDIR)/$(CONFIG)/dualstack_socket_test endpoint_pair_test: $(BINDIR)/$(CONFIG)/endpoint_pair_test +error_test: $(BINDIR)/$(CONFIG)/error_test ev_epoll_linux_test: $(BINDIR)/$(CONFIG)/ev_epoll_linux_test fd_conservation_posix_test: $(BINDIR)/$(CONFIG)/fd_conservation_posix_test fd_posix_test: $(BINDIR)/$(CONFIG)/fd_posix_test @@ -1291,6 +1292,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/dns_resolver_test \ $(BINDIR)/$(CONFIG)/dualstack_socket_test \ $(BINDIR)/$(CONFIG)/endpoint_pair_test \ + $(BINDIR)/$(CONFIG)/error_test \ $(BINDIR)/$(CONFIG)/ev_epoll_linux_test \ $(BINDIR)/$(CONFIG)/fd_conservation_posix_test \ $(BINDIR)/$(CONFIG)/fd_posix_test \ @@ -1669,6 +1671,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/dualstack_socket_test || ( echo test dualstack_socket_test failed ; exit 1 ) $(E) "[RUN] Testing endpoint_pair_test" $(Q) $(BINDIR)/$(CONFIG)/endpoint_pair_test || ( echo test endpoint_pair_test failed ; exit 1 ) + $(E) "[RUN] Testing error_test" + $(Q) $(BINDIR)/$(CONFIG)/error_test || ( echo test error_test failed ; exit 1 ) $(E) "[RUN] Testing ev_epoll_linux_test" $(Q) $(BINDIR)/$(CONFIG)/ev_epoll_linux_test || ( echo test ev_epoll_linux_test failed ; exit 1 ) $(E) "[RUN] Testing fd_conservation_posix_test" @@ -8621,6 +8625,38 @@ endif endif +ERROR_TEST_SRC = \ + test/core/iomgr/error_test.c \ + +ERROR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(ERROR_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/error_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/error_test: $(ERROR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(ERROR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/error_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/iomgr/error_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_error_test: $(ERROR_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(ERROR_TEST_OBJS:.o=.dep) +endif +endif + + EV_EPOLL_LINUX_TEST_SRC = \ test/core/iomgr/ev_epoll_linux_test.c \ diff --git a/build.yaml b/build.yaml index 33cd692b455..7d33738c300 100644 --- a/build.yaml +++ b/build.yaml @@ -1686,6 +1686,17 @@ targets: - gpr exclude_iomgrs: - uv +- name: error_test + cpu_cost: 30 + build: test + language: c + src: + - test/core/iomgr/error_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: ev_epoll_linux_test build: test language: c diff --git a/grpc.def b/grpc.def index e387c8b7125..dba75882e20 100644 --- a/grpc.def +++ b/grpc.def @@ -257,6 +257,7 @@ EXPORTS gpr_ref_non_zero gpr_refn gpr_unref + gpr_ref_is_unique gpr_stats_init gpr_stats_inc gpr_stats_read diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index b8af8008336..4fe9f5b0b82 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -295,6 +295,7 @@ gpr_ref_type gpr_ref_import; gpr_ref_non_zero_type gpr_ref_non_zero_import; gpr_refn_type gpr_refn_import; gpr_unref_type gpr_unref_import; +gpr_ref_is_unique_type gpr_ref_is_unique_import; gpr_stats_init_type gpr_stats_init_import; gpr_stats_inc_type gpr_stats_inc_import; gpr_stats_read_type gpr_stats_read_import; @@ -587,6 +588,7 @@ void grpc_rb_load_imports(HMODULE library) { gpr_ref_non_zero_import = (gpr_ref_non_zero_type) GetProcAddress(library, "gpr_ref_non_zero"); gpr_refn_import = (gpr_refn_type) GetProcAddress(library, "gpr_refn"); gpr_unref_import = (gpr_unref_type) GetProcAddress(library, "gpr_unref"); + gpr_ref_is_unique_import = (gpr_ref_is_unique_type) GetProcAddress(library, "gpr_ref_is_unique"); gpr_stats_init_import = (gpr_stats_init_type) GetProcAddress(library, "gpr_stats_init"); gpr_stats_inc_import = (gpr_stats_inc_type) GetProcAddress(library, "gpr_stats_inc"); gpr_stats_read_import = (gpr_stats_read_type) GetProcAddress(library, "gpr_stats_read"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 3229e2a77dc..01d653fba68 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -836,6 +836,9 @@ extern gpr_refn_type gpr_refn_import; typedef int(*gpr_unref_type)(gpr_refcount *r); extern gpr_unref_type gpr_unref_import; #define gpr_unref gpr_unref_import +typedef int(*gpr_ref_is_unique_type)(gpr_refcount *r); +extern gpr_ref_is_unique_type gpr_ref_is_unique_import; +#define gpr_ref_is_unique gpr_ref_is_unique_import typedef void(*gpr_stats_init_type)(gpr_stats_counter *c, intptr_t n); extern gpr_stats_init_type gpr_stats_init_import; #define gpr_stats_init gpr_stats_init_import diff --git a/test/core/iomgr/error_test.c b/test/core/iomgr/error_test.c new file mode 100644 index 00000000000..5137ef9758f --- /dev/null +++ b/test/core/iomgr/error_test.c @@ -0,0 +1,140 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/error.h" + +#include +#include +#include +#include +#include + +#include + +#include "test/core/util/test_config.h" + +static void test_set_get_int() +{ + grpc_error* error = GRPC_ERROR_CREATE("Test"); + intptr_t i = 0; + GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_FILE_LINE, &i)); + GPR_ASSERT(i); // line set will never be 0 + GPR_ASSERT(!grpc_error_get_int(error, GRPC_ERROR_INT_ERRNO, &i)); + GPR_ASSERT(!grpc_error_get_int(error, GRPC_ERROR_INT_SIZE, &i)); + + intptr_t errno = 314; + error = grpc_error_set_int(error, GRPC_ERROR_INT_ERRNO, errno); + GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_ERRNO, &i)); + GPR_ASSERT(i == errno); + + intptr_t line = 555; + error = grpc_error_set_int(error, GRPC_ERROR_INT_FILE_LINE, line); + GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_FILE_LINE, &i)); + GPR_ASSERT(i == line); +} + +static void test_set_get_str() +{ + grpc_error* error = GRPC_ERROR_CREATE("Test"); + GPR_ASSERT(!grpc_error_get_str(error, GRPC_ERROR_STR_SYSCALL)); + GPR_ASSERT(!grpc_error_get_str(error, GRPC_ERROR_STR_TSI_ERROR)); + + const char* c = grpc_error_get_str(error, GRPC_ERROR_STR_FILE); + GPR_ASSERT(c); + GPR_ASSERT(!strcmp(c, "test/core/iomgr/error_test.c")); + + c = grpc_error_get_str(error, GRPC_ERROR_STR_DESCRIPTION); + GPR_ASSERT(c); + GPR_ASSERT(!strcmp(c, "Test")); + + error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "message"); + c = grpc_error_get_str(error, GRPC_ERROR_STR_GRPC_MESSAGE); + GPR_ASSERT(c); + GPR_ASSERT(!strcmp(c, "message")); + + error = grpc_error_set_str(error, GRPC_ERROR_STR_DESCRIPTION, "new desc"); + c = grpc_error_get_str(error, GRPC_ERROR_STR_DESCRIPTION); + GPR_ASSERT(c); + GPR_ASSERT(!strcmp(c, "new desc")); +} + +static void test_copy_and_unref() +{ + grpc_error* error1 = GRPC_ERROR_CREATE("Test"); + grpc_error* error2 = grpc_error_set_str(error1, GRPC_ERROR_STR_GRPC_MESSAGE, "message"); + const char* c = grpc_error_get_str(error1, GRPC_ERROR_STR_GRPC_MESSAGE); + GPR_ASSERT(c); + GPR_ASSERT(!strcmp(c, "message")); + c = grpc_error_get_str(error2, GRPC_ERROR_STR_GRPC_MESSAGE); + GPR_ASSERT(c); + GPR_ASSERT(!strcmp(c, "message")); + GPR_ASSERT(error1 == error2); + + GRPC_ERROR_REF(error1); + grpc_error* error3 = grpc_error_set_str(error1, GRPC_ERROR_STR_DESCRIPTION, "reset"); + GPR_ASSERT(error3 != error1); // should not be the same because of extra ref + c = grpc_error_get_str(error3, GRPC_ERROR_STR_GRPC_MESSAGE); + GPR_ASSERT(c); + GPR_ASSERT(!strcmp(c, "message")); + + c = grpc_error_get_str(error1, GRPC_ERROR_STR_DESCRIPTION); + GPR_ASSERT(c); + GPR_ASSERT(!strcmp(c, "Test")); + + c = grpc_error_get_str(error3, GRPC_ERROR_STR_DESCRIPTION); + GPR_ASSERT(c); + GPR_ASSERT(!strcmp(c, "reset")); +} + +static void print_error_strings() +{ + grpc_error* error = grpc_error_set_int(GRPC_ERROR_CREATE("Error"), + GRPC_ERROR_INT_GRPC_STATUS, + GRPC_STATUS_UNIMPLEMENTED); + error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, 0); + error = grpc_error_set_int(error, GRPC_ERROR_INT_SIZE, 666); + error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "message"); + gpr_log(GPR_DEBUG, "%s", grpc_error_string(error)); +} + +int main(int argc, char **argv) { + grpc_test_init(argc, argv); + grpc_init(); + test_set_get_int(); + test_set_get_str(); + test_copy_and_unref(); + print_error_strings(); + grpc_shutdown(); + + return 0; +} diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index b04e4bd06fa..9256d72db79 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -421,6 +421,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "error_test", + "src": [ + "test/core/iomgr/error_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index d93b55a0233..a7d6b6a8c68 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -515,6 +515,28 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 30, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "error_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index 12a45ee1f85..1f71f00b44b 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -297,6 +297,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "endpoint_pair_test", "vcxpr {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "error_test", "vcxproj\test\error_test\error_test.vcxproj", "{42720233-A6D4-66BC-CCA2-06B57261D0B3}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fling_client", "vcxproj\test\fling_client\fling_client.vcxproj", "{0647D598-9611-F659-EA36-DF995C9F736B}" ProjectSection(myProperties) = preProject lib = "False" @@ -2001,6 +2012,22 @@ Global {37166D50-3AAA-1156-19F6-5901DFA55172}.Release-DLL|Win32.Build.0 = Release|Win32 {37166D50-3AAA-1156-19F6-5901DFA55172}.Release-DLL|x64.ActiveCfg = Release|x64 {37166D50-3AAA-1156-19F6-5901DFA55172}.Release-DLL|x64.Build.0 = Release|x64 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Debug|Win32.ActiveCfg = Debug|Win32 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Debug|x64.ActiveCfg = Debug|x64 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Release|Win32.ActiveCfg = Release|Win32 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Release|x64.ActiveCfg = Release|x64 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Debug|Win32.Build.0 = Debug|Win32 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Debug|x64.Build.0 = Debug|x64 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Release|Win32.Build.0 = Release|Win32 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Release|x64.Build.0 = Release|x64 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Debug-DLL|x64.Build.0 = Debug|x64 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Release-DLL|Win32.Build.0 = Release|Win32 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Release-DLL|x64.ActiveCfg = Release|x64 + {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Release-DLL|x64.Build.0 = Release|x64 {0647D598-9611-F659-EA36-DF995C9F736B}.Debug|Win32.ActiveCfg = Debug|Win32 {0647D598-9611-F659-EA36-DF995C9F736B}.Debug|x64.ActiveCfg = Debug|x64 {0647D598-9611-F659-EA36-DF995C9F736B}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/test/error_test/error_test.vcxproj b/vsprojects/vcxproj/test/error_test/error_test.vcxproj new file mode 100644 index 00000000000..c7d0e5a1000 --- /dev/null +++ b/vsprojects/vcxproj/test/error_test/error_test.vcxproj @@ -0,0 +1,199 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {42720233-A6D4-66BC-CCA2-06B57261D0B3} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + error_test + static + Debug + static + Debug + + + error_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/error_test/error_test.vcxproj.filters b/vsprojects/vcxproj/test/error_test/error_test.vcxproj.filters new file mode 100644 index 00000000000..d2bb372482f --- /dev/null +++ b/vsprojects/vcxproj/test/error_test/error_test.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\core\iomgr + + + + + + {3226d467-9fe9-12e9-8b53-f24f5a4c34c0} + + + {3d86bf5c-b8eb-36f4-efe2-7e8596188481} + + + {91f383bc-92fa-7058-d971-20189eadc0ad} + + + + From 77532e8bf3d4c93b680ac63a5e436cea92c708f5 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 27 Feb 2017 11:52:39 -0800 Subject: [PATCH 005/188] Destroy pointer args when destructing a ChannelArguments --- include/grpc++/support/channel_arguments.h | 2 +- src/cpp/common/channel_arguments.cc | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/include/grpc++/support/channel_arguments.h b/include/grpc++/support/channel_arguments.h index efdf7772ad7..80a3bfda3d9 100644 --- a/include/grpc++/support/channel_arguments.h +++ b/include/grpc++/support/channel_arguments.h @@ -54,7 +54,7 @@ class ResourceQuota; class ChannelArguments { public: ChannelArguments(); - ~ChannelArguments() {} + ~ChannelArguments(); ChannelArguments(const ChannelArguments& other); ChannelArguments& operator=(ChannelArguments other) { diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc index 65f32774999..34fd4491d0a 100644 --- a/src/cpp/common/channel_arguments.cc +++ b/src/cpp/common/channel_arguments.cc @@ -81,6 +81,16 @@ ChannelArguments::ChannelArguments(const ChannelArguments& other) } } +ChannelArguments::~ChannelArguments() { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + for (auto it = args_.begin(); it != args_.end(); ++it) { + if (it->type == GRPC_ARG_POINTER) { + it->value.pointer.vtable->destroy(&exec_ctx, it->value.pointer.p); + } + } + grpc_exec_ctx_finish(&exec_ctx); +} + void ChannelArguments::Swap(ChannelArguments& other) { args_.swap(other.args_); strings_.swap(other.strings_); From c5b3b25f8d94404a6069cacd756be47d8e9d8de3 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 22 Feb 2017 11:25:10 -0800 Subject: [PATCH 006/188] Change ssl_server_fuzzer.c to use ssl_test_data.h instead of loading mock SSL data from file --- test/core/security/BUILD | 2 +- test/core/security/ssl_server_fuzzer.c | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/test/core/security/BUILD b/test/core/security/BUILD index e750c39b7c6..1cb03c5cfe7 100644 --- a/test/core/security/BUILD +++ b/test/core/security/BUILD @@ -34,7 +34,7 @@ load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer") grpc_fuzzer( name = "ssl_server_fuzzer", srcs = ["ssl_server_fuzzer.c"], - deps = ["//:gpr", "//:grpc", "//test/core/util:grpc_test_util"], + deps = ["//:gpr", "//:grpc", "//test/core/util:grpc_test_util", "//test/core/end2end:ssl_test_data"], corpus = "corpus", copts = ["-std=c99"], ) diff --git a/test/core/security/ssl_server_fuzzer.c b/test/core/security/ssl_server_fuzzer.c index f789278add8..fe98799144f 100644 --- a/test/core/security/ssl_server_fuzzer.c +++ b/test/core/security/ssl_server_fuzzer.c @@ -38,6 +38,7 @@ #include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/transport/security_connector.h" +#include "test/core/end2end/data/ssl_test_data.h" #include "test/core/util/memory_counters.h" #include "test/core/util/mock_endpoint.h" @@ -50,6 +51,7 @@ bool leak_check = false; #define SSL_KEY_PATH "src/core/lib/tsi/test_creds/server1.key" #define SSL_CA_PATH "src/core/lib/tsi/test_creds/ca.pem" + static void discard_write(grpc_slice slice) {} static void dont_log(gpr_log_func_args *args) {} @@ -88,12 +90,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { // Load key pair and establish server SSL credentials. grpc_ssl_pem_key_cert_pair pem_key_cert_pair; grpc_slice ca_slice, cert_slice, key_slice; - GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", - grpc_load_file(SSL_CA_PATH, 1, &ca_slice))); - GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", - grpc_load_file(SSL_CERT_PATH, 1, &cert_slice))); - GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", - grpc_load_file(SSL_KEY_PATH, 1, &key_slice))); + ca_slice = grpc_slice_from_static_string(test_root_cert); + cert_slice = grpc_slice_from_static_string(test_server1_cert); + key_slice = grpc_slice_from_static_string(test_server1_key); const char *ca_cert = (const char *)GRPC_SLICE_START_PTR(ca_slice); pem_key_cert_pair.private_key = (const char *)GRPC_SLICE_START_PTR(key_slice); pem_key_cert_pair.cert_chain = (const char *)GRPC_SLICE_START_PTR(cert_slice); From d9b257a1547f8d3d6ddf66036e19561e64de6531 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 22 Feb 2017 15:14:28 -0800 Subject: [PATCH 007/188] Add fuzzer options for oss-fuzz --- test/core/security/ssl_server_fuzzer.c | 5 ----- tools/fuzzer/options/api_fuzzer.options | 3 +++ tools/fuzzer/options/client_fuzzer.options | 3 +++ tools/fuzzer/options/fuzzer.options | 2 ++ tools/fuzzer/options/fuzzer_response.options | 2 ++ tools/fuzzer/options/fuzzer_serverlist.options | 2 ++ tools/fuzzer/options/hpack_parser_fuzzer_test.options | 3 +++ tools/fuzzer/options/percent_decode_fuzzer.options | 2 ++ tools/fuzzer/options/percent_encode_fuzzer.options | 2 ++ tools/fuzzer/options/request_fuzzer.options | 3 +++ tools/fuzzer/options/response_fuzzer.options | 3 +++ tools/fuzzer/options/server_fuzzer.options | 3 +++ tools/fuzzer/options/ssl_server_fuzzer.options | 2 ++ tools/fuzzer/options/uri_fuzzer_test.options | 2 ++ 14 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 tools/fuzzer/options/api_fuzzer.options create mode 100644 tools/fuzzer/options/client_fuzzer.options create mode 100644 tools/fuzzer/options/fuzzer.options create mode 100644 tools/fuzzer/options/fuzzer_response.options create mode 100644 tools/fuzzer/options/fuzzer_serverlist.options create mode 100644 tools/fuzzer/options/hpack_parser_fuzzer_test.options create mode 100644 tools/fuzzer/options/percent_decode_fuzzer.options create mode 100644 tools/fuzzer/options/percent_encode_fuzzer.options create mode 100644 tools/fuzzer/options/request_fuzzer.options create mode 100644 tools/fuzzer/options/response_fuzzer.options create mode 100644 tools/fuzzer/options/server_fuzzer.options create mode 100644 tools/fuzzer/options/ssl_server_fuzzer.options create mode 100644 tools/fuzzer/options/uri_fuzzer_test.options diff --git a/test/core/security/ssl_server_fuzzer.c b/test/core/security/ssl_server_fuzzer.c index fe98799144f..7a3612c419b 100644 --- a/test/core/security/ssl_server_fuzzer.c +++ b/test/core/security/ssl_server_fuzzer.c @@ -47,11 +47,6 @@ bool squelch = true; // Turning this on will fail the leak check. bool leak_check = false; -#define SSL_CERT_PATH "src/core/lib/tsi/test_creds/server1.pem" -#define SSL_KEY_PATH "src/core/lib/tsi/test_creds/server1.key" -#define SSL_CA_PATH "src/core/lib/tsi/test_creds/ca.pem" - - static void discard_write(grpc_slice slice) {} static void dont_log(gpr_log_func_args *args) {} diff --git a/tools/fuzzer/options/api_fuzzer.options b/tools/fuzzer/options/api_fuzzer.options new file mode 100644 index 00000000000..8871ae21b6a --- /dev/null +++ b/tools/fuzzer/options/api_fuzzer.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 2048 +dict = api_fuzzer.dictionary diff --git a/tools/fuzzer/options/client_fuzzer.options b/tools/fuzzer/options/client_fuzzer.options new file mode 100644 index 00000000000..fd2eebf7d25 --- /dev/null +++ b/tools/fuzzer/options/client_fuzzer.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 2048 +dict = hpack.dictionary diff --git a/tools/fuzzer/options/fuzzer.options b/tools/fuzzer/options/fuzzer.options new file mode 100644 index 00000000000..5d468bc6e48 --- /dev/null +++ b/tools/fuzzer/options/fuzzer.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 512 diff --git a/tools/fuzzer/options/fuzzer_response.options b/tools/fuzzer/options/fuzzer_response.options new file mode 100644 index 00000000000..5dcdfac7a69 --- /dev/null +++ b/tools/fuzzer/options/fuzzer_response.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 128 diff --git a/tools/fuzzer/options/fuzzer_serverlist.options b/tools/fuzzer/options/fuzzer_serverlist.options new file mode 100644 index 00000000000..5dcdfac7a69 --- /dev/null +++ b/tools/fuzzer/options/fuzzer_serverlist.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 128 diff --git a/tools/fuzzer/options/hpack_parser_fuzzer_test.options b/tools/fuzzer/options/hpack_parser_fuzzer_test.options new file mode 100644 index 00000000000..584487fafc2 --- /dev/null +++ b/tools/fuzzer/options/hpack_parser_fuzzer_test.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 512 +dict = hpack.dictionary diff --git a/tools/fuzzer/options/percent_decode_fuzzer.options b/tools/fuzzer/options/percent_decode_fuzzer.options new file mode 100644 index 00000000000..ea2785e1104 --- /dev/null +++ b/tools/fuzzer/options/percent_decode_fuzzer.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 32 diff --git a/tools/fuzzer/options/percent_encode_fuzzer.options b/tools/fuzzer/options/percent_encode_fuzzer.options new file mode 100644 index 00000000000..ea2785e1104 --- /dev/null +++ b/tools/fuzzer/options/percent_encode_fuzzer.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 32 diff --git a/tools/fuzzer/options/request_fuzzer.options b/tools/fuzzer/options/request_fuzzer.options new file mode 100644 index 00000000000..fd32ac16e1b --- /dev/null +++ b/tools/fuzzer/options/request_fuzzer.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 2048 + diff --git a/tools/fuzzer/options/response_fuzzer.options b/tools/fuzzer/options/response_fuzzer.options new file mode 100644 index 00000000000..fd32ac16e1b --- /dev/null +++ b/tools/fuzzer/options/response_fuzzer.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 2048 + diff --git a/tools/fuzzer/options/server_fuzzer.options b/tools/fuzzer/options/server_fuzzer.options new file mode 100644 index 00000000000..fd2eebf7d25 --- /dev/null +++ b/tools/fuzzer/options/server_fuzzer.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 2048 +dict = hpack.dictionary diff --git a/tools/fuzzer/options/ssl_server_fuzzer.options b/tools/fuzzer/options/ssl_server_fuzzer.options new file mode 100644 index 00000000000..60bd9b0b2fa --- /dev/null +++ b/tools/fuzzer/options/ssl_server_fuzzer.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 2048 diff --git a/tools/fuzzer/options/uri_fuzzer_test.options b/tools/fuzzer/options/uri_fuzzer_test.options new file mode 100644 index 00000000000..5dcdfac7a69 --- /dev/null +++ b/tools/fuzzer/options/uri_fuzzer_test.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 128 From 7acc37e502431b054a1e6bb823fedbca267f983a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Feb 2017 10:01:37 -0800 Subject: [PATCH 008/188] Trim deadline setting out of call creation path: reduces ClientChannelFilter init/destroy from 128ns to 46ns on my machine --- src/core/ext/client_channel/client_channel.c | 81 +++----------------- src/core/lib/channel/deadline_filter.c | 21 +---- src/core/lib/channel/deadline_filter.h | 10 --- test/core/end2end/tests/negative_deadline.c | 10 +-- 4 files changed, 15 insertions(+), 107 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 34015c534e1..341dc28000e 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -71,7 +71,8 @@ */ typedef enum { - WAIT_FOR_READY_UNSET, + /* zero so it can be default initialized */ + WAIT_FOR_READY_UNSET = 0, WAIT_FOR_READY_FALSE, WAIT_FOR_READY_TRUE } wait_for_ready_value; @@ -608,7 +609,8 @@ static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx, #define CANCELLED_CALL ((grpc_subchannel_call *)1) typedef enum { - GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING, + /* zero so that it can be default-initialized */ + GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING = 0, GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL } subchannel_creation_phase; @@ -1066,8 +1068,8 @@ static void read_service_config_locked(grpc_exec_ctx *exec_ctx, void *arg, gpr_time_add(calld->call_start_time, method_params->timeout); if (gpr_time_cmp(per_method_deadline, calld->deadline) < 0) { calld->deadline = per_method_deadline; - // Reset deadline timer. - grpc_deadline_state_reset(exec_ctx, elem, calld->deadline); + // Start deadline timer. + grpc_deadline_state_start(exec_ctx, elem, calld->deadline); } } if (method_params->wait_for_ready != WAIT_FOR_READY_UNSET) { @@ -1082,85 +1084,20 @@ static void read_service_config_locked(grpc_exec_ctx *exec_ctx, void *arg, GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "read_service_config"); } -static void initial_read_service_config_locked(grpc_exec_ctx *exec_ctx, - void *arg, - grpc_error *error_ignored) { - grpc_call_element *elem = arg; - channel_data *chand = elem->channel_data; - call_data *calld = elem->call_data; - // If the resolver has already returned results, then we can access - // the service config parameters immediately. Otherwise, we need to - // defer that work until the resolver returns an initial result. - // TODO(roth): This code is almost but not quite identical to the code - // in read_service_config() above. It would be nice to find a way to - // combine them, to avoid having to maintain it twice. - if (chand->lb_policy != NULL) { - // We already have a resolver result, so check for service config. - if (chand->method_params_table != NULL) { - grpc_slice_hash_table *method_params_table = - grpc_slice_hash_table_ref(chand->method_params_table); - method_parameters *method_params = grpc_method_config_table_get( - exec_ctx, method_params_table, calld->path); - if (method_params != NULL) { - if (gpr_time_cmp(method_params->timeout, - gpr_time_0(GPR_CLOCK_MONOTONIC)) != 0) { - gpr_timespec per_method_deadline = - gpr_time_add(calld->call_start_time, method_params->timeout); - calld->deadline = gpr_time_min(calld->deadline, per_method_deadline); - } - if (method_params->wait_for_ready != WAIT_FOR_READY_UNSET) { - calld->wait_for_ready_from_service_config = - method_params->wait_for_ready; - } - } - grpc_slice_hash_table_unref(exec_ctx, method_params_table); - } - } else { - // We don't yet have a resolver result, so register a callback to - // get the service config data once the resolver returns. - // Take a reference to the call stack to be owned by the callback. - GRPC_CALL_STACK_REF(calld->owning_call, "read_service_config"); - grpc_closure_init(&calld->read_service_config, read_service_config_locked, - elem, grpc_combiner_scheduler(chand->combiner, false)); - grpc_closure_list_append(&chand->waiting_for_config_closures, - &calld->read_service_config, GRPC_ERROR_NONE); - } - // Start the deadline timer with the current deadline value. If we - // do not yet have service config data, then the timer may be reset - // later. - grpc_deadline_state_start(exec_ctx, elem, calld->deadline); - GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, - "initial_read_service_config"); -} - /* Constructor for call_data */ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_element_args *args) { - channel_data *chand = elem->channel_data; call_data *calld = elem->call_data; + channel_data *chand = elem->channel_data; // Initialize data members. grpc_deadline_state_init(exec_ctx, elem, args->call_stack); calld->path = grpc_slice_ref_internal(args->path); calld->call_start_time = args->start_time; calld->deadline = gpr_convert_clock_type(args->deadline, GPR_CLOCK_MONOTONIC); - calld->wait_for_ready_from_service_config = WAIT_FOR_READY_UNSET; - calld->cancel_error = GRPC_ERROR_NONE; - gpr_atm_rel_store(&calld->subchannel_call, 0); - calld->connected_subchannel = NULL; - calld->waiting_ops = NULL; - calld->waiting_ops_count = 0; - calld->waiting_ops_capacity = 0; - calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; calld->owning_call = args->call_stack; - calld->pollent = NULL; - GRPC_CALL_STACK_REF(calld->owning_call, "initial_read_service_config"); - grpc_closure_sched( - exec_ctx, - grpc_closure_init(&calld->read_service_config, - initial_read_service_config_locked, elem, - grpc_combiner_scheduler(chand->combiner, false)), - GRPC_ERROR_NONE); + grpc_closure_init(&calld->read_service_config, read_service_config_locked, + elem, grpc_combiner_scheduler(chand->combiner, false)); return GRPC_ERROR_NONE; } diff --git a/src/core/lib/channel/deadline_filter.c b/src/core/lib/channel/deadline_filter.c index 5a12d62f1d1..0e9443937cd 100644 --- a/src/core/lib/channel/deadline_filter.c +++ b/src/core/lib/channel/deadline_filter.c @@ -78,21 +78,9 @@ retry: (grpc_deadline_timer_state)gpr_atm_acq_load(&deadline_state->timer_state); switch (cur_state) { case GRPC_DEADLINE_STATE_PENDING: + case GRPC_DEADLINE_STATE_FINISHED: // Note: We do not start the timer if there is already a timer return; - case GRPC_DEADLINE_STATE_FINISHED: - if (gpr_atm_rel_cas(&deadline_state->timer_state, - GRPC_DEADLINE_STATE_FINISHED, - GRPC_DEADLINE_STATE_PENDING)) { - // If we've already created and destroyed a timer, we always create a - // new closure: we have no other guarantee that the inlined closure is - // not in use (it may hold a pending call to timer_callback) - closure = grpc_closure_create(timer_callback, elem, - grpc_schedule_on_exec_ctx); - } else { - goto retry; - } - break; case GRPC_DEADLINE_STATE_INITIAL: if (gpr_atm_rel_cas(&deadline_state->timer_state, GRPC_DEADLINE_STATE_INITIAL, @@ -189,13 +177,6 @@ void grpc_deadline_state_start(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, } } -void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - gpr_timespec new_deadline) { - grpc_deadline_state* deadline_state = elem->call_data; - cancel_timer_if_needed(exec_ctx, deadline_state); - start_timer_if_needed(exec_ctx, elem, new_deadline); -} - void grpc_deadline_state_client_start_transport_stream_op( grpc_exec_ctx* exec_ctx, grpc_call_element* elem, grpc_transport_stream_op* op) { diff --git a/src/core/lib/channel/deadline_filter.h b/src/core/lib/channel/deadline_filter.h index 72cd5cb9296..363ac58b6d5 100644 --- a/src/core/lib/channel/deadline_filter.h +++ b/src/core/lib/channel/deadline_filter.h @@ -73,16 +73,6 @@ void grpc_deadline_state_destroy(grpc_exec_ctx* exec_ctx, void grpc_deadline_state_start(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, gpr_timespec deadline); -// Cancels the existing timer and starts a new one with new_deadline. -// -// Note: It is generally safe to call this with an earlier deadline -// value than the current one, but not the reverse. No checks are done -// to ensure that the timer callback is not invoked while it is in the -// process of being reset, which means that attempting to increase the -// deadline may result in the timer being called twice. -void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - gpr_timespec new_deadline); - // To be called from the client-side filter's start_transport_stream_op() // method. Ensures that the deadline timer is cancelled when the call // is completed. diff --git a/test/core/end2end/tests/negative_deadline.c b/test/core/end2end/tests/negative_deadline.c index 0b61efbac97..f3607290360 100644 --- a/test/core/end2end/tests/negative_deadline.c +++ b/test/core/end2end/tests/negative_deadline.c @@ -124,6 +124,11 @@ static void simple_request_body(grpc_end2end_test_config config, memset(ops, 0, sizeof(ops)); op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = 0; + op->reserved = NULL; + op++; op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; op->data.recv_status_on_client.status = &status; @@ -136,11 +141,6 @@ static void simple_request_body(grpc_end2end_test_config config, op->flags = 0; op->reserved = NULL; op++; - op->op = GRPC_OP_SEND_INITIAL_METADATA; - op->data.send_initial_metadata.count = 0; - op->flags = 0; - op->reserved = NULL; - op++; op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; op->flags = 0; op->reserved = NULL; From a7ed26885c61d37ac16267b640fb519806105549 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Feb 2017 13:26:37 -0800 Subject: [PATCH 009/188] Fix some bugs --- src/core/ext/client_channel/client_channel.c | 90 +++++++++----------- test/core/end2end/tests/negative_deadline.c | 2 +- 2 files changed, 42 insertions(+), 50 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 341dc28000e..e437340e143 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -632,7 +632,6 @@ typedef struct client_channel_call_data { gpr_timespec call_start_time; gpr_timespec deadline; wait_for_ready_value wait_for_ready_from_service_config; - grpc_closure read_service_config; grpc_error *cancel_error; @@ -705,6 +704,45 @@ static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, call_data *calld) { gpr_free(ops); } +static void apply_final_configuration_locked(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem) { + /* apply service-config level configuration to the call (now that we're + * certain it exists) */ + channel_data *chand = elem->channel_data; + call_data *calld = elem->call_data; + /* Get the method config table from channel data. */ + grpc_slice_hash_table *method_params_table = NULL; + if (chand->method_params_table != NULL) { + method_params_table = grpc_slice_hash_table_ref(chand->method_params_table); + } + /* If the method config table was present, use it. */ + if (method_params_table != NULL) { + const method_parameters *method_params = grpc_method_config_table_get( + exec_ctx, method_params_table, calld->path); + if (method_params != NULL) { + const bool have_method_timeout = + gpr_time_cmp(method_params->timeout, gpr_time_0(GPR_TIMESPAN)) != 0; + if (have_method_timeout || + method_params->wait_for_ready != WAIT_FOR_READY_UNSET) { + if (have_method_timeout) { + const gpr_timespec per_method_deadline = + gpr_time_add(calld->call_start_time, method_params->timeout); + if (gpr_time_cmp(per_method_deadline, calld->deadline) < 0) { + calld->deadline = per_method_deadline; + } + } + if (method_params->wait_for_ready != WAIT_FOR_READY_UNSET) { + calld->wait_for_ready_from_service_config = + method_params->wait_for_ready; + } + } + } + grpc_slice_hash_table_unref(exec_ctx, method_params_table); + } + /* Start deadline timer. */ + grpc_deadline_state_start(exec_ctx, elem, calld->deadline); +} + static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_call_element *elem = arg; @@ -733,6 +771,7 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, } else { /* Create call on subchannel. */ grpc_subchannel_call *subchannel_call = NULL; + apply_final_configuration_locked(exec_ctx, elem); grpc_error *new_error = grpc_connected_subchannel_create_call( exec_ctx, calld->connected_subchannel, calld->pollent, calld->path, calld->call_start_time, calld->deadline, &subchannel_call); @@ -960,6 +999,7 @@ static void start_transport_stream_op_locked_inner(grpc_exec_ctx *exec_ctx, if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && calld->connected_subchannel != NULL) { grpc_subchannel_call *subchannel_call = NULL; + apply_final_configuration_locked(exec_ctx, elem); grpc_error *error = grpc_connected_subchannel_create_call( exec_ctx, calld->connected_subchannel, calld->pollent, calld->path, calld->call_start_time, calld->deadline, &subchannel_call); @@ -1039,65 +1079,17 @@ static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, GPR_TIMER_END("cc_start_transport_stream_op", 0); } -// Gets data from the service config. Invoked when the resolver returns -// its initial result. -static void read_service_config_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - grpc_call_element *elem = arg; - channel_data *chand = elem->channel_data; - call_data *calld = elem->call_data; - // If this is an error, there's no point in looking at the service config. - if (error == GRPC_ERROR_NONE) { - // Get the method config table from channel data. - grpc_slice_hash_table *method_params_table = NULL; - if (chand->method_params_table != NULL) { - method_params_table = - grpc_slice_hash_table_ref(chand->method_params_table); - } - // If the method config table was present, use it. - if (method_params_table != NULL) { - const method_parameters *method_params = grpc_method_config_table_get( - exec_ctx, method_params_table, calld->path); - if (method_params != NULL) { - const bool have_method_timeout = - gpr_time_cmp(method_params->timeout, gpr_time_0(GPR_TIMESPAN)) != 0; - if (have_method_timeout || - method_params->wait_for_ready != WAIT_FOR_READY_UNSET) { - if (have_method_timeout) { - const gpr_timespec per_method_deadline = - gpr_time_add(calld->call_start_time, method_params->timeout); - if (gpr_time_cmp(per_method_deadline, calld->deadline) < 0) { - calld->deadline = per_method_deadline; - // Start deadline timer. - grpc_deadline_state_start(exec_ctx, elem, calld->deadline); - } - } - if (method_params->wait_for_ready != WAIT_FOR_READY_UNSET) { - calld->wait_for_ready_from_service_config = - method_params->wait_for_ready; - } - } - } - grpc_slice_hash_table_unref(exec_ctx, method_params_table); - } - } - GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "read_service_config"); -} - /* Constructor for call_data */ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_element_args *args) { call_data *calld = elem->call_data; - channel_data *chand = elem->channel_data; // Initialize data members. grpc_deadline_state_init(exec_ctx, elem, args->call_stack); calld->path = grpc_slice_ref_internal(args->path); calld->call_start_time = args->start_time; calld->deadline = gpr_convert_clock_type(args->deadline, GPR_CLOCK_MONOTONIC); calld->owning_call = args->call_stack; - grpc_closure_init(&calld->read_service_config, read_service_config_locked, - elem, grpc_combiner_scheduler(chand->combiner, false)); return GRPC_ERROR_NONE; } diff --git a/test/core/end2end/tests/negative_deadline.c b/test/core/end2end/tests/negative_deadline.c index f3607290360..06b1715b0c9 100644 --- a/test/core/end2end/tests/negative_deadline.c +++ b/test/core/end2end/tests/negative_deadline.c @@ -175,7 +175,7 @@ static void test_invoke_simple_request(grpc_end2end_test_config config, void negative_deadline(grpc_end2end_test_config config) { size_t i; - for (i = 1; i <= 4; i++) { + for (i = 2; i <= 4; i++) { test_invoke_simple_request(config, i); } } From 2c3d0a92c7ff1bcbcb883e633e17049908e17cf0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Feb 2017 15:26:24 -0800 Subject: [PATCH 010/188] Better cleanup If the caller specifies a timeout, set it at call initialization When we send initial metadata, take service config into account (iff service config specifies a shorter deadline) Saves a combiner on a critical path --- src/core/ext/client_channel/client_channel.c | 3 ++- src/core/lib/channel/deadline_filter.c | 21 +++++++++++++++++++- src/core/lib/channel/deadline_filter.h | 10 ++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index e437340e143..7eb6c4a1ce9 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -740,7 +740,7 @@ static void apply_final_configuration_locked(grpc_exec_ctx *exec_ctx, grpc_slice_hash_table_unref(exec_ctx, method_params_table); } /* Start deadline timer. */ - grpc_deadline_state_start(exec_ctx, elem, calld->deadline); + grpc_deadline_state_reset(exec_ctx, elem, calld->deadline); } static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, @@ -1090,6 +1090,7 @@ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, calld->call_start_time = args->start_time; calld->deadline = gpr_convert_clock_type(args->deadline, GPR_CLOCK_MONOTONIC); calld->owning_call = args->call_stack; + grpc_deadline_state_start(exec_ctx, elem, calld->deadline); return GRPC_ERROR_NONE; } diff --git a/src/core/lib/channel/deadline_filter.c b/src/core/lib/channel/deadline_filter.c index 0e9443937cd..5a12d62f1d1 100644 --- a/src/core/lib/channel/deadline_filter.c +++ b/src/core/lib/channel/deadline_filter.c @@ -78,9 +78,21 @@ retry: (grpc_deadline_timer_state)gpr_atm_acq_load(&deadline_state->timer_state); switch (cur_state) { case GRPC_DEADLINE_STATE_PENDING: - case GRPC_DEADLINE_STATE_FINISHED: // Note: We do not start the timer if there is already a timer return; + case GRPC_DEADLINE_STATE_FINISHED: + if (gpr_atm_rel_cas(&deadline_state->timer_state, + GRPC_DEADLINE_STATE_FINISHED, + GRPC_DEADLINE_STATE_PENDING)) { + // If we've already created and destroyed a timer, we always create a + // new closure: we have no other guarantee that the inlined closure is + // not in use (it may hold a pending call to timer_callback) + closure = grpc_closure_create(timer_callback, elem, + grpc_schedule_on_exec_ctx); + } else { + goto retry; + } + break; case GRPC_DEADLINE_STATE_INITIAL: if (gpr_atm_rel_cas(&deadline_state->timer_state, GRPC_DEADLINE_STATE_INITIAL, @@ -177,6 +189,13 @@ void grpc_deadline_state_start(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, } } +void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, + gpr_timespec new_deadline) { + grpc_deadline_state* deadline_state = elem->call_data; + cancel_timer_if_needed(exec_ctx, deadline_state); + start_timer_if_needed(exec_ctx, elem, new_deadline); +} + void grpc_deadline_state_client_start_transport_stream_op( grpc_exec_ctx* exec_ctx, grpc_call_element* elem, grpc_transport_stream_op* op) { diff --git a/src/core/lib/channel/deadline_filter.h b/src/core/lib/channel/deadline_filter.h index 363ac58b6d5..72cd5cb9296 100644 --- a/src/core/lib/channel/deadline_filter.h +++ b/src/core/lib/channel/deadline_filter.h @@ -73,6 +73,16 @@ void grpc_deadline_state_destroy(grpc_exec_ctx* exec_ctx, void grpc_deadline_state_start(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, gpr_timespec deadline); +// Cancels the existing timer and starts a new one with new_deadline. +// +// Note: It is generally safe to call this with an earlier deadline +// value than the current one, but not the reverse. No checks are done +// to ensure that the timer callback is not invoked while it is in the +// process of being reset, which means that attempting to increase the +// deadline may result in the timer being called twice. +void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, + gpr_timespec new_deadline); + // To be called from the client-side filter's start_transport_stream_op() // method. Ensures that the deadline timer is cancelled when the call // is completed. From 24992060c88dd93ac797b37fd077dd71239db655 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 21 Feb 2017 13:17:41 -0800 Subject: [PATCH 011/188] Finish streams that error. Don't histogram failed ops. Fail on failed write --- test/cpp/qps/client_async.cc | 4 +++- test/cpp/qps/client_sync.cc | 15 +++++++++++++-- test/cpp/qps/server_sync.cc | 4 +++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 4032039ea10..396d308e2a5 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -112,7 +112,9 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { next_state_ = State::RESP_DONE; return true; case State::RESP_DONE: - entry->set_value((UsageTimer::Now() - start_) * 1e9); + if (status_.ok()) { + entry->set_value((UsageTimer::Now() - start_) * 1e9); + } callback_(status_, &response_, entry); next_state_ = State::INVALID; return false; diff --git a/test/cpp/qps/client_sync.cc b/test/cpp/qps/client_sync.cc index 498416c64a1..a944c454960 100644 --- a/test/cpp/qps/client_sync.cc +++ b/test/cpp/qps/client_sync.cc @@ -129,7 +129,9 @@ class SynchronousUnaryClient final : public SynchronousClient { grpc::ClientContext context; grpc::Status s = stub->UnaryCall(&context, request_, &responses_[thread_idx]); - entry->set_value((UsageTimer::Now() - start) * 1e9); + if (s.ok()) { + entry->set_value((UsageTimer::Now() - start) * 1e9); + } entry->set_status(s.error_code()); return true; } @@ -154,7 +156,7 @@ class SynchronousStreamingClient final : public SynchronousClient { (*stream)->WritesDone(); Status s = (*stream)->Finish(); if (!s.ok()) { - gpr_log(GPR_ERROR, "Stream %zu received an error %s", i, + gpr_log(GPR_ERROR, "Stream %" PRIuPTR " received an error %s", i, s.error_message().c_str()); } } @@ -170,8 +172,17 @@ class SynchronousStreamingClient final : public SynchronousClient { if (stream_[thread_idx]->Write(request_) && stream_[thread_idx]->Read(&responses_[thread_idx])) { entry->set_value((UsageTimer::Now() - start) * 1e9); + // don't set the status since there isn't one yet return true; } + stream_[thread_idx]->WritesDone(); + Status s = stream_[thread_idx]->Finish(); + // don't set the value since the stream is failed and shouldn't be timed + entry->set_status(s.error_code()); + if (!s.ok()) { + gpr_log(GPR_ERROR, "Stream %" PRIuPTR " received an error %s", thread_idx, + s.error_message().c_str()); + } auto* stub = channels_[thread_idx % channels_.size()].get_stub(); context_[thread_idx].~ClientContext(); new (&context_[thread_idx]) ClientContext(); diff --git a/test/cpp/qps/server_sync.cc b/test/cpp/qps/server_sync.cc index 8076a4a6b9a..f79284d2254 100644 --- a/test/cpp/qps/server_sync.cc +++ b/test/cpp/qps/server_sync.cc @@ -74,7 +74,9 @@ class BenchmarkServiceImpl final : public BenchmarkService::Service { return Status(grpc::StatusCode::INTERNAL, "Error creating payload."); } } - stream->Write(response); + if (!stream->Write(response)) { + return Status(StatusCode::INTERNAL, "Server couldn't respond"); + } } return Status::OK; } From 7bc4d3edd8b5a8bc7760ac2e6fdd98cada79d20a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Feb 2017 16:05:10 -0800 Subject: [PATCH 012/188] Revert unnecessary change --- test/core/end2end/tests/negative_deadline.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/core/end2end/tests/negative_deadline.c b/test/core/end2end/tests/negative_deadline.c index 06b1715b0c9..0b61efbac97 100644 --- a/test/core/end2end/tests/negative_deadline.c +++ b/test/core/end2end/tests/negative_deadline.c @@ -124,11 +124,6 @@ static void simple_request_body(grpc_end2end_test_config config, memset(ops, 0, sizeof(ops)); op = ops; - op->op = GRPC_OP_SEND_INITIAL_METADATA; - op->data.send_initial_metadata.count = 0; - op->flags = 0; - op->reserved = NULL; - op++; op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; op->data.recv_status_on_client.status = &status; @@ -141,6 +136,11 @@ static void simple_request_body(grpc_end2end_test_config config, op->flags = 0; op->reserved = NULL; op++; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = 0; + op->reserved = NULL; + op++; op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; op->flags = 0; op->reserved = NULL; @@ -175,7 +175,7 @@ static void test_invoke_simple_request(grpc_end2end_test_config config, void negative_deadline(grpc_end2end_test_config config) { size_t i; - for (i = 2; i <= 4; i++) { + for (i = 1; i <= 4; i++) { test_invoke_simple_request(config, i); } } From 36dd42ec01d864982b20719d0a12a11d273e1397 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Feb 2017 16:06:45 -0800 Subject: [PATCH 013/188] Diff minimization --- src/core/ext/client_channel/client_channel.c | 81 ++++++++++---------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 7eb6c4a1ce9..229bf570fcd 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -654,6 +654,9 @@ typedef struct client_channel_call_data { grpc_linked_mdelem lb_token_mdelem; } call_data; +static void apply_final_configuration_locked(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem); + grpc_subchannel_call *grpc_client_channel_get_subchannel_call( grpc_call_element *call_elem) { grpc_subchannel_call *scc = GET_CALL((call_data *)call_elem->call_data); @@ -704,45 +707,6 @@ static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, call_data *calld) { gpr_free(ops); } -static void apply_final_configuration_locked(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem) { - /* apply service-config level configuration to the call (now that we're - * certain it exists) */ - channel_data *chand = elem->channel_data; - call_data *calld = elem->call_data; - /* Get the method config table from channel data. */ - grpc_slice_hash_table *method_params_table = NULL; - if (chand->method_params_table != NULL) { - method_params_table = grpc_slice_hash_table_ref(chand->method_params_table); - } - /* If the method config table was present, use it. */ - if (method_params_table != NULL) { - const method_parameters *method_params = grpc_method_config_table_get( - exec_ctx, method_params_table, calld->path); - if (method_params != NULL) { - const bool have_method_timeout = - gpr_time_cmp(method_params->timeout, gpr_time_0(GPR_TIMESPAN)) != 0; - if (have_method_timeout || - method_params->wait_for_ready != WAIT_FOR_READY_UNSET) { - if (have_method_timeout) { - const gpr_timespec per_method_deadline = - gpr_time_add(calld->call_start_time, method_params->timeout); - if (gpr_time_cmp(per_method_deadline, calld->deadline) < 0) { - calld->deadline = per_method_deadline; - } - } - if (method_params->wait_for_ready != WAIT_FOR_READY_UNSET) { - calld->wait_for_ready_from_service_config = - method_params->wait_for_ready; - } - } - } - grpc_slice_hash_table_unref(exec_ctx, method_params_table); - } - /* Start deadline timer. */ - grpc_deadline_state_reset(exec_ctx, elem, calld->deadline); -} - static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_call_element *elem = arg; @@ -1079,6 +1043,45 @@ static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, GPR_TIMER_END("cc_start_transport_stream_op", 0); } +static void apply_final_configuration_locked(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem) { + /* apply service-config level configuration to the call (now that we're + * certain it exists) */ + channel_data *chand = elem->channel_data; + call_data *calld = elem->call_data; + /* Get the method config table from channel data. */ + grpc_slice_hash_table *method_params_table = NULL; + if (chand->method_params_table != NULL) { + method_params_table = grpc_slice_hash_table_ref(chand->method_params_table); + } + /* If the method config table was present, use it. */ + if (method_params_table != NULL) { + const method_parameters *method_params = grpc_method_config_table_get( + exec_ctx, method_params_table, calld->path); + if (method_params != NULL) { + const bool have_method_timeout = + gpr_time_cmp(method_params->timeout, gpr_time_0(GPR_TIMESPAN)) != 0; + if (have_method_timeout || + method_params->wait_for_ready != WAIT_FOR_READY_UNSET) { + if (have_method_timeout) { + const gpr_timespec per_method_deadline = + gpr_time_add(calld->call_start_time, method_params->timeout); + if (gpr_time_cmp(per_method_deadline, calld->deadline) < 0) { + calld->deadline = per_method_deadline; + } + } + if (method_params->wait_for_ready != WAIT_FOR_READY_UNSET) { + calld->wait_for_ready_from_service_config = + method_params->wait_for_ready; + } + } + } + grpc_slice_hash_table_unref(exec_ctx, method_params_table); + } + /* Start deadline timer. */ + grpc_deadline_state_reset(exec_ctx, elem, calld->deadline); +} + /* Constructor for call_data */ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, From f8a7d93a2c7a5e5ba03b1d2c3c356bbf34198463 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 1 Mar 2017 12:12:24 -0800 Subject: [PATCH 014/188] Fix mem leak in error_test --- test/core/iomgr/error_test.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/core/iomgr/error_test.c b/test/core/iomgr/error_test.c index 5137ef9758f..8a0e7625e0d 100644 --- a/test/core/iomgr/error_test.c +++ b/test/core/iomgr/error_test.c @@ -61,11 +61,14 @@ static void test_set_get_int() error = grpc_error_set_int(error, GRPC_ERROR_INT_FILE_LINE, line); GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_FILE_LINE, &i)); GPR_ASSERT(i == line); + + GRPC_ERROR_UNREF(error); } static void test_set_get_str() { grpc_error* error = GRPC_ERROR_CREATE("Test"); + GPR_ASSERT(!grpc_error_get_str(error, GRPC_ERROR_STR_SYSCALL)); GPR_ASSERT(!grpc_error_get_str(error, GRPC_ERROR_STR_TSI_ERROR)); @@ -86,10 +89,13 @@ static void test_set_get_str() c = grpc_error_get_str(error, GRPC_ERROR_STR_DESCRIPTION); GPR_ASSERT(c); GPR_ASSERT(!strcmp(c, "new desc")); + + GRPC_ERROR_UNREF(error); } static void test_copy_and_unref() { + // error1 has one ref grpc_error* error1 = GRPC_ERROR_CREATE("Test"); grpc_error* error2 = grpc_error_set_str(error1, GRPC_ERROR_STR_GRPC_MESSAGE, "message"); const char* c = grpc_error_get_str(error1, GRPC_ERROR_STR_GRPC_MESSAGE); @@ -100,7 +106,9 @@ static void test_copy_and_unref() GPR_ASSERT(!strcmp(c, "message")); GPR_ASSERT(error1 == error2); + // error 1 has two refs GRPC_ERROR_REF(error1); + // this gives error3 a ref to the new error, and decrements error1 to one ref grpc_error* error3 = grpc_error_set_str(error1, GRPC_ERROR_STR_DESCRIPTION, "reset"); GPR_ASSERT(error3 != error1); // should not be the same because of extra ref c = grpc_error_get_str(error3, GRPC_ERROR_STR_GRPC_MESSAGE); @@ -114,6 +122,9 @@ static void test_copy_and_unref() c = grpc_error_get_str(error3, GRPC_ERROR_STR_DESCRIPTION); GPR_ASSERT(c); GPR_ASSERT(!strcmp(c, "reset")); + + GRPC_ERROR_UNREF(error1); + GRPC_ERROR_UNREF(error3); } static void print_error_strings() @@ -125,6 +136,7 @@ static void print_error_strings() error = grpc_error_set_int(error, GRPC_ERROR_INT_SIZE, 666); error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "message"); gpr_log(GPR_DEBUG, "%s", grpc_error_string(error)); + GRPC_ERROR_UNREF(error); } int main(int argc, char **argv) { From 8174cceb14b39091500f6f3562987c5570462f87 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 1 Mar 2017 15:38:41 -0800 Subject: [PATCH 015/188] Generate CSV file --- test/core/memory_usage/client.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/core/memory_usage/client.c b/test/core/memory_usage/client.c index 09f0e2d8670..c33ffbb0a0a 100644 --- a/test/core/memory_usage/client.c +++ b/test/core/memory_usage/client.c @@ -310,6 +310,26 @@ int main(int argc, char **argv) { server_calls_end.total_size_relative - after_server_create.total_size_relative); + const char *csv_file = "memory_usage.csv"; + FILE *csv = fopen(csv_file, "w"); + if (csv) { + fprintf(csv, "%f,%zi,%zi,%f,%zi\n", + (double)(client_calls_inflight.total_size_relative - + client_benchmark_calls_start.total_size_relative) / + benchmark_iterations, + client_channel_end.total_size_relative - + client_channel_start.total_size_relative, + after_server_create.total_size_relative - + before_server_create.total_size_relative, + (double)(server_calls_inflight.total_size_relative - + server_benchmark_calls_start.total_size_relative) / + benchmark_iterations, + server_calls_end.total_size_relative - + after_server_create.total_size_relative); + fclose(csv); + gpr_log(GPR_INFO, "Summary written to %s", csv_file); + } + grpc_memory_counters_destroy(); return 0; } From 2144f948aaa72d9adc96fe35bd3b33e41eb8ac3e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 1 Mar 2017 15:46:25 -0800 Subject: [PATCH 016/188] Run memory measurements as part of microbenchmark runs --- tools/profiling/latency_profile/run_latency_profile.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/profiling/latency_profile/run_latency_profile.sh b/tools/profiling/latency_profile/run_latency_profile.sh index e9baee09574..41423fc3c19 100755 --- a/tools/profiling/latency_profile/run_latency_profile.sh +++ b/tools/profiling/latency_profile/run_latency_profile.sh @@ -44,4 +44,9 @@ else PYTHON=python2.7 fi +make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server +bins/opt/memory_profile_test +bq load microbenchmarks.memory memory_usage.csv + $PYTHON tools/run_tests/run_microbenchmark.py --collect summary perf latency --bigquery_upload + From ffe4f5e0840c1879e974b2463384b0a461bba468 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 1 Mar 2017 12:12:24 -0800 Subject: [PATCH 017/188] Fix mem leak in error_test --- test/core/iomgr/error_test.c | 82 ++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/test/core/iomgr/error_test.c b/test/core/iomgr/error_test.c index 5137ef9758f..0452e4576a4 100644 --- a/test/core/iomgr/error_test.c +++ b/test/core/iomgr/error_test.c @@ -43,91 +43,93 @@ #include "test/core/util/test_config.h" -static void test_set_get_int() -{ +static void test_set_get_int() { grpc_error* error = GRPC_ERROR_CREATE("Test"); + GPR_ASSERT(error); intptr_t i = 0; GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_FILE_LINE, &i)); - GPR_ASSERT(i); // line set will never be 0 + GPR_ASSERT(i); // line set will never be 0 GPR_ASSERT(!grpc_error_get_int(error, GRPC_ERROR_INT_ERRNO, &i)); GPR_ASSERT(!grpc_error_get_int(error, GRPC_ERROR_INT_SIZE, &i)); - intptr_t errno = 314; - error = grpc_error_set_int(error, GRPC_ERROR_INT_ERRNO, errno); + intptr_t errnumber = 314; + error = grpc_error_set_int(error, GRPC_ERROR_INT_ERRNO, errnumber); GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_ERRNO, &i)); - GPR_ASSERT(i == errno); + GPR_ASSERT(i == errnumber); - intptr_t line = 555; - error = grpc_error_set_int(error, GRPC_ERROR_INT_FILE_LINE, line); - GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_FILE_LINE, &i)); - GPR_ASSERT(i == line); + intptr_t http = 2; + error = grpc_error_set_int(error, GRPC_ERROR_INT_HTTP2_ERROR, http); + GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_HTTP2_ERROR, &i)); + GPR_ASSERT(i == http); + + GRPC_ERROR_UNREF(error); } -static void test_set_get_str() -{ +static void test_set_get_str() { grpc_error* error = GRPC_ERROR_CREATE("Test"); + GPR_ASSERT(!grpc_error_get_str(error, GRPC_ERROR_STR_SYSCALL)); GPR_ASSERT(!grpc_error_get_str(error, GRPC_ERROR_STR_TSI_ERROR)); const char* c = grpc_error_get_str(error, GRPC_ERROR_STR_FILE); GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "test/core/iomgr/error_test.c")); + GPR_ASSERT(strstr(c, "error_test.c")); // __FILE__ expands differently on + // Windows. All should at least + // contain error_test.c c = grpc_error_get_str(error, GRPC_ERROR_STR_DESCRIPTION); GPR_ASSERT(c); GPR_ASSERT(!strcmp(c, "Test")); - error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "message"); + error = + grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "longer message"); c = grpc_error_get_str(error, GRPC_ERROR_STR_GRPC_MESSAGE); GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "message")); + GPR_ASSERT(!strcmp(c, "longer message")); - error = grpc_error_set_str(error, GRPC_ERROR_STR_DESCRIPTION, "new desc"); - c = grpc_error_get_str(error, GRPC_ERROR_STR_DESCRIPTION); - GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "new desc")); + GRPC_ERROR_UNREF(error); } -static void test_copy_and_unref() -{ - grpc_error* error1 = GRPC_ERROR_CREATE("Test"); - grpc_error* error2 = grpc_error_set_str(error1, GRPC_ERROR_STR_GRPC_MESSAGE, "message"); +static void test_copy_and_unref() { + // error1 has one ref + grpc_error* error1 = grpc_error_set_str( + GRPC_ERROR_CREATE("Test"), GRPC_ERROR_STR_GRPC_MESSAGE, "message"); const char* c = grpc_error_get_str(error1, GRPC_ERROR_STR_GRPC_MESSAGE); GPR_ASSERT(c); GPR_ASSERT(!strcmp(c, "message")); - c = grpc_error_get_str(error2, GRPC_ERROR_STR_GRPC_MESSAGE); - GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "message")); - GPR_ASSERT(error1 == error2); + // error 1 has two refs GRPC_ERROR_REF(error1); - grpc_error* error3 = grpc_error_set_str(error1, GRPC_ERROR_STR_DESCRIPTION, "reset"); - GPR_ASSERT(error3 != error1); // should not be the same because of extra ref + // this gives error3 a ref to the new error, and decrements error1 to one ref + grpc_error* error3 = + grpc_error_set_str(error1, GRPC_ERROR_STR_SYSCALL, "syscall"); + GPR_ASSERT(error3 != error1); // should not be the same because of extra ref c = grpc_error_get_str(error3, GRPC_ERROR_STR_GRPC_MESSAGE); GPR_ASSERT(c); GPR_ASSERT(!strcmp(c, "message")); - c = grpc_error_get_str(error1, GRPC_ERROR_STR_DESCRIPTION); + // error 1 should not have a syscall but 3 should + GPR_ASSERT(!grpc_error_get_str(error1, GRPC_ERROR_STR_SYSCALL)); + c = grpc_error_get_str(error3, GRPC_ERROR_STR_SYSCALL); GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "Test")); + GPR_ASSERT(!strcmp(c, "syscall")); - c = grpc_error_get_str(error3, GRPC_ERROR_STR_DESCRIPTION); - GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "reset")); + GRPC_ERROR_UNREF(error1); + GRPC_ERROR_UNREF(error3); } -static void print_error_strings() -{ - grpc_error* error = grpc_error_set_int(GRPC_ERROR_CREATE("Error"), - GRPC_ERROR_INT_GRPC_STATUS, - GRPC_STATUS_UNIMPLEMENTED); +static void print_error_strings() { + grpc_error* error = + grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_GRPC_STATUS, + GRPC_STATUS_UNIMPLEMENTED); error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, 0); error = grpc_error_set_int(error, GRPC_ERROR_INT_SIZE, 666); error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "message"); gpr_log(GPR_DEBUG, "%s", grpc_error_string(error)); + GRPC_ERROR_UNREF(error); } -int main(int argc, char **argv) { +int main(int argc, char** argv) { grpc_test_init(argc, argv); grpc_init(); test_set_get_int(); From d3d46a5385f9566cd630755679004c0088865744 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Fri, 3 Mar 2017 01:30:55 +0000 Subject: [PATCH 018/188] Enable suppressed-message lint --- .pylintrc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index 04c0088c88e..f41ef6542df 100644 --- a/.pylintrc +++ b/.pylintrc @@ -17,7 +17,6 @@ notes=FIXME,XXX #TODO: Enable no-init #TODO: Enable duplicate-code #TODO: Enable invalid-name -#TODO: Enable suppressed-message #TODO: Enable locally-disabled #TODO: Enable protected-access #TODO: Enable no-name-in-module @@ -37,4 +36,4 @@ notes=FIXME,XXX #TODO: Enable too-many-nested-blocks #TODO: Enable super-init-not-called -disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,suppressed-message,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called +disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called From 18f025e0a3e56d4e698f921220e18954b14723cc Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Fri, 3 Mar 2017 01:41:34 +0000 Subject: [PATCH 019/188] Fix and enable super-init-not-called lint --- .pylintrc | 3 +-- .../grpc/framework/interfaces/base/base.py | 23 ++++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.pylintrc b/.pylintrc index f41ef6542df..6c788cd072d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -34,6 +34,5 @@ notes=FIXME,XXX #TODO: Enable useless-else-on-loop #TODO: Enable too-many-return-statements #TODO: Enable too-many-nested-blocks -#TODO: Enable super-init-not-called -disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called +disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks diff --git a/src/python/grpcio/grpc/framework/interfaces/base/base.py b/src/python/grpcio/grpc/framework/interfaces/base/base.py index cb3328296c5..8e694e3f12c 100644 --- a/src/python/grpcio/grpc/framework/interfaces/base/base.py +++ b/src/python/grpcio/grpc/framework/interfaces/base/base.py @@ -50,22 +50,23 @@ from grpc.framework.foundation import abandonment # pylint: disable=unused-impo class NoSuchMethodError(Exception): """Indicates that an unrecognized operation has been called. - Attributes: - code: A code value to communicate to the other side of the operation along - with indication of operation termination. May be None. - details: A details value to communicate to the other side of the operation - along with indication of operation termination. May be None. - """ - - def __init__(self, code, details): - """Constructor. - - Args: + Attributes: code: A code value to communicate to the other side of the operation along with indication of operation termination. May be None. details: A details value to communicate to the other side of the operation along with indication of operation termination. May be None. """ + + def __init__(self, code, details): + """Constructor. + + Args: + code: A code value to communicate to the other side of the operation + along with indication of operation termination. May be None. + details: A details value to communicate to the other side of the + operation along with indication of operation termination. May be None. + """ + super(NoSuchMethodError, self).__init__() self.code = code self.details = details From db0f85b0105a598953489eebfa28385e1364bd7a Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Fri, 3 Mar 2017 05:41:50 +0000 Subject: [PATCH 020/188] Configure and enable too-many-arguments lint --- .pylintrc | 9 +++++++-- src/python/grpcio/grpc/beta/_client_adaptations.py | 2 ++ src/python/grpcio/grpc/beta/implementations.py | 2 ++ src/python/grpcio/grpc/framework/interfaces/base/base.py | 2 ++ src/python/grpcio/grpc/framework/interfaces/face/face.py | 2 ++ 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index 6c788cd072d..a8d14599405 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,6 +3,12 @@ # not include "unused_" and "ignored_" by default? dummy-variables-rgx=^ignored_|^unused_ +[DESIGN] +# NOTE(nathaniel): Not particularly attached to this value; it just seems to +# be what works for us at the moment (excepting the dead-code-walking Beta +# API). +max-args=6 + [MISCELLANEOUS] # NOTE(nathaniel): We are big fans of "TODO(): " and # "NOTE(): ". We do not allow "TODO:", @@ -13,7 +19,6 @@ notes=FIXME,XXX #TODO: Enable missing-docstring #TODO: Enable too-few-public-methods -#TODO: Enable too-many-arguments #TODO: Enable no-init #TODO: Enable duplicate-code #TODO: Enable invalid-name @@ -35,4 +40,4 @@ notes=FIXME,XXX #TODO: Enable too-many-return-statements #TODO: Enable too-many-nested-blocks -disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks +disable=missing-docstring,too-few-public-methods,no-init,duplicate-code,invalid-name,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks diff --git a/src/python/grpcio/grpc/beta/_client_adaptations.py b/src/python/grpcio/grpc/beta/_client_adaptations.py index 901cb856124..54ed9a10d19 100644 --- a/src/python/grpcio/grpc/beta/_client_adaptations.py +++ b/src/python/grpcio/grpc/beta/_client_adaptations.py @@ -35,6 +35,8 @@ from grpc.framework.common import cardinality from grpc.framework.foundation import future from grpc.framework.interfaces.face import face +# pylint: disable=too-many-arguments + _STATUS_CODE_TO_ABORTION_KIND_AND_ABORTION_ERROR_CLASS = { grpc.StatusCode.CANCELLED: (face.Abortion.Kind.CANCELLED, face.CancellationError), diff --git a/src/python/grpcio/grpc/beta/implementations.py b/src/python/grpcio/grpc/beta/implementations.py index 0b795776890..113fd38f8a6 100644 --- a/src/python/grpcio/grpc/beta/implementations.py +++ b/src/python/grpcio/grpc/beta/implementations.py @@ -41,6 +41,8 @@ from grpc.beta import interfaces # pylint: disable=unused-import from grpc.framework.common import cardinality # pylint: disable=unused-import from grpc.framework.interfaces.face import face # pylint: disable=unused-import +# pylint: disable=too-many-arguments + ChannelCredentials = grpc.ChannelCredentials ssl_channel_credentials = grpc.ssl_channel_credentials CallCredentials = grpc.CallCredentials diff --git a/src/python/grpcio/grpc/framework/interfaces/base/base.py b/src/python/grpcio/grpc/framework/interfaces/base/base.py index 8e694e3f12c..aa80e65f57a 100644 --- a/src/python/grpcio/grpc/framework/interfaces/base/base.py +++ b/src/python/grpcio/grpc/framework/interfaces/base/base.py @@ -46,6 +46,8 @@ import six # abandonment is referenced from specification in this module. from grpc.framework.foundation import abandonment # pylint: disable=unused-import +# pylint: disable=too-many-arguments + class NoSuchMethodError(Exception): """Indicates that an unrecognized operation has been called. diff --git a/src/python/grpcio/grpc/framework/interfaces/face/face.py b/src/python/grpcio/grpc/framework/interfaces/face/face.py index 6c7e2a3af6a..c6c44fe4e4c 100644 --- a/src/python/grpcio/grpc/framework/interfaces/face/face.py +++ b/src/python/grpcio/grpc/framework/interfaces/face/face.py @@ -42,6 +42,8 @@ from grpc.framework.foundation import abandonment # pylint: disable=unused-impo from grpc.framework.foundation import future # pylint: disable=unused-import from grpc.framework.foundation import stream # pylint: disable=unused-import +# pylint: disable=too-many-arguments + class NoSuchMethodError(Exception): """Raised by customer code to indicate an unrecognized method. From c3038a5cb379d19ec1f2352ee18f0012035dd21c Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Fri, 3 Mar 2017 05:54:30 +0000 Subject: [PATCH 021/188] Suppress and enable too-many-locals lint --- .pylintrc | 3 +-- src/python/grpcio/grpc/beta/_client_adaptations.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.pylintrc b/.pylintrc index a8d14599405..6f24d1bf3e6 100644 --- a/.pylintrc +++ b/.pylintrc @@ -31,7 +31,6 @@ notes=FIXME,XXX # enable cyclic-import after a 1.7-or-later pylint release that recognizes our # disable=cyclic-import suppressions. #TODO: Enable too-many-instance-attributes -#TODO: Enable too-many-locals #TODO: Enable too-many-lines #TODO: Enable redefined-variable-type #TODO: Enable next-method-called @@ -40,4 +39,4 @@ notes=FIXME,XXX #TODO: Enable too-many-return-statements #TODO: Enable too-many-nested-blocks -disable=missing-docstring,too-few-public-methods,no-init,duplicate-code,invalid-name,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks +disable=missing-docstring,too-few-public-methods,no-init,duplicate-code,invalid-name,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks diff --git a/src/python/grpcio/grpc/beta/_client_adaptations.py b/src/python/grpcio/grpc/beta/_client_adaptations.py index 54ed9a10d19..9b5f5766d25 100644 --- a/src/python/grpcio/grpc/beta/_client_adaptations.py +++ b/src/python/grpcio/grpc/beta/_client_adaptations.py @@ -35,7 +35,7 @@ from grpc.framework.common import cardinality from grpc.framework.foundation import future from grpc.framework.interfaces.face import face -# pylint: disable=too-many-arguments +# pylint: disable=too-many-arguments,too-many-locals _STATUS_CODE_TO_ABORTION_KIND_AND_ABORTION_ERROR_CLASS = { grpc.StatusCode.CANCELLED: (face.Abortion.Kind.CANCELLED, From 8fccf59d047ab8f837f7a12122ef81a663bd6885 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Fri, 3 Mar 2017 06:08:02 +0000 Subject: [PATCH 022/188] Enable too-many-return-statements lint --- .pylintrc | 3 +-- src/python/grpcio/grpc/beta/_server_adaptations.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index 6f24d1bf3e6..dbfdc737316 100644 --- a/.pylintrc +++ b/.pylintrc @@ -36,7 +36,6 @@ notes=FIXME,XXX #TODO: Enable next-method-called #TODO: Enable import-error #TODO: Enable useless-else-on-loop -#TODO: Enable too-many-return-statements #TODO: Enable too-many-nested-blocks -disable=missing-docstring,too-few-public-methods,no-init,duplicate-code,invalid-name,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks +disable=missing-docstring,too-few-public-methods,no-init,duplicate-code,invalid-name,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-nested-blocks diff --git a/src/python/grpcio/grpc/beta/_server_adaptations.py b/src/python/grpcio/grpc/beta/_server_adaptations.py index 81348d5d87f..ed6aa4c3efb 100644 --- a/src/python/grpcio/grpc/beta/_server_adaptations.py +++ b/src/python/grpcio/grpc/beta/_server_adaptations.py @@ -41,6 +41,8 @@ from grpc.framework.foundation import logging_pool from grpc.framework.foundation import stream from grpc.framework.interfaces.face import face +# pylint: disable=too-many-return-statements + _DEFAULT_POOL_SIZE = 8 From 3ec1366eb6cc9d6f7a00b0de8c73a5c657daf21d Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Fri, 3 Mar 2017 06:23:53 +0000 Subject: [PATCH 023/188] Enable unused-argument lint --- .pylintrc | 3 +-- src/python/grpcio/grpc/_channel.py | 10 +++------- src/python/grpcio/grpc/beta/_client_adaptations.py | 2 +- src/python/grpcio/grpc/beta/_server_adaptations.py | 2 +- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.pylintrc b/.pylintrc index dbfdc737316..41027479061 100644 --- a/.pylintrc +++ b/.pylintrc @@ -25,7 +25,6 @@ notes=FIXME,XXX #TODO: Enable locally-disabled #TODO: Enable protected-access #TODO: Enable no-name-in-module -#TODO: Enable unused-argument #TODO: Enable wrong-import-order # TODO(https://github.com/PyCQA/pylint/issues/59#issuecomment-283774279): # enable cyclic-import after a 1.7-or-later pylint release that recognizes our @@ -38,4 +37,4 @@ notes=FIXME,XXX #TODO: Enable useless-else-on-loop #TODO: Enable too-many-nested-blocks -disable=missing-docstring,too-few-public-methods,no-init,duplicate-code,invalid-name,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-nested-blocks +disable=missing-docstring,too-few-public-methods,no-init,duplicate-code,invalid-name,locally-disabled,protected-access,no-name-in-module,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-nested-blocks diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index b4145fee91a..4329c9934ce 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -237,7 +237,7 @@ def _consume_request_iterator(request_iterator, state, call, cygrpc.Operations(operations), event_handler) state.due.add(cygrpc.OperationType.send_close_from_client) - def stop_consumption_thread(timeout): + def stop_consumption_thread(timeout): # pylint: disable=unused-argument with state.condition: if state.code is None: call.cancel() @@ -736,7 +736,7 @@ def _run_channel_spin_thread(state): state.managed_calls = None return - def stop_channel_spin(timeout): + def stop_channel_spin(timeout): # pylint: disable=unused-argument with state.lock: if state.managed_calls is not None: for call in state.managed_calls: @@ -877,12 +877,8 @@ def _moot(state): def _subscribe(state, callback, try_to_connect): with state.lock: if not state.callbacks_and_connectivities and not state.polling: - - def cancel_all_subscriptions(timeout): - _moot(state) - polling_thread = _common.CleanupThread( - cancel_all_subscriptions, + lambda timeout: _moot(state), target=_poll_connectivity, args=(state, state.channel, bool(try_to_connect))) polling_thread.start() diff --git a/src/python/grpcio/grpc/beta/_client_adaptations.py b/src/python/grpcio/grpc/beta/_client_adaptations.py index 9b5f5766d25..3c69acc0195 100644 --- a/src/python/grpcio/grpc/beta/_client_adaptations.py +++ b/src/python/grpcio/grpc/beta/_client_adaptations.py @@ -35,7 +35,7 @@ from grpc.framework.common import cardinality from grpc.framework.foundation import future from grpc.framework.interfaces.face import face -# pylint: disable=too-many-arguments,too-many-locals +# pylint: disable=too-many-arguments,too-many-locals,unused-argument _STATUS_CODE_TO_ABORTION_KIND_AND_ABORTION_ERROR_CLASS = { grpc.StatusCode.CANCELLED: (face.Abortion.Kind.CANCELLED, diff --git a/src/python/grpcio/grpc/beta/_server_adaptations.py b/src/python/grpcio/grpc/beta/_server_adaptations.py index ed6aa4c3efb..cf10c26d2fe 100644 --- a/src/python/grpcio/grpc/beta/_server_adaptations.py +++ b/src/python/grpcio/grpc/beta/_server_adaptations.py @@ -181,7 +181,7 @@ def _run_request_pipe_thread(request_iterator, request_consumer, return request_consumer.terminate() - def stop_request_pipe(timeout): + def stop_request_pipe(timeout): # pylint: disable=unused-argument thread_joined.set() request_pipe_thread = _common.CleanupThread( From 19e6b88af0e4a3b9f3cb3fbfdc8fca96d320f26d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 07:57:53 -0800 Subject: [PATCH 024/188] Include build env --- test/core/memory_usage/client.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/core/memory_usage/client.c b/test/core/memory_usage/client.c index c33ffbb0a0a..107abbc1b3d 100644 --- a/test/core/memory_usage/client.c +++ b/test/core/memory_usage/client.c @@ -43,6 +43,7 @@ #include #include #include +#include "src/core/lib/support/env.h" #include "src/core/lib/support/string.h" #include "test/core/util/memory_counters.h" #include "test/core/util/test_config.h" @@ -313,7 +314,9 @@ int main(int argc, char **argv) { const char *csv_file = "memory_usage.csv"; FILE *csv = fopen(csv_file, "w"); if (csv) { - fprintf(csv, "%f,%zi,%zi,%f,%zi\n", + char *env_build = gpr_getenv("BUILD_NUMBER"); + char *env_job = gpr_getenv("JOB_NAME"); + fprintf(csv, "%f,%zi,%zi,%f,%zi,%s,%s\n", (double)(client_calls_inflight.total_size_relative - client_benchmark_calls_start.total_size_relative) / benchmark_iterations, @@ -325,7 +328,8 @@ int main(int argc, char **argv) { server_benchmark_calls_start.total_size_relative) / benchmark_iterations, server_calls_end.total_size_relative - - after_server_create.total_size_relative); + after_server_create.total_size_relative, + env_build == NULL ? "" : env_build, env_job == NULL ? "" : env_job); fclose(csv); gpr_log(GPR_INFO, "Summary written to %s", csv_file); } From 5e3215338f3f69c626cced758e9c49e03252aa24 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 08:50:25 -0800 Subject: [PATCH 025/188] Move helpers to a common place, use them everywhere --- CMakeLists.txt | 49 ++++ Makefile | 90 ++++-- build.yaml | 16 ++ include/grpc/impl/codegen/sync.h | 8 + test/cpp/microbenchmarks/bm_call_create.cc | 10 +- test/cpp/microbenchmarks/bm_chttp2_hpack.cc | 14 +- test/cpp/microbenchmarks/bm_closure.cc | 110 +++---- test/cpp/microbenchmarks/bm_cq.cc | 23 +- test/cpp/microbenchmarks/bm_error.cc | 26 ++ test/cpp/microbenchmarks/bm_fullstack.cc | 269 ++---------------- test/cpp/microbenchmarks/bm_metadata.cc | 45 ++- test/cpp/microbenchmarks/helpers.h | 119 ++++++++ .../generated/sources_and_headers.json | 25 ++ .../grpc_benchmark/grpc_benchmark.vcxproj | 201 +++++++++++++ .../grpc_benchmark.vcxproj.filters | 21 ++ 15 files changed, 663 insertions(+), 363 deletions(-) create mode 100644 test/cpp/microbenchmarks/helpers.h create mode 100644 vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj create mode 100644 vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index 5694bd195e9..494300e9704 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3096,6 +3096,48 @@ endif() if (gRPC_BUILD_TESTS) +add_library(grpc_benchmark +) + +if(WIN32 AND MSVC) + set_target_properties(grpc_benchmark PROPERTIES COMPILE_PDB_NAME "grpc_benchmark" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc_benchmark.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + + +target_include_directories(grpc_benchmark + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${ZLIB_INCLUDE_DIR} + PRIVATE ${BENCHMARK}/include + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(grpc_benchmark + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + benchmark + grpc++ + grpc + ${_gRPC_GFLAGS_LIBRARIES} +) + + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_library(grpc_cli_libs test/cpp/util/cli_call.cc test/cpp/util/cli_credentials.cc @@ -7505,6 +7547,7 @@ target_include_directories(bm_call_create target_link_libraries(bm_call_create ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7543,6 +7586,7 @@ target_include_directories(bm_chttp2_hpack target_link_libraries(bm_chttp2_hpack ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7581,6 +7625,7 @@ target_include_directories(bm_closure target_link_libraries(bm_closure ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7619,6 +7664,7 @@ target_include_directories(bm_cq target_link_libraries(bm_cq ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7657,6 +7703,7 @@ target_include_directories(bm_error target_link_libraries(bm_error ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7695,6 +7742,7 @@ target_include_directories(bm_fullstack target_link_libraries(bm_fullstack ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7733,6 +7781,7 @@ target_include_directories(bm_metadata target_link_libraries(bm_metadata ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc_test_util grpc diff --git a/Makefile b/Makefile index c16b4ff8ea6..9b872a39b25 100644 --- a/Makefile +++ b/Makefile @@ -4960,6 +4960,54 @@ ifneq ($(NO_DEPS),true) endif +LIBGRPC_BENCHMARK_SRC = \ + +PUBLIC_HEADERS_CXX += \ + +LIBGRPC_BENCHMARK_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC_BENCHMARK_SRC)))) + + +ifeq ($(NO_SECURE),true) + +# You can't build secure libraries if you don't have OpenSSL. + +$(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: openssl_dep_error + + +else + +ifeq ($(NO_PROTOBUF),true) + +# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay. + +$(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: protobuf_dep_error + + +else + +$(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC_BENCHMARK_OBJS) + $(E) "[AR] Creating $@" + $(Q) mkdir -p `dirname $@` + $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a + $(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBGRPC_BENCHMARK_OBJS) +ifeq ($(SYSTEM),Darwin) + $(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a +endif + + + + +endif + +endif + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(LIBGRPC_BENCHMARK_OBJS:.o=.dep) +endif +endif + + LIBGRPC_CLI_LIBS_SRC = \ test/cpp/util/cli_call.cc \ test/cpp/util/cli_credentials.cc \ @@ -12479,16 +12527,16 @@ $(BINDIR)/$(CONFIG)/bm_call_create: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_call_create: $(PROTOBUF_DEP) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_call_create: $(PROTOBUF_DEP) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_call_create + $(Q) $(LDXX) $(LDFLAGS) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_call_create endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_call_create.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_call_create.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_call_create: $(BM_CALL_CREATE_OBJS:.o=.dep) @@ -12522,16 +12570,16 @@ $(BINDIR)/$(CONFIG)/bm_chttp2_hpack: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_chttp2_hpack: $(PROTOBUF_DEP) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_chttp2_hpack: $(PROTOBUF_DEP) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_chttp2_hpack + $(Q) $(LDXX) $(LDFLAGS) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_chttp2_hpack endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_chttp2_hpack.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_chttp2_hpack.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_chttp2_hpack: $(BM_CHTTP2_HPACK_OBJS:.o=.dep) @@ -12565,16 +12613,16 @@ $(BINDIR)/$(CONFIG)/bm_closure: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_closure: $(PROTOBUF_DEP) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_closure: $(PROTOBUF_DEP) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_closure + $(Q) $(LDXX) $(LDFLAGS) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_closure endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_closure.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_closure.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_closure: $(BM_CLOSURE_OBJS:.o=.dep) @@ -12608,16 +12656,16 @@ $(BINDIR)/$(CONFIG)/bm_cq: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_cq: $(PROTOBUF_DEP) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_cq: $(PROTOBUF_DEP) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_cq + $(Q) $(LDXX) $(LDFLAGS) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_cq endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_cq.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_cq.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_cq: $(BM_CQ_OBJS:.o=.dep) @@ -12651,16 +12699,16 @@ $(BINDIR)/$(CONFIG)/bm_error: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_error: $(PROTOBUF_DEP) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_error: $(PROTOBUF_DEP) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_error + $(Q) $(LDXX) $(LDFLAGS) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_error endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_error.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_error.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_error: $(BM_ERROR_OBJS:.o=.dep) @@ -12694,16 +12742,16 @@ $(BINDIR)/$(CONFIG)/bm_fullstack: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_fullstack: $(PROTOBUF_DEP) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_fullstack: $(PROTOBUF_DEP) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack + $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_fullstack: $(BM_FULLSTACK_OBJS:.o=.dep) @@ -12737,16 +12785,16 @@ $(BINDIR)/$(CONFIG)/bm_metadata: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_metadata: $(PROTOBUF_DEP) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_metadata: $(PROTOBUF_DEP) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_metadata + $(Q) $(LDXX) $(LDFLAGS) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_metadata endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_metadata.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_metadata.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_metadata: $(BM_METADATA_OBJS:.o=.dep) diff --git a/build.yaml b/build.yaml index f59640f8abb..dc0ba108072 100644 --- a/build.yaml +++ b/build.yaml @@ -1214,6 +1214,15 @@ libs: - grpc++_codegen_base_src secure: false vs_project_guid: '{6EE56155-DF7C-4F6E-BFC4-F6F776BEB211}' +- name: grpc_benchmark + build: test + language: c++ + headers: + - test/cpp/microbenchmarks/helpers.h + deps: + - benchmark + - grpc++ + - grpc - name: grpc_cli_libs build: private language: c++ @@ -3011,6 +3020,7 @@ targets: src: - test/cpp/microbenchmarks/bm_call_create.cc deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3030,6 +3040,7 @@ targets: src: - test/cpp/microbenchmarks/bm_chttp2_hpack.cc deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3049,6 +3060,7 @@ targets: src: - test/cpp/microbenchmarks/bm_closure.cc deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3068,6 +3080,7 @@ targets: src: - test/cpp/microbenchmarks/bm_cq.cc deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3087,6 +3100,7 @@ targets: src: - test/cpp/microbenchmarks/bm_error.cc deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3106,6 +3120,7 @@ targets: src: - test/cpp/microbenchmarks/bm_fullstack.cc deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3129,6 +3144,7 @@ targets: src: - test/cpp/microbenchmarks/bm_metadata.cc deps: + - grpc_benchmark - benchmark - grpc_test_util - grpc diff --git a/include/grpc/impl/codegen/sync.h b/include/grpc/impl/codegen/sync.h index 96aec0c2c09..6a8e8a644f9 100644 --- a/include/grpc/impl/codegen/sync.h +++ b/include/grpc/impl/codegen/sync.h @@ -52,6 +52,10 @@ provides no memory barriers. */ +#ifdef __cplusplus +extern "C" { +#endif + /* Platform-specific type declarations of gpr_mu and gpr_cv. */ #include #include @@ -64,4 +68,8 @@ #error Unable to determine platform for sync #endif +#ifdef __cplusplus +} +#endif + #endif /* GRPC_IMPL_CODEGEN_SYNC_H */ diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index e2e5bbbe00e..8fed12ef4ee 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -55,13 +55,9 @@ extern "C" { #include "src/core/lib/transport/transport_impl.h" } +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -static struct Init { - Init() { grpc_init(); } - ~Init() { grpc_shutdown(); } -} g_init; - class BaseChannelFixture { public: BaseChannelFixture(grpc_channel *channel) : channel_(channel) {} @@ -89,6 +85,7 @@ class LameChannel : public BaseChannelFixture { template static void BM_CallCreateDestroy(benchmark::State &state) { + TrackCounters track_counters; Fixture fixture; grpc_completion_queue *cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); @@ -100,6 +97,7 @@ static void BM_CallCreateDestroy(benchmark::State &state) { deadline, NULL)); } grpc_completion_queue_destroy(cq); + track_counters.Finish(state); } BENCHMARK_TEMPLATE(BM_CallCreateDestroy, InsecureChannel); @@ -316,6 +314,7 @@ class SendEmptyMetadata { // perform on said filter. template static void BM_IsolatedFilter(benchmark::State &state) { + TrackCounters track_counters; Fixture fixture; std::ostringstream label; @@ -371,6 +370,7 @@ static void BM_IsolatedFilter(benchmark::State &state) { gpr_free(call_stack); state.SetLabel(label.str()); + track_counters.Finish(state); } typedef Fixture NoFilter; diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc index 5fb3f371306..029a89db3a1 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc @@ -42,18 +42,15 @@ extern "C" { #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/transport/static_metadata.h" } +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -static struct Init { - Init() { grpc_init(); } - ~Init() { grpc_shutdown(); } -} g_init; - //////////////////////////////////////////////////////////////////////////////// // HPACK encoder // static void BM_HpackEncoderInitDestroy(benchmark::State &state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_chttp2_hpack_compressor c; while (state.KeepRunning()) { @@ -62,11 +59,13 @@ static void BM_HpackEncoderInitDestroy(benchmark::State &state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_HpackEncoderInitDestroy); template static void BM_HpackEncoderEncodeHeader(benchmark::State &state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_metadata_batch b; @@ -102,6 +101,7 @@ static void BM_HpackEncoderEncodeHeader(benchmark::State &state) { << " header_bytes/iter:" << (static_cast(stats.header_bytes) / static_cast(state.iterations())); state.SetLabel(label.str()); + track_counters.Finish(state); } namespace hpack_encoder_fixtures { @@ -216,6 +216,7 @@ BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, // static void BM_HpackParserInitDestroy(benchmark::State &state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_chttp2_hpack_parser p; while (state.KeepRunning()) { @@ -224,6 +225,7 @@ static void BM_HpackParserInitDestroy(benchmark::State &state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_HpackParserInitDestroy); @@ -234,6 +236,7 @@ static void UnrefHeader(grpc_exec_ctx *exec_ctx, void *user_data, template static void BM_HpackParserParseHeader(benchmark::State &state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; std::vector init_slices = Fixture::GetInitSlices(); std::vector benchmark_slices = Fixture::GetBenchmarkSlices(); @@ -252,6 +255,7 @@ static void BM_HpackParserParseHeader(benchmark::State &state) { } grpc_chttp2_hpack_parser_destroy(&exec_ctx, &p); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } namespace hpack_parser_fixtures { diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc index 1f54e8c8b17..8111c9c9fa3 100644 --- a/test/cpp/microbenchmarks/bm_closure.cc +++ b/test/cpp/microbenchmarks/bm_closure.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -42,86 +42,49 @@ extern "C" { #include "src/core/lib/support/spinlock.h" } +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -#include - #ifdef GPR_LOW_LEVEL_COUNTERS extern "C" gpr_atm gpr_mu_locks; #endif -static class InitializeStuff { - public: - InitializeStuff() { grpc_init(); } - ~InitializeStuff() { grpc_shutdown(); } -} initialize_stuff; - -class TrackCounters { - public: - TrackCounters(benchmark::State& state) : state_(state) {} - - ~TrackCounters() { - std::ostringstream out; -#ifdef GPR_LOW_LEVEL_COUNTERS - out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - - mu_locks_at_start_) / - (double)state_.iterations()) - << " atm_cas/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) - - atm_cas_at_start_) / - (double)state_.iterations()) - << " atm_add/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - - atm_add_at_start_) / - (double)state_.iterations()); -#endif - state_.SetLabel(out.str()); - } - - private: - benchmark::State& state_; -#ifdef GPR_LOW_LEVEL_COUNTERS - const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks); - const size_t atm_cas_at_start_ = - gpr_atm_no_barrier_load(&gpr_counter_atm_cas); - const size_t atm_add_at_start_ = - gpr_atm_no_barrier_load(&gpr_counter_atm_add); -#endif -}; - static void BM_NoOpExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; while (state.KeepRunning()) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_exec_ctx_finish(&exec_ctx); } + track_counters.Finish(state); } BENCHMARK(BM_NoOpExecCtx); static void BM_WellFlushed(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; while (state.KeepRunning()) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_WellFlushed); static void DoNothing(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {} static void BM_ClosureInitAgainstExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_closure c; while (state.KeepRunning()) { benchmark::DoNotOptimize( grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx)); } + track_counters.Finish(state); } BENCHMARK(BM_ClosureInitAgainstExecCtx); static void BM_ClosureInitAgainstCombiner(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -131,11 +94,12 @@ static void BM_ClosureInitAgainstCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureInitAgainstCombiner); static void BM_ClosureRunOnExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_closure c; grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -144,11 +108,12 @@ static void BM_ClosureRunOnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureRunOnExecCtx); static void BM_ClosureCreateAndRun(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; while (state.KeepRunning()) { grpc_closure_run(&exec_ctx, grpc_closure_create(DoNothing, NULL, @@ -156,11 +121,12 @@ static void BM_ClosureCreateAndRun(benchmark::State& state) { GRPC_ERROR_NONE); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureCreateAndRun); static void BM_ClosureInitAndRun(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_closure c; while (state.KeepRunning()) { @@ -169,11 +135,12 @@ static void BM_ClosureInitAndRun(benchmark::State& state) { GRPC_ERROR_NONE); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureInitAndRun); static void BM_ClosureSchedOnExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_closure c; grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -182,11 +149,12 @@ static void BM_ClosureSchedOnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSchedOnExecCtx); static void BM_ClosureSched2OnExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_closure c1; grpc_closure c2; grpc_closure_init(&c1, DoNothing, NULL, grpc_schedule_on_exec_ctx); @@ -198,11 +166,12 @@ static void BM_ClosureSched2OnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched2OnExecCtx); static void BM_ClosureSched3OnExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_closure c1; grpc_closure c2; grpc_closure c3; @@ -217,11 +186,12 @@ static void BM_ClosureSched3OnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched3OnExecCtx); static void BM_AcquireMutex(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; // for comparison with the combiner stuff below gpr_mu mu; gpr_mu_init(&mu); @@ -232,11 +202,12 @@ static void BM_AcquireMutex(benchmark::State& state) { gpr_mu_unlock(&mu); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_AcquireMutex); static void BM_TryAcquireMutex(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; // for comparison with the combiner stuff below gpr_mu mu; gpr_mu_init(&mu); @@ -250,11 +221,12 @@ static void BM_TryAcquireMutex(benchmark::State& state) { } } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_TryAcquireMutex); static void BM_AcquireSpinlock(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; // for comparison with the combiner stuff below gpr_spinlock mu = GPR_SPINLOCK_INITIALIZER; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -264,11 +236,12 @@ static void BM_AcquireSpinlock(benchmark::State& state) { gpr_spinlock_unlock(&mu); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_AcquireSpinlock); static void BM_TryAcquireSpinlock(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; // for comparison with the combiner stuff below gpr_spinlock mu = GPR_SPINLOCK_INITIALIZER; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -281,11 +254,12 @@ static void BM_TryAcquireSpinlock(benchmark::State& state) { } } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_TryAcquireSpinlock); static void BM_ClosureSchedOnCombiner(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c; grpc_closure_init(&c, DoNothing, NULL, @@ -297,11 +271,12 @@ static void BM_ClosureSchedOnCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSchedOnCombiner); static void BM_ClosureSched2OnCombiner(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c1; grpc_closure c2; @@ -317,11 +292,12 @@ static void BM_ClosureSched2OnCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched2OnCombiner); static void BM_ClosureSched3OnCombiner(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c1; grpc_closure c2; @@ -341,11 +317,12 @@ static void BM_ClosureSched3OnCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched3OnCombiner); static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner1 = grpc_combiner_create(NULL); grpc_combiner* combiner2 = grpc_combiner_create(NULL); grpc_closure c1; @@ -363,11 +340,12 @@ static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) { GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished"); GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched2OnTwoCombiners); static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner1 = grpc_combiner_create(NULL); grpc_combiner* combiner2 = grpc_combiner_create(NULL); grpc_closure c1; @@ -393,6 +371,7 @@ static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) { GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished"); GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched4OnTwoCombiners); @@ -428,16 +407,17 @@ class Rescheduler { }; static void BM_ClosureReschedOnExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; Rescheduler r(state, grpc_schedule_on_exec_ctx); r.ScheduleFirst(&exec_ctx); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureReschedOnExecCtx); static void BM_ClosureReschedOnCombiner(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_combiner* combiner = grpc_combiner_create(NULL); Rescheduler r(state, grpc_combiner_scheduler(combiner, false)); @@ -445,11 +425,12 @@ static void BM_ClosureReschedOnCombiner(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureReschedOnCombiner); static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_combiner* combiner = grpc_combiner_create(NULL); Rescheduler r(state, grpc_combiner_finally_scheduler(combiner, false)); @@ -458,6 +439,7 @@ static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureReschedOnCombinerFinally); diff --git a/test/cpp/microbenchmarks/bm_cq.cc b/test/cpp/microbenchmarks/bm_cq.cc index c017474bf4a..c433adb7967 100644 --- a/test/cpp/microbenchmarks/bm_cq.cc +++ b/test/cpp/microbenchmarks/bm_cq.cc @@ -38,6 +38,7 @@ #include #include +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" extern "C" { @@ -47,27 +48,21 @@ extern "C" { namespace grpc { namespace testing { -static class InitializeStuff { - public: - InitializeStuff() { init_lib_.init(); } - ~InitializeStuff() { init_lib_.shutdown(); } - - private: - internal::GrpcLibrary init_lib_; - internal::GrpcLibraryInitializer init_; -} initialize_stuff; - static void BM_CreateDestroyCpp(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { CompletionQueue cq; } + track_counters.Finish(state); } BENCHMARK(BM_CreateDestroyCpp); static void BM_CreateDestroyCore(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { grpc_completion_queue_destroy(grpc_completion_queue_create(NULL)); } + track_counters.Finish(state); } BENCHMARK(BM_CreateDestroyCore); @@ -80,6 +75,7 @@ class DummyTag final : public CompletionQueueTag { }; static void BM_Pass1Cpp(benchmark::State& state) { + TrackCounters track_counters; CompletionQueue cq; grpc_completion_queue* c_cq = cq.cq(); while (state.KeepRunning()) { @@ -94,10 +90,12 @@ static void BM_Pass1Cpp(benchmark::State& state) { bool ok; cq.Next(&tag, &ok); } + track_counters.Finish(state); } BENCHMARK(BM_Pass1Cpp); static void BM_Pass1Core(benchmark::State& state) { + TrackCounters track_counters; grpc_completion_queue* cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); while (state.KeepRunning()) { @@ -110,10 +108,12 @@ static void BM_Pass1Core(benchmark::State& state) { grpc_completion_queue_next(cq, deadline, NULL); } grpc_completion_queue_destroy(cq); + track_counters.Finish(state); } BENCHMARK(BM_Pass1Core); static void BM_Pluck1Core(benchmark::State& state) { + TrackCounters track_counters; grpc_completion_queue* cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); while (state.KeepRunning()) { @@ -126,16 +126,19 @@ static void BM_Pluck1Core(benchmark::State& state) { grpc_completion_queue_pluck(cq, NULL, deadline, NULL); } grpc_completion_queue_destroy(cq); + track_counters.Finish(state); } BENCHMARK(BM_Pluck1Core); static void BM_EmptyCore(benchmark::State& state) { + TrackCounters track_counters; grpc_completion_queue* cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); while (state.KeepRunning()) { grpc_completion_queue_next(cq, deadline, NULL); } grpc_completion_queue_destroy(cq); + track_counters.Finish(state); } BENCHMARK(BM_EmptyCore); diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc index 8a4b86f281a..964cd4c5dab 100644 --- a/test/cpp/microbenchmarks/bm_error.cc +++ b/test/cpp/microbenchmarks/bm_error.cc @@ -40,6 +40,7 @@ extern "C" { #include "src/core/lib/transport/error_utils.h" } +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" class ErrorDeleter { @@ -49,31 +50,38 @@ class ErrorDeleter { typedef std::unique_ptr ErrorPtr; static void BM_ErrorCreate(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { GRPC_ERROR_UNREF(GRPC_ERROR_CREATE("Error")); } + track_counters.Finish(state); } BENCHMARK(BM_ErrorCreate); static void BM_ErrorCreateAndSetStatus(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { GRPC_ERROR_UNREF(grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_ABORTED)); } + track_counters.Finish(state); } BENCHMARK(BM_ErrorCreateAndSetStatus); static void BM_ErrorRefUnref(benchmark::State& state) { + TrackCounters track_counters; grpc_error* error = GRPC_ERROR_CREATE("Error"); while (state.KeepRunning()) { GRPC_ERROR_UNREF(GRPC_ERROR_REF(error)); } GRPC_ERROR_UNREF(error); + track_counters.Finish(state); } BENCHMARK(BM_ErrorRefUnref); static void BM_ErrorUnrefNone(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { GRPC_ERROR_UNREF(GRPC_ERROR_NONE); } @@ -81,30 +89,36 @@ static void BM_ErrorUnrefNone(benchmark::State& state) { BENCHMARK(BM_ErrorUnrefNone); static void BM_ErrorGetIntFromNoError(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { intptr_t value; grpc_error_get_int(GRPC_ERROR_NONE, GRPC_ERROR_INT_GRPC_STATUS, &value); } + track_counters.Finish(state); } BENCHMARK(BM_ErrorGetIntFromNoError); static void BM_ErrorGetMissingInt(benchmark::State& state) { + TrackCounters track_counters; ErrorPtr error( grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_INDEX, 1)); while (state.KeepRunning()) { intptr_t value; grpc_error_get_int(error.get(), GRPC_ERROR_INT_OFFSET, &value); } + track_counters.Finish(state); } BENCHMARK(BM_ErrorGetMissingInt); static void BM_ErrorGetPresentInt(benchmark::State& state) { + TrackCounters track_counters; ErrorPtr error( grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_OFFSET, 1)); while (state.KeepRunning()) { intptr_t value; grpc_error_get_int(error.get(), GRPC_ERROR_INT_OFFSET, &value); } + track_counters.Finish(state); } BENCHMARK(BM_ErrorGetPresentInt); @@ -177,22 +191,27 @@ class ErrorWithNestedGrpcStatus { template static void BM_ErrorStringOnNewError(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { Fixture fixture; grpc_error_string(fixture.error()); } + track_counters.Finish(state); } template static void BM_ErrorStringRepeatedly(benchmark::State& state) { + TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_error_string(fixture.error()); } + track_counters.Finish(state); } template static void BM_ErrorGetStatus(benchmark::State& state) { + TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_status_code status; @@ -200,34 +219,41 @@ static void BM_ErrorGetStatus(benchmark::State& state) { grpc_error_get_status(fixture.error(), fixture.deadline(), &status, &msg, NULL); } + track_counters.Finish(state); } template static void BM_ErrorGetStatusCode(benchmark::State& state) { + TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_status_code status; grpc_error_get_status(fixture.error(), fixture.deadline(), &status, NULL, NULL); } + track_counters.Finish(state); } template static void BM_ErrorHttpError(benchmark::State& state) { + TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_http2_error_code error; grpc_error_get_status(fixture.error(), fixture.deadline(), NULL, NULL, &error); } + track_counters.Finish(state); } template static void BM_HasClearGrpcStatus(benchmark::State& state) { + TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_error_has_clear_grpc_status(fixture.error()); } + track_counters.Finish(state); } #define BENCHMARK_SUITE(fixture) \ diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index 48e131f1be0..d43109ab33f 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -37,7 +37,6 @@ #include #include -#include #include #include #include @@ -46,7 +45,6 @@ extern "C" { #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" -#include "src/core/ext/transport/chttp2/transport/internal.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/endpoint_pair.h" @@ -55,35 +53,20 @@ extern "C" { #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/completion_queue.h" #include "src/core/lib/surface/server.h" -#include "test/core/util/memory_counters.h" #include "test/core/util/passthru_endpoint.h" #include "test/core/util/port.h" -#include "test/core/util/trickle_endpoint.h" } #include "src/core/lib/profiling/timers.h" #include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" namespace grpc { namespace testing { -static class InitializeStuff { - public: - InitializeStuff() { - grpc_memory_counters_init(); - init_lib_.init(); - rq_ = grpc_resource_quota_create("bm"); - } - - ~InitializeStuff() { init_lib_.shutdown(); } - - grpc_resource_quota* rq() { return rq_; } - - private: - internal::GrpcLibrary init_lib_; - grpc_resource_quota* rq_; -} initialize_stuff; +// force library initialization +auto& force_library_initialization = Library::get(); /******************************************************************************* * FIXTURES @@ -99,54 +82,7 @@ static void ApplyCommonChannelArguments(ChannelArguments* c) { c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX); } -#ifdef GPR_LOW_LEVEL_COUNTERS -extern "C" gpr_atm gpr_mu_locks; -extern "C" gpr_atm gpr_counter_atm_cas; -extern "C" gpr_atm gpr_counter_atm_add; -#endif - -class BaseFixture { - public: - void Finish(benchmark::State& s) { - std::ostringstream out; - this->AddToLabel(out, s); -#ifdef GPR_LOW_LEVEL_COUNTERS - out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - - mu_locks_at_start_) / - (double)s.iterations()) - << " atm_cas/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) - - atm_cas_at_start_) / - (double)s.iterations()) - << " atm_add/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - - atm_add_at_start_) / - (double)s.iterations()); -#endif - grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); - out << " allocs/iter:" - << ((double)(counters_at_end.total_allocs_absolute - - counters_at_start_.total_allocs_absolute) / - (double)s.iterations()); - auto label = out.str(); - if (label.length() && label[0] == ' ') { - label = label.substr(1); - } - s.SetLabel(label); - } - - virtual void AddToLabel(std::ostream& out, benchmark::State& s) = 0; - - private: -#ifdef GPR_LOW_LEVEL_COUNTERS - const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks); - const size_t atm_cas_at_start_ = - gpr_atm_no_barrier_load(&gpr_counter_atm_cas); - const size_t atm_add_at_start_ = - gpr_atm_no_barrier_load(&gpr_counter_atm_add); -#endif - grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot(); -}; +class BaseFixture : public TrackCounters {}; class FullstackFixture : public BaseFixture { public: @@ -184,8 +120,6 @@ class TCP : public FullstackFixture { public: TCP(Service* service) : FullstackFixture(service, MakeAddress()) {} - void AddToLabel(std::ostream& out, benchmark::State& state) {} - private: static grpc::string MakeAddress() { int port = grpc_pick_unused_port_or_die(); @@ -199,8 +133,6 @@ class UDS : public FullstackFixture { public: UDS(Service* service) : FullstackFixture(service, MakeAddress()) {} - void AddToLabel(std::ostream& out, benchmark::State& state) override {} - private: static grpc::string MakeAddress() { int port = grpc_pick_unused_port_or_die(); // just for a unique id - not a @@ -213,8 +145,7 @@ class UDS : public FullstackFixture { class EndpointPairFixture : public BaseFixture { public: - EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) - : endpoint_pair_(endpoints) { + EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) { ServerBuilder b; cq_ = b.AddCompletionQueue(true); b.RegisterService(service); @@ -227,7 +158,7 @@ class EndpointPairFixture : public BaseFixture { { const grpc_channel_args* server_args = grpc_server_get_channel_args(server_->c_server()); - server_transport_ = grpc_create_chttp2_transport( + grpc_transport* transport = grpc_create_chttp2_transport( &exec_ctx, server_args, endpoints.server, 0 /* is_client */); grpc_pollset** pollsets; @@ -238,9 +169,9 @@ class EndpointPairFixture : public BaseFixture { grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]); } - grpc_server_setup_transport(&exec_ctx, server_->c_server(), - server_transport_, NULL, server_args); - grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL); + grpc_server_setup_transport(&exec_ctx, server_->c_server(), transport, + NULL, server_args); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); } /* create channel */ @@ -250,13 +181,12 @@ class EndpointPairFixture : public BaseFixture { ApplyCommonChannelArguments(&args); grpc_channel_args c_args = args.c_channel_args(); - client_transport_ = + grpc_transport* transport = grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1); - GPR_ASSERT(client_transport_); - grpc_channel* channel = - grpc_channel_create(&exec_ctx, "target", &c_args, - GRPC_CLIENT_DIRECT_CHANNEL, client_transport_); - grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL); + GPR_ASSERT(transport); + grpc_channel* channel = grpc_channel_create( + &exec_ctx, "target", &c_args, GRPC_CLIENT_DIRECT_CHANNEL, transport); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); channel_ = CreateChannelInternal("", channel); } @@ -276,11 +206,6 @@ class EndpointPairFixture : public BaseFixture { ServerCompletionQueue* cq() { return cq_.get(); } std::shared_ptr channel() { return channel_; } - protected: - grpc_endpoint_pair endpoint_pair_; - grpc_transport* client_transport_; - grpc_transport* server_transport_; - private: std::unique_ptr server_; std::unique_ptr cq_; @@ -291,10 +216,7 @@ class SockPair : public EndpointPairFixture { public: SockPair(Service* service) : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair( - "test", initialize_stuff.rq(), 8192)) { - } - - void AddToLabel(std::ostream& out, benchmark::State& state) {} + "test", Library::get().rq(), 8192)) {} }; class InProcessCHTTP2 : public EndpointPairFixture { @@ -303,6 +225,7 @@ class InProcessCHTTP2 : public EndpointPairFixture { : EndpointPairFixture(service, MakeEndpoints()) {} void AddToLabel(std::ostream& out, benchmark::State& state) { + EndpointPairFixture::AddToLabel(out, state); out << " writes/iter:" << ((double)stats_.num_writes / (double)state.iterations()); } @@ -312,79 +235,10 @@ class InProcessCHTTP2 : public EndpointPairFixture { grpc_endpoint_pair MakeEndpoints() { grpc_endpoint_pair p; - grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(), - &stats_); - return p; - } -}; - -class TrickledCHTTP2 : public EndpointPairFixture { - public: - TrickledCHTTP2(Service* service, size_t megabits_per_second) - : EndpointPairFixture(service, MakeEndpoints(megabits_per_second)) {} - - void AddToLabel(std::ostream& out, benchmark::State& state) { - out << " writes/iter:" - << ((double)stats_.num_writes / (double)state.iterations()) - << " cli_transport_stalls/iter:" - << ((double) - client_stats_.streams_stalled_due_to_transport_flow_control / - (double)state.iterations()) - << " cli_stream_stalls/iter:" - << ((double)client_stats_.streams_stalled_due_to_stream_flow_control / - (double)state.iterations()) - << " svr_transport_stalls/iter:" - << ((double) - server_stats_.streams_stalled_due_to_transport_flow_control / - (double)state.iterations()) - << " svr_stream_stalls/iter:" - << ((double)server_stats_.streams_stalled_due_to_stream_flow_control / - (double)state.iterations()); - } - - void Step() { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - size_t client_backlog = - grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.client); - size_t server_backlog = - grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server); - grpc_exec_ctx_finish(&exec_ctx); - - UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_, - client_backlog); - UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_, - server_backlog); - } - - private: - grpc_passthru_endpoint_stats stats_; - struct Stats { - int streams_stalled_due_to_stream_flow_control = 0; - int streams_stalled_due_to_transport_flow_control = 0; - }; - Stats client_stats_; - Stats server_stats_; - - grpc_endpoint_pair MakeEndpoints(size_t kilobits) { - grpc_endpoint_pair p; - grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(), + grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(), &stats_); - double bytes_per_second = 125.0 * kilobits; - p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second); - p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second); return p; } - - void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) { - if (backlog == 0) { - if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) { - s->streams_stalled_due_to_stream_flow_control++; - } - if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) { - s->streams_stalled_due_to_transport_flow_control++; - } - } - } }; /******************************************************************************* @@ -712,7 +566,6 @@ static void BM_StreamingPingPongMsgs(benchmark::State& state) { } while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); request_rw->Write(send_request, tag(0)); // Start client send response_rw.Read(&recv_request, tag(1)); // Start server recv request_rw->Read(&recv_response, tag(2)); // Start client recv @@ -870,81 +723,6 @@ static void BM_PumpStreamServerToClient(benchmark::State& state) { state.SetBytesProcessed(state.range(0) * state.iterations()); } -static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) { - while (true) { - switch (fixture->cq()->AsyncNext( - t, ok, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_micros(100, GPR_TIMESPAN)))) { - case CompletionQueue::TIMEOUT: - fixture->Step(); - break; - case CompletionQueue::SHUTDOWN: - GPR_ASSERT(false); - break; - case CompletionQueue::GOT_EVENT: - return; - } - } -} - -static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture( - new TrickledCHTTP2(&service, state.range(1))); - { - EchoResponse send_response; - EchoResponse recv_response; - if (state.range(0) > 0) { - send_response.set_message(std::string(state.range(0), 'a')); - } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - TrickleCQNext(fixture.get(), &t, &ok); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - request_rw->Read(&recv_response, tag(0)); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - response_rw.Write(send_response, tag(1)); - while (true) { - TrickleCQNext(fixture.get(), &t, &ok); - if (t == tag(0)) { - request_rw->Read(&recv_response, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); - } - } - } - response_rw.Finish(Status::OK, tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - TrickleCQNext(fixture.get(), &t, &ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); -} - /******************************************************************************* * CONFIGURATIONS */ @@ -1034,19 +812,6 @@ BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) ->Range(0, 128 * 1024 * 1024); -static void TrickleArgs(benchmark::internal::Benchmark* b) { - for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { - for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) { - double expected_time = - static_cast(14 + i) / (125.0 * static_cast(j)); - if (expected_time > 0.01) continue; - b->Args({i, j}); - } - } -} - -BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs); - // Generate Args for StreamingPingPong benchmarks. Currently generates args for // only "small streams" (i.e streams with 0, 1 or 2 messages) static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) { diff --git a/test/cpp/microbenchmarks/bm_metadata.cc b/test/cpp/microbenchmarks/bm_metadata.cc index 7f81fbabcc1..f3b2190af69 100644 --- a/test/cpp/microbenchmarks/bm_metadata.cc +++ b/test/cpp/microbenchmarks/bm_metadata.cc @@ -40,61 +40,69 @@ extern "C" { #include "src/core/lib/transport/static_metadata.h" } +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -static class InitializeStuff { - public: - InitializeStuff() { grpc_init(); } - ~InitializeStuff() { grpc_shutdown(); } -} initialize_stuff; - static void BM_SliceFromStatic(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { benchmark::DoNotOptimize(grpc_slice_from_static_string("abc")); } + track_counters.Finish(state); } BENCHMARK(BM_SliceFromStatic); static void BM_SliceFromCopied(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { grpc_slice_unref(grpc_slice_from_copied_string("abc")); } + track_counters.Finish(state); } BENCHMARK(BM_SliceFromCopied); static void BM_SliceIntern(benchmark::State& state) { + TrackCounters track_counters; gpr_slice slice = grpc_slice_from_static_string("abc"); while (state.KeepRunning()) { grpc_slice_unref(grpc_slice_intern(slice)); } + track_counters.Finish(state); } BENCHMARK(BM_SliceIntern); static void BM_SliceReIntern(benchmark::State& state) { + TrackCounters track_counters; gpr_slice slice = grpc_slice_intern(grpc_slice_from_static_string("abc")); while (state.KeepRunning()) { grpc_slice_unref(grpc_slice_intern(slice)); } grpc_slice_unref(slice); + track_counters.Finish(state); } BENCHMARK(BM_SliceReIntern); static void BM_SliceInternStaticMetadata(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { grpc_slice_intern(GRPC_MDSTR_GZIP); } + track_counters.Finish(state); } BENCHMARK(BM_SliceInternStaticMetadata); static void BM_SliceInternEqualToStaticMetadata(benchmark::State& state) { + TrackCounters track_counters; gpr_slice slice = grpc_slice_from_static_string("gzip"); while (state.KeepRunning()) { grpc_slice_intern(slice); } + track_counters.Finish(state); } BENCHMARK(BM_SliceInternEqualToStaticMetadata); static void BM_MetadataFromNonInternedSlices(benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_from_static_string("key"); gpr_slice v = grpc_slice_from_static_string("value"); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -102,10 +110,12 @@ static void BM_MetadataFromNonInternedSlices(benchmark::State& state) { GRPC_MDELEM_UNREF(&exec_ctx, grpc_mdelem_create(&exec_ctx, k, v, NULL)); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromNonInternedSlices); static void BM_MetadataFromInternedSlices(benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value")); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -115,11 +125,13 @@ static void BM_MetadataFromInternedSlices(benchmark::State& state) { grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); grpc_slice_unref(v); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedSlices); static void BM_MetadataFromInternedSlicesAlreadyInIndex( benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value")); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -131,10 +143,12 @@ static void BM_MetadataFromInternedSlicesAlreadyInIndex( grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); grpc_slice_unref(v); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedSlicesAlreadyInIndex); static void BM_MetadataFromInternedKey(benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_from_static_string("value"); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -143,11 +157,13 @@ static void BM_MetadataFromInternedKey(benchmark::State& state) { } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedKey); static void BM_MetadataFromNonInternedSlicesWithBackingStore( benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_from_static_string("key"); gpr_slice v = grpc_slice_from_static_string("value"); char backing_store[sizeof(grpc_mdelem_data)]; @@ -159,11 +175,13 @@ static void BM_MetadataFromNonInternedSlicesWithBackingStore( reinterpret_cast(backing_store))); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromNonInternedSlicesWithBackingStore); static void BM_MetadataFromInternedSlicesWithBackingStore( benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value")); char backing_store[sizeof(grpc_mdelem_data)]; @@ -177,11 +195,13 @@ static void BM_MetadataFromInternedSlicesWithBackingStore( grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); grpc_slice_unref(v); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedSlicesWithBackingStore); static void BM_MetadataFromInternedKeyWithBackingStore( benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_from_static_string("value"); char backing_store[sizeof(grpc_mdelem_data)]; @@ -194,10 +214,12 @@ static void BM_MetadataFromInternedKeyWithBackingStore( } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedKeyWithBackingStore); static void BM_MetadataFromStaticMetadataStrings(benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = GRPC_MDSTR_STATUS; gpr_slice v = GRPC_MDSTR_200; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -206,11 +228,13 @@ static void BM_MetadataFromStaticMetadataStrings(benchmark::State& state) { } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromStaticMetadataStrings); static void BM_MetadataFromStaticMetadataStringsNotIndexed( benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = GRPC_MDSTR_STATUS; gpr_slice v = GRPC_MDSTR_GZIP; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -219,10 +243,12 @@ static void BM_MetadataFromStaticMetadataStringsNotIndexed( } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromStaticMetadataStringsNotIndexed); static void BM_MetadataRefUnrefExternal(benchmark::State& state) { + TrackCounters track_counters; char backing_store[sizeof(grpc_mdelem_data)]; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_mdelem el = @@ -234,10 +260,12 @@ static void BM_MetadataRefUnrefExternal(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefExternal); static void BM_MetadataRefUnrefInterned(benchmark::State& state) { + TrackCounters track_counters; char backing_store[sizeof(grpc_mdelem_data)]; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); @@ -251,10 +279,12 @@ static void BM_MetadataRefUnrefInterned(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefInterned); static void BM_MetadataRefUnrefAllocated(benchmark::State& state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_mdelem el = grpc_mdelem_create(&exec_ctx, grpc_slice_from_static_string("a"), @@ -264,10 +294,12 @@ static void BM_MetadataRefUnrefAllocated(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefAllocated); static void BM_MetadataRefUnrefStatic(benchmark::State& state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_mdelem el = grpc_mdelem_create(&exec_ctx, GRPC_MDSTR_STATUS, GRPC_MDSTR_200, NULL); @@ -276,6 +308,7 @@ static void BM_MetadataRefUnrefStatic(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefStatic); diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h new file mode 100644 index 00000000000..ea9024dce59 --- /dev/null +++ b/test/cpp/microbenchmarks/helpers.h @@ -0,0 +1,119 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef TEST_CPP_MICROBENCHMARKS_COUNTERS_H +#define TEST_CPP_MICROBENCHMARKS_COUNTERS_H + +#include + +extern "C" { +#include +#include "test/core/util/memory_counters.h" +} + +#include +#include "third_party/benchmark/include/benchmark/benchmark.h" + +class Library { + public: + static Library& get() { + static Library lib; + return lib; + } + + grpc_resource_quota* rq() { return rq_; } + + private: + Library() { + grpc_memory_counters_init(); + init_lib_.init(); + rq_ = grpc_resource_quota_create("bm"); + } + + ~Library() { init_lib_.shutdown(); } + + grpc::internal::GrpcLibrary init_lib_; + grpc_resource_quota* rq_; +}; + +#ifdef GPR_LOW_LEVEL_COUNTERS +extern "C" gpr_atm gpr_mu_locks; +extern "C" gpr_atm gpr_counter_atm_cas; +extern "C" gpr_atm gpr_counter_atm_add; +#endif + +class TrackCounters { + public: + virtual void Finish(benchmark::State& state) { + std::ostringstream out; + AddToLabel(out, state); + auto label = out.str(); + if (label.length() && label[0] == ' ') { + label = label.substr(1); + } + state.SetLabel(label); + } + + virtual void AddToLabel(std::ostream& out, benchmark::State& state) { +#ifdef GPR_LOW_LEVEL_COUNTERS + out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - + mu_locks_at_start_) / + (double)state.iterations()) + << " atm_cas/iter:" + << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) - + atm_cas_at_start_) / + (double)state.iterations()) + << " atm_add/iter:" + << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - + atm_add_at_start_) / + (double)state.iterations()); +#endif + grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); + out << " allocs/iter:" + << ((double)(counters_at_end.total_allocs_absolute - + counters_at_start_.total_allocs_absolute) / + (double)state.iterations()); + } + + private: +#ifdef GPR_LOW_LEVEL_COUNTERS + const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks); + const size_t atm_cas_at_start_ = + gpr_atm_no_barrier_load(&gpr_counter_atm_cas); + const size_t atm_add_at_start_ = + gpr_atm_no_barrier_load(&gpr_counter_atm_add); +#endif + grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot(); +}; + +#endif diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index ba48994c8e4..0da73ad4b33 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2386,6 +2386,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2406,6 +2407,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2426,6 +2428,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2446,6 +2449,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2466,6 +2470,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2486,6 +2491,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2504,6 +2510,7 @@ "gpr", "gpr_test_util", "grpc", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -5707,6 +5714,24 @@ "third_party": false, "type": "lib" }, + { + "deps": [ + "benchmark", + "grpc", + "grpc++" + ], + "headers": [ + "test/cpp/microbenchmarks/helpers.h" + ], + "is_filegroup": false, + "language": "c++", + "name": "grpc_benchmark", + "src": [ + "test/cpp/microbenchmarks/helpers.h" + ], + "third_party": false, + "type": "lib" + }, { "deps": [ "grpc++", diff --git a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj new file mode 100644 index 00000000000..e487427d4d7 --- /dev/null +++ b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj @@ -0,0 +1,201 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {31FCED31-7D88-BE3D-2D61-0840F08E0850} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + grpc_benchmark + static + Debug + static + Debug + + + grpc_benchmark + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Windows + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Windows + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Windows + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Windows + true + false + true + true + + + + + + + + + + + + + {07978586-E47C-8709-A63E-895FBF3C3C7D} + + + {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters new file mode 100644 index 00000000000..6c8407f6213 --- /dev/null +++ b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\cpp\microbenchmarks + + + + + + {46d1162d-13b8-d144-8b76-77a6d982a9f1} + + + {1d2b47d7-8fc3-a5b6-cc85-47e31600e9d7} + + + {2a1ac913-6c7b-fbd2-6e8f-1215e92b4af8} + + + + From e4a24195971fe2f909a76b210efcb2aee6ed0575 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 08:53:52 -0800 Subject: [PATCH 026/188] Bugfixes --- test/cpp/microbenchmarks/bm_call_create.cc | 2 ++ test/cpp/microbenchmarks/bm_chttp2_hpack.cc | 2 ++ test/cpp/microbenchmarks/bm_closure.cc | 4 +--- test/cpp/microbenchmarks/bm_cq.cc | 2 ++ test/cpp/microbenchmarks/bm_error.cc | 2 ++ test/cpp/microbenchmarks/bm_metadata.cc | 2 ++ 6 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index 8fed12ef4ee..f25bcd2df82 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -58,6 +58,8 @@ extern "C" { #include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" +auto &force_library_initialization = Library::get(); + class BaseChannelFixture { public: BaseChannelFixture(grpc_channel *channel) : channel_(channel) {} diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc index 029a89db3a1..563db758f71 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc @@ -45,6 +45,8 @@ extern "C" { #include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" +auto &force_library_initialization = Library::get(); + //////////////////////////////////////////////////////////////////////////////// // HPACK encoder // diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc index 8111c9c9fa3..28a385b6c11 100644 --- a/test/cpp/microbenchmarks/bm_closure.cc +++ b/test/cpp/microbenchmarks/bm_closure.cc @@ -45,9 +45,7 @@ extern "C" { #include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -#ifdef GPR_LOW_LEVEL_COUNTERS -extern "C" gpr_atm gpr_mu_locks; -#endif +auto& force_library_initialization = Library::get(); static void BM_NoOpExecCtx(benchmark::State& state) { TrackCounters track_counters; diff --git a/test/cpp/microbenchmarks/bm_cq.cc b/test/cpp/microbenchmarks/bm_cq.cc index c433adb7967..91e6a851017 100644 --- a/test/cpp/microbenchmarks/bm_cq.cc +++ b/test/cpp/microbenchmarks/bm_cq.cc @@ -48,6 +48,8 @@ extern "C" { namespace grpc { namespace testing { +auto& force_library_initialization = Library::get(); + static void BM_CreateDestroyCpp(benchmark::State& state) { TrackCounters track_counters; while (state.KeepRunning()) { diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc index 964cd4c5dab..95f19e75860 100644 --- a/test/cpp/microbenchmarks/bm_error.cc +++ b/test/cpp/microbenchmarks/bm_error.cc @@ -43,6 +43,8 @@ extern "C" { #include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" +auto& force_library_initialization = Library::get(); + class ErrorDeleter { public: void operator()(grpc_error* error) { GRPC_ERROR_UNREF(error); } diff --git a/test/cpp/microbenchmarks/bm_metadata.cc b/test/cpp/microbenchmarks/bm_metadata.cc index f3b2190af69..07c55f42e90 100644 --- a/test/cpp/microbenchmarks/bm_metadata.cc +++ b/test/cpp/microbenchmarks/bm_metadata.cc @@ -43,6 +43,8 @@ extern "C" { #include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" +auto& force_library_initialization = Library::get(); + static void BM_SliceFromStatic(benchmark::State& state) { TrackCounters track_counters; while (state.KeepRunning()) { From 627a59892aba5c360a1ecde2ef2aad3aa4b8a796 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 09:47:21 -0800 Subject: [PATCH 027/188] Prep for splitting bm_fullstack --- build.yaml | 2 + test/cpp/microbenchmarks/bm_fullstack.cc | 299 +----------------- .../fullstack_context_mutators.h | 158 +++++++++ test/cpp/microbenchmarks/fullstack_fixtures.h | 236 ++++++++++++++ .../generated/sources_and_headers.json | 4 + .../grpc_benchmark/grpc_benchmark.vcxproj | 2 + .../grpc_benchmark.vcxproj.filters | 6 + 7 files changed, 410 insertions(+), 297 deletions(-) create mode 100644 test/cpp/microbenchmarks/fullstack_context_mutators.h create mode 100644 test/cpp/microbenchmarks/fullstack_fixtures.h diff --git a/build.yaml b/build.yaml index dc0ba108072..eccfedcea7a 100644 --- a/build.yaml +++ b/build.yaml @@ -1218,6 +1218,8 @@ libs: build: test language: c++ headers: + - test/cpp/microbenchmarks/fullstack_context_mutators.h + - test/cpp/microbenchmarks/fullstack_fixtures.h - test/cpp/microbenchmarks/helpers.h deps: - benchmark diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index d43109ab33f..b60d29b6075 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -35,31 +35,11 @@ #include -#include -#include -#include -#include -#include -#include -#include - -extern "C" { -#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/iomgr/endpoint.h" -#include "src/core/lib/iomgr/endpoint_pair.h" -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/iomgr/tcp_posix.h" -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/surface/completion_queue.h" -#include "src/core/lib/surface/server.h" -#include "test/core/util/passthru_endpoint.h" -#include "test/core/util/port.h" -} #include "src/core/lib/profiling/timers.h" #include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" -#include "test/cpp/microbenchmarks/helpers.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" +#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" #include "third_party/benchmark/include/benchmark/benchmark.h" namespace grpc { @@ -68,281 +48,6 @@ namespace testing { // force library initialization auto& force_library_initialization = Library::get(); -/******************************************************************************* - * FIXTURES - */ - -static void ApplyCommonServerBuilderConfig(ServerBuilder* b) { - b->SetMaxReceiveMessageSize(INT_MAX); - b->SetMaxSendMessageSize(INT_MAX); -} - -static void ApplyCommonChannelArguments(ChannelArguments* c) { - c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX); - c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX); -} - -class BaseFixture : public TrackCounters {}; - -class FullstackFixture : public BaseFixture { - public: - FullstackFixture(Service* service, const grpc::string& address) { - ServerBuilder b; - b.AddListeningPort(address, InsecureServerCredentials()); - cq_ = b.AddCompletionQueue(true); - b.RegisterService(service); - ApplyCommonServerBuilderConfig(&b); - server_ = b.BuildAndStart(); - ChannelArguments args; - ApplyCommonChannelArguments(&args); - channel_ = CreateCustomChannel(address, InsecureChannelCredentials(), args); - } - - virtual ~FullstackFixture() { - server_->Shutdown(); - cq_->Shutdown(); - void* tag; - bool ok; - while (cq_->Next(&tag, &ok)) { - } - } - - ServerCompletionQueue* cq() { return cq_.get(); } - std::shared_ptr channel() { return channel_; } - - private: - std::unique_ptr server_; - std::unique_ptr cq_; - std::shared_ptr channel_; -}; - -class TCP : public FullstackFixture { - public: - TCP(Service* service) : FullstackFixture(service, MakeAddress()) {} - - private: - static grpc::string MakeAddress() { - int port = grpc_pick_unused_port_or_die(); - std::stringstream addr; - addr << "localhost:" << port; - return addr.str(); - } -}; - -class UDS : public FullstackFixture { - public: - UDS(Service* service) : FullstackFixture(service, MakeAddress()) {} - - private: - static grpc::string MakeAddress() { - int port = grpc_pick_unused_port_or_die(); // just for a unique id - not a - // real port - std::stringstream addr; - addr << "unix:/tmp/bm_fullstack." << port; - return addr.str(); - } -}; - -class EndpointPairFixture : public BaseFixture { - public: - EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) { - ServerBuilder b; - cq_ = b.AddCompletionQueue(true); - b.RegisterService(service); - ApplyCommonServerBuilderConfig(&b); - server_ = b.BuildAndStart(); - - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - - /* add server endpoint to server_ */ - { - const grpc_channel_args* server_args = - grpc_server_get_channel_args(server_->c_server()); - grpc_transport* transport = grpc_create_chttp2_transport( - &exec_ctx, server_args, endpoints.server, 0 /* is_client */); - - grpc_pollset** pollsets; - size_t num_pollsets = 0; - grpc_server_get_pollsets(server_->c_server(), &pollsets, &num_pollsets); - - for (size_t i = 0; i < num_pollsets; i++) { - grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]); - } - - grpc_server_setup_transport(&exec_ctx, server_->c_server(), transport, - NULL, server_args); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); - } - - /* create channel */ - { - ChannelArguments args; - args.SetString(GRPC_ARG_DEFAULT_AUTHORITY, "test.authority"); - ApplyCommonChannelArguments(&args); - - grpc_channel_args c_args = args.c_channel_args(); - grpc_transport* transport = - grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1); - GPR_ASSERT(transport); - grpc_channel* channel = grpc_channel_create( - &exec_ctx, "target", &c_args, GRPC_CLIENT_DIRECT_CHANNEL, transport); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); - - channel_ = CreateChannelInternal("", channel); - } - - grpc_exec_ctx_finish(&exec_ctx); - } - - virtual ~EndpointPairFixture() { - server_->Shutdown(); - cq_->Shutdown(); - void* tag; - bool ok; - while (cq_->Next(&tag, &ok)) { - } - } - - ServerCompletionQueue* cq() { return cq_.get(); } - std::shared_ptr channel() { return channel_; } - - private: - std::unique_ptr server_; - std::unique_ptr cq_; - std::shared_ptr channel_; -}; - -class SockPair : public EndpointPairFixture { - public: - SockPair(Service* service) - : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair( - "test", Library::get().rq(), 8192)) {} -}; - -class InProcessCHTTP2 : public EndpointPairFixture { - public: - InProcessCHTTP2(Service* service) - : EndpointPairFixture(service, MakeEndpoints()) {} - - void AddToLabel(std::ostream& out, benchmark::State& state) { - EndpointPairFixture::AddToLabel(out, state); - out << " writes/iter:" - << ((double)stats_.num_writes / (double)state.iterations()); - } - - private: - grpc_passthru_endpoint_stats stats_; - - grpc_endpoint_pair MakeEndpoints() { - grpc_endpoint_pair p; - grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(), - &stats_); - return p; - } -}; - -/******************************************************************************* - * CONTEXT MUTATORS - */ - -static const int kPregenerateKeyCount = 100000; - -template -auto MakeVector(size_t length, F f) -> std::vector { - std::vector out; - out.reserve(length); - for (size_t i = 0; i < length; i++) { - out.push_back(f()); - } - return out; -} - -class NoOpMutator { - public: - template - NoOpMutator(ContextType* context) {} -}; - -template -class RandomBinaryMetadata { - public: - static const grpc::string& Key() { return kKey; } - - static const grpc::string& Value() { - return kValues[rand() % kValues.size()]; - } - - private: - static const grpc::string kKey; - static const std::vector kValues; - - static grpc::string GenerateOneString() { - grpc::string s; - s.reserve(length + 1); - for (int i = 0; i < length; i++) { - s += (char)rand(); - } - return s; - } -}; - -template -const grpc::string RandomBinaryMetadata::kKey = "foo-bin"; - -template -const std::vector RandomBinaryMetadata::kValues = - MakeVector(kPregenerateKeyCount, GenerateOneString); - -template -class RandomAsciiMetadata { - public: - static const grpc::string& Key() { return kKey; } - - static const grpc::string& Value() { - return kValues[rand() % kValues.size()]; - } - - private: - static const grpc::string kKey; - static const std::vector kValues; - - static grpc::string GenerateOneString() { - grpc::string s; - s.reserve(length + 1); - for (int i = 0; i < length; i++) { - s += (char)(rand() % 26 + 'a'); - } - return s; - } -}; - -template -const grpc::string RandomAsciiMetadata::kKey = "foo"; - -template -const std::vector RandomAsciiMetadata::kValues = - MakeVector(kPregenerateKeyCount, GenerateOneString); - -template -class Client_AddMetadata : public NoOpMutator { - public: - Client_AddMetadata(ClientContext* context) : NoOpMutator(context) { - for (int i = 0; i < kNumKeys; i++) { - context->AddMetadata(Generator::Key(), Generator::Value()); - } - } -}; - -template -class Server_AddInitialMetadata : public NoOpMutator { - public: - Server_AddInitialMetadata(ServerContext* context) : NoOpMutator(context) { - for (int i = 0; i < kNumKeys; i++) { - context->AddInitialMetadata(Generator::Key(), Generator::Value()); - } - } -}; - /******************************************************************************* * BENCHMARKING KERNELS */ diff --git a/test/cpp/microbenchmarks/fullstack_context_mutators.h b/test/cpp/microbenchmarks/fullstack_context_mutators.h new file mode 100644 index 00000000000..676f9aa1cc6 --- /dev/null +++ b/test/cpp/microbenchmarks/fullstack_context_mutators.h @@ -0,0 +1,158 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef TEST_CPP_MICROBENCHMARKS_FULLSTACK_CONTEXT_MUTATORS_H +#define TEST_CPP_MICROBENCHMARKS_FULLSTACK_CONTEXT_MUTATORS_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test/cpp/microbenchmarks/helpers.h" + +namespace grpc { +namespace testing { + +/******************************************************************************* + * CONTEXT MUTATORS + */ + +static const int kPregenerateKeyCount = 100000; + +template +auto MakeVector(size_t length, F f) -> std::vector { + std::vector out; + out.reserve(length); + for (size_t i = 0; i < length; i++) { + out.push_back(f()); + } + return out; +} + +class NoOpMutator { + public: + template + NoOpMutator(ContextType* context) {} +}; + +template +class RandomBinaryMetadata { + public: + static const grpc::string& Key() { return kKey; } + + static const grpc::string& Value() { + return kValues[rand() % kValues.size()]; + } + + private: + static const grpc::string kKey; + static const std::vector kValues; + + static grpc::string GenerateOneString() { + grpc::string s; + s.reserve(length + 1); + for (int i = 0; i < length; i++) { + s += (char)rand(); + } + return s; + } +}; + +template +class RandomAsciiMetadata { + public: + static const grpc::string& Key() { return kKey; } + + static const grpc::string& Value() { + return kValues[rand() % kValues.size()]; + } + + private: + static const grpc::string kKey; + static const std::vector kValues; + + static grpc::string GenerateOneString() { + grpc::string s; + s.reserve(length + 1); + for (int i = 0; i < length; i++) { + s += (char)(rand() % 26 + 'a'); + } + return s; + } +}; + +template +class Client_AddMetadata : public NoOpMutator { + public: + Client_AddMetadata(ClientContext* context) : NoOpMutator(context) { + for (int i = 0; i < kNumKeys; i++) { + context->AddMetadata(Generator::Key(), Generator::Value()); + } + } +}; + +template +class Server_AddInitialMetadata : public NoOpMutator { + public: + Server_AddInitialMetadata(ServerContext* context) : NoOpMutator(context) { + for (int i = 0; i < kNumKeys; i++) { + context->AddInitialMetadata(Generator::Key(), Generator::Value()); + } + } +}; + +// static initialization + +template +const grpc::string RandomBinaryMetadata::kKey = "foo-bin"; + +template +const std::vector RandomBinaryMetadata::kValues = + MakeVector(kPregenerateKeyCount, GenerateOneString); + +template +const grpc::string RandomAsciiMetadata::kKey = "foo"; + +template +const std::vector RandomAsciiMetadata::kValues = + MakeVector(kPregenerateKeyCount, GenerateOneString); + +} // namespace testing +} // namespace grpc + +#endif diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h new file mode 100644 index 00000000000..62ba288bbb7 --- /dev/null +++ b/test/cpp/microbenchmarks/fullstack_fixtures.h @@ -0,0 +1,236 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H +#define TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H + +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/endpoint.h" +#include "src/core/lib/iomgr/endpoint_pair.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/iomgr/tcp_posix.h" +#include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/completion_queue.h" +#include "src/core/lib/surface/server.h" +#include "test/core/util/passthru_endpoint.h" +#include "test/core/util/port.h" +} + +#include "test/cpp/microbenchmarks/helpers.h" + +namespace grpc { +namespace testing { + +static void ApplyCommonServerBuilderConfig(ServerBuilder* b) { + b->SetMaxReceiveMessageSize(INT_MAX); + b->SetMaxSendMessageSize(INT_MAX); +} + +static void ApplyCommonChannelArguments(ChannelArguments* c) { + c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX); + c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX); +} + +class BaseFixture : public TrackCounters {}; + +class FullstackFixture : public BaseFixture { + public: + FullstackFixture(Service* service, const grpc::string& address) { + ServerBuilder b; + b.AddListeningPort(address, InsecureServerCredentials()); + cq_ = b.AddCompletionQueue(true); + b.RegisterService(service); + ApplyCommonServerBuilderConfig(&b); + server_ = b.BuildAndStart(); + ChannelArguments args; + ApplyCommonChannelArguments(&args); + channel_ = CreateCustomChannel(address, InsecureChannelCredentials(), args); + } + + virtual ~FullstackFixture() { + server_->Shutdown(); + cq_->Shutdown(); + void* tag; + bool ok; + while (cq_->Next(&tag, &ok)) { + } + } + + ServerCompletionQueue* cq() { return cq_.get(); } + std::shared_ptr channel() { return channel_; } + + private: + std::unique_ptr server_; + std::unique_ptr cq_; + std::shared_ptr channel_; +}; + +class TCP : public FullstackFixture { + public: + TCP(Service* service) : FullstackFixture(service, MakeAddress()) {} + + private: + static grpc::string MakeAddress() { + int port = grpc_pick_unused_port_or_die(); + std::stringstream addr; + addr << "localhost:" << port; + return addr.str(); + } +}; + +class UDS : public FullstackFixture { + public: + UDS(Service* service) : FullstackFixture(service, MakeAddress()) {} + + private: + static grpc::string MakeAddress() { + int port = grpc_pick_unused_port_or_die(); // just for a unique id - not a + // real port + std::stringstream addr; + addr << "unix:/tmp/bm_fullstack." << port; + return addr.str(); + } +}; + +class EndpointPairFixture : public BaseFixture { + public: + EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) { + ServerBuilder b; + cq_ = b.AddCompletionQueue(true); + b.RegisterService(service); + ApplyCommonServerBuilderConfig(&b); + server_ = b.BuildAndStart(); + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + + /* add server endpoint to server_ */ + { + const grpc_channel_args* server_args = + grpc_server_get_channel_args(server_->c_server()); + grpc_transport* transport = grpc_create_chttp2_transport( + &exec_ctx, server_args, endpoints.server, 0 /* is_client */); + + grpc_pollset** pollsets; + size_t num_pollsets = 0; + grpc_server_get_pollsets(server_->c_server(), &pollsets, &num_pollsets); + + for (size_t i = 0; i < num_pollsets; i++) { + grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]); + } + + grpc_server_setup_transport(&exec_ctx, server_->c_server(), transport, + NULL, server_args); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); + } + + /* create channel */ + { + ChannelArguments args; + args.SetString(GRPC_ARG_DEFAULT_AUTHORITY, "test.authority"); + ApplyCommonChannelArguments(&args); + + grpc_channel_args c_args = args.c_channel_args(); + grpc_transport* transport = + grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1); + GPR_ASSERT(transport); + grpc_channel* channel = grpc_channel_create( + &exec_ctx, "target", &c_args, GRPC_CLIENT_DIRECT_CHANNEL, transport); + grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); + + channel_ = CreateChannelInternal("", channel); + } + + grpc_exec_ctx_finish(&exec_ctx); + } + + virtual ~EndpointPairFixture() { + server_->Shutdown(); + cq_->Shutdown(); + void* tag; + bool ok; + while (cq_->Next(&tag, &ok)) { + } + } + + ServerCompletionQueue* cq() { return cq_.get(); } + std::shared_ptr channel() { return channel_; } + + private: + std::unique_ptr server_; + std::unique_ptr cq_; + std::shared_ptr channel_; +}; + +class SockPair : public EndpointPairFixture { + public: + SockPair(Service* service) + : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair( + "test", Library::get().rq(), 8192)) {} +}; + +class InProcessCHTTP2 : public EndpointPairFixture { + public: + InProcessCHTTP2(Service* service) + : EndpointPairFixture(service, MakeEndpoints()) {} + + void AddToLabel(std::ostream& out, benchmark::State& state) { + EndpointPairFixture::AddToLabel(out, state); + out << " writes/iter:" + << ((double)stats_.num_writes / (double)state.iterations()); + } + + private: + grpc_passthru_endpoint_stats stats_; + + grpc_endpoint_pair MakeEndpoints() { + grpc_endpoint_pair p; + grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(), + &stats_); + return p; + } +}; + +} // namespace testing +} // namespace grpc + +#endif diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 0da73ad4b33..7251ea3e15d 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -5721,12 +5721,16 @@ "grpc++" ], "headers": [ + "test/cpp/microbenchmarks/fullstack_context_mutators.h", + "test/cpp/microbenchmarks/fullstack_fixtures.h", "test/cpp/microbenchmarks/helpers.h" ], "is_filegroup": false, "language": "c++", "name": "grpc_benchmark", "src": [ + "test/cpp/microbenchmarks/fullstack_context_mutators.h", + "test/cpp/microbenchmarks/fullstack_fixtures.h", "test/cpp/microbenchmarks/helpers.h" ], "third_party": false, diff --git a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj index e487427d4d7..62e10bf1777 100644 --- a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj +++ b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj @@ -160,6 +160,8 @@ + + diff --git a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters index 6c8407f6213..3e003edd5ee 100644 --- a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters +++ b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters @@ -1,6 +1,12 @@ + + test\cpp\microbenchmarks + + + test\cpp\microbenchmarks + test\cpp\microbenchmarks From 62ac65b1833673365011cf861aa243112530c331 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 09:57:43 -0800 Subject: [PATCH 028/188] Split bm_fullstack unary ping pong out --- CMakeLists.txt | 42 ++++ Makefile | 48 ++++ build.yaml | 24 ++ test/cpp/microbenchmarks/bm_fullstack.cc | 141 ------------ .../bm_fullstack_unary_ping_pong.cc | 205 ++++++++++++++++++ .../generated/sources_and_headers.json | 21 ++ tools/run_tests/generated/tests.json | 27 +++ 7 files changed, 367 insertions(+), 141 deletions(-) create mode 100644 test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 494300e9704..465d47ebc38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -589,6 +589,9 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_fullstack) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_fullstack_unary_ping_pong) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_metadata) endif() add_dependencies(buildtests_cxx channel_arguments_test) @@ -7758,6 +7761,45 @@ endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_executable(bm_fullstack_unary_ping_pong + test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_fullstack_unary_ping_pong + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_fullstack_unary_ping_pong + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark + benchmark + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + add_executable(bm_metadata test/cpp/microbenchmarks/bm_metadata.cc third_party/googletest/src/gtest-all.cc diff --git a/Makefile b/Makefile index 9b872a39b25..8eec00cfbe6 100644 --- a/Makefile +++ b/Makefile @@ -1050,6 +1050,7 @@ bm_closure: $(BINDIR)/$(CONFIG)/bm_closure bm_cq: $(BINDIR)/$(CONFIG)/bm_cq bm_error: $(BINDIR)/$(CONFIG)/bm_error bm_fullstack: $(BINDIR)/$(CONFIG)/bm_fullstack +bm_fullstack_unary_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong bm_metadata: $(BINDIR)/$(CONFIG)/bm_metadata channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test channel_filter_test: $(BINDIR)/$(CONFIG)/channel_filter_test @@ -1464,6 +1465,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_cq \ $(BINDIR)/$(CONFIG)/bm_error \ $(BINDIR)/$(CONFIG)/bm_fullstack \ + $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ @@ -1575,6 +1577,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_cq \ $(BINDIR)/$(CONFIG)/bm_error \ $(BINDIR)/$(CONFIG)/bm_fullstack \ + $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ @@ -1911,6 +1914,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/bm_error || ( echo test bm_error failed ; exit 1 ) $(E) "[RUN] Testing bm_fullstack" $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack || ( echo test bm_fullstack failed ; exit 1 ) + $(E) "[RUN] Testing bm_fullstack_unary_ping_pong" + $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong || ( echo test bm_fullstack_unary_ping_pong failed ; exit 1 ) $(E) "[RUN] Testing bm_metadata" $(Q) $(BINDIR)/$(CONFIG)/bm_metadata || ( echo test bm_metadata failed ; exit 1 ) $(E) "[RUN] Testing channel_arguments_test" @@ -12762,6 +12767,49 @@ endif endif +BM_FULLSTACK_UNARY_PING_PONG_SRC = \ + test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc \ + +BM_FULLSTACK_UNARY_PING_PONG_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_UNARY_PING_PONG_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong: $(PROTOBUF_DEP) $(BM_FULLSTACK_UNARY_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_UNARY_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_fullstack_unary_ping_pong: $(BM_FULLSTACK_UNARY_PING_PONG_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_FULLSTACK_UNARY_PING_PONG_OBJS:.o=.dep) +endif +endif + + BM_METADATA_SRC = \ test/cpp/microbenchmarks/bm_metadata.cc \ diff --git a/build.yaml b/build.yaml index eccfedcea7a..aa927459740 100644 --- a/build.yaml +++ b/build.yaml @@ -3140,6 +3140,30 @@ targets: - linux - posix timeout_seconds: 1200 +- name: bm_fullstack_unary_ping_pong + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc + deps: + - grpc_benchmark + - benchmark + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + excluded_poll_engines: + - poll + - poll-cv + platforms: + - mac + - linux + - posix + timeout_seconds: 1200 - name: bm_metadata build: test language: c++ diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index b60d29b6075..5908b4d11ac 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -54,79 +54,6 @@ auto& force_library_initialization = Library::get(); static void* tag(intptr_t x) { return reinterpret_cast(x); } -template -static void BM_UnaryPingPong(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - EchoRequest send_request; - EchoResponse send_response; - EchoResponse recv_response; - if (state.range(0) > 0) { - send_request.set_message(std::string(state.range(0), 'a')); - } - if (state.range(1) > 0) { - send_response.set_message(std::string(state.range(1), 'a')); - } - Status recv_status; - struct ServerEnv { - ServerContext ctx; - EchoRequest recv_request; - grpc::ServerAsyncResponseWriter response_writer; - ServerEnv() : response_writer(&ctx) {} - }; - uint8_t server_env_buffer[2 * sizeof(ServerEnv)]; - ServerEnv* server_env[2] = { - reinterpret_cast(server_env_buffer), - reinterpret_cast(server_env_buffer + sizeof(ServerEnv))}; - new (server_env[0]) ServerEnv; - new (server_env[1]) ServerEnv; - service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request, - &server_env[0]->response_writer, fixture->cq(), - fixture->cq(), tag(0)); - service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request, - &server_env[1]->response_writer, fixture->cq(), - fixture->cq(), tag(1)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - recv_response.Clear(); - ClientContext cli_ctx; - ClientContextMutator cli_ctx_mut(&cli_ctx); - std::unique_ptr> response_reader( - stub->AsyncEcho(&cli_ctx, send_request, fixture->cq())); - void* t; - bool ok; - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - GPR_ASSERT(t == tag(0) || t == tag(1)); - intptr_t slot = reinterpret_cast(t); - ServerEnv* senv = server_env[slot]; - ServerContextMutator svr_ctx_mut(&senv->ctx); - senv->response_writer.Finish(send_response, Status::OK, tag(3)); - response_reader->Finish(&recv_response, &recv_status, tag(4)); - for (int i = (1 << 3) | (1 << 4); i != 0;) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int tagnum = (int)reinterpret_cast(t); - GPR_ASSERT(i & (1 << tagnum)); - i -= 1 << tagnum; - } - GPR_ASSERT(recv_status.ok()); - - senv->~ServerEnv(); - senv = new (senv) ServerEnv(); - service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer, - fixture->cq(), fixture->cq(), tag(slot)); - } - fixture->Finish(state); - fixture.reset(); - server_env[0]->~ServerEnv(); - server_env[1]->~ServerEnv(); - state.SetBytesProcessed(state.range(0) * state.iterations() + - state.range(1) * state.iterations()); -} - // Repeatedly makes Streaming Bidi calls (exchanging a configurable number of // messages in each call) in a loop on a single channel // @@ -432,74 +359,6 @@ static void BM_PumpStreamServerToClient(benchmark::State& state) { * CONFIGURATIONS */ -static void SweepSizesArgs(benchmark::internal::Benchmark* b) { - b->Args({0, 0}); - for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { - b->Args({i, 0}); - b->Args({0, i}); - b->Args({i, i}); - } -} - -BENCHMARK_TEMPLATE(BM_UnaryPingPong, TCP, NoOpMutator, NoOpMutator) - ->Apply(SweepSizesArgs); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, UDS, NoOpMutator, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, SockPair, NoOpMutator, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, NoOpMutator) - ->Apply(SweepSizesArgs); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, - NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 2>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 2>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 2>, - NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 100>) - ->Args({0, 0}); - BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP) ->Range(0, 128 * 1024 * 1024); BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) diff --git a/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc new file mode 100644 index 00000000000..ae26a080eb0 --- /dev/null +++ b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc @@ -0,0 +1,205 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Benchmark gRPC end2end in various configurations */ + +#include + +#include "src/core/lib/profiling/timers.h" +#include "src/cpp/client/create_channel_internal.h" +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" +#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +namespace grpc { +namespace testing { + +// force library initialization +auto& force_library_initialization = Library::get(); + +/******************************************************************************* + * BENCHMARKING KERNELS + */ + +static void* tag(intptr_t x) { return reinterpret_cast(x); } + +template +static void BM_UnaryPingPong(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + EchoRequest send_request; + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_request.set_message(std::string(state.range(0), 'a')); + } + if (state.range(1) > 0) { + send_response.set_message(std::string(state.range(1), 'a')); + } + Status recv_status; + struct ServerEnv { + ServerContext ctx; + EchoRequest recv_request; + grpc::ServerAsyncResponseWriter response_writer; + ServerEnv() : response_writer(&ctx) {} + }; + uint8_t server_env_buffer[2 * sizeof(ServerEnv)]; + ServerEnv* server_env[2] = { + reinterpret_cast(server_env_buffer), + reinterpret_cast(server_env_buffer + sizeof(ServerEnv))}; + new (server_env[0]) ServerEnv; + new (server_env[1]) ServerEnv; + service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request, + &server_env[0]->response_writer, fixture->cq(), + fixture->cq(), tag(0)); + service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request, + &server_env[1]->response_writer, fixture->cq(), + fixture->cq(), tag(1)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + recv_response.Clear(); + ClientContext cli_ctx; + ClientContextMutator cli_ctx_mut(&cli_ctx); + std::unique_ptr> response_reader( + stub->AsyncEcho(&cli_ctx, send_request, fixture->cq())); + void* t; + bool ok; + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + GPR_ASSERT(t == tag(0) || t == tag(1)); + intptr_t slot = reinterpret_cast(t); + ServerEnv* senv = server_env[slot]; + ServerContextMutator svr_ctx_mut(&senv->ctx); + senv->response_writer.Finish(send_response, Status::OK, tag(3)); + response_reader->Finish(&recv_response, &recv_status, tag(4)); + for (int i = (1 << 3) | (1 << 4); i != 0;) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int tagnum = (int)reinterpret_cast(t); + GPR_ASSERT(i & (1 << tagnum)); + i -= 1 << tagnum; + } + GPR_ASSERT(recv_status.ok()); + + senv->~ServerEnv(); + senv = new (senv) ServerEnv(); + service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer, + fixture->cq(), fixture->cq(), tag(slot)); + } + fixture->Finish(state); + fixture.reset(); + server_env[0]->~ServerEnv(); + server_env[1]->~ServerEnv(); + state.SetBytesProcessed(state.range(0) * state.iterations() + + state.range(1) * state.iterations()); +} + +/******************************************************************************* + * CONFIGURATIONS + */ + +static void SweepSizesArgs(benchmark::internal::Benchmark* b) { + b->Args({0, 0}); + for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { + b->Args({i, 0}); + b->Args({0, i}); + b->Args({i, i}); + } +} + +BENCHMARK_TEMPLATE(BM_UnaryPingPong, TCP, NoOpMutator, NoOpMutator) + ->Apply(SweepSizesArgs); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, UDS, NoOpMutator, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, SockPair, NoOpMutator, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, NoOpMutator) + ->Apply(SweepSizesArgs); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, + NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 2>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 2>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 2>, + NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 100>) + ->Args({0, 0}); + +} // namespace testing +} // namespace grpc + +BENCHMARK_MAIN(); diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 7251ea3e15d..a5ee8a61cb2 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2504,6 +2504,27 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_fullstack_unary_ping_pong", + "src": [ + "test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "benchmark", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index ab7938d0ca1..e01ee5975b7 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2676,6 +2676,33 @@ ], "timeout_seconds": 1200 }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "excluded_poll_engines": [ + "poll", + "poll-cv" + ], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_fullstack_unary_ping_pong", + "platforms": [ + "linux", + "mac", + "posix" + ], + "timeout_seconds": 1200 + }, { "args": [ "--benchmark_min_time=0" From 12d22e472beecc5307e2dff0b7255c2ba9b4f879 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 10:14:20 -0800 Subject: [PATCH 029/188] Further splitting --- CMakeLists.txt | 52 ++- Makefile | 78 +++- build.yaml | 28 +- test/cpp/microbenchmarks/bm_fullstack.cc | 408 ------------------ .../bm_fullstack_streaming_ping_pong.cc | 197 +++++++++ .../bm_fullstack_streaming_pump.cc | 197 +++++++++ .../generated/sources_and_headers.json | 25 +- tools/run_tests/generated/tests.json | 29 +- 8 files changed, 581 insertions(+), 433 deletions(-) delete mode 100644 test/cpp/microbenchmarks/bm_fullstack.cc create mode 100644 test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc create mode 100644 test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 465d47ebc38..4d85fd5516e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -586,7 +586,10 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_error) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) -add_dependencies(buildtests_cxx bm_fullstack) +add_dependencies(buildtests_cxx bm_fullstack_streaming_ping_pong) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_fullstack_streaming_pump) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_fullstack_unary_ping_pong) @@ -7722,13 +7725,52 @@ endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) -add_executable(bm_fullstack - test/cpp/microbenchmarks/bm_fullstack.cc +add_executable(bm_fullstack_streaming_ping_pong + test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_fullstack_streaming_ping_pong + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_fullstack_streaming_ping_pong + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark + benchmark + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(bm_fullstack_streaming_pump + test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc third_party/googletest/src/gtest-all.cc ) -target_include_directories(bm_fullstack +target_include_directories(bm_fullstack_streaming_pump PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${BORINGSSL_ROOT_DIR}/include @@ -7742,7 +7784,7 @@ target_include_directories(bm_fullstack PRIVATE ${_gRPC_PROTO_GENS_DIR} ) -target_link_libraries(bm_fullstack +target_link_libraries(bm_fullstack_streaming_pump ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} grpc_benchmark diff --git a/Makefile b/Makefile index 8eec00cfbe6..1187c8de065 100644 --- a/Makefile +++ b/Makefile @@ -1049,7 +1049,8 @@ bm_chttp2_hpack: $(BINDIR)/$(CONFIG)/bm_chttp2_hpack bm_closure: $(BINDIR)/$(CONFIG)/bm_closure bm_cq: $(BINDIR)/$(CONFIG)/bm_cq bm_error: $(BINDIR)/$(CONFIG)/bm_error -bm_fullstack: $(BINDIR)/$(CONFIG)/bm_fullstack +bm_fullstack_streaming_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong +bm_fullstack_streaming_pump: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump bm_fullstack_unary_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong bm_metadata: $(BINDIR)/$(CONFIG)/bm_metadata channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test @@ -1464,7 +1465,8 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_closure \ $(BINDIR)/$(CONFIG)/bm_cq \ $(BINDIR)/$(CONFIG)/bm_error \ - $(BINDIR)/$(CONFIG)/bm_fullstack \ + $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong \ + $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump \ $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ @@ -1576,7 +1578,8 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_closure \ $(BINDIR)/$(CONFIG)/bm_cq \ $(BINDIR)/$(CONFIG)/bm_error \ - $(BINDIR)/$(CONFIG)/bm_fullstack \ + $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong \ + $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump \ $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ @@ -1912,8 +1915,10 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/bm_cq || ( echo test bm_cq failed ; exit 1 ) $(E) "[RUN] Testing bm_error" $(Q) $(BINDIR)/$(CONFIG)/bm_error || ( echo test bm_error failed ; exit 1 ) - $(E) "[RUN] Testing bm_fullstack" - $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack || ( echo test bm_fullstack failed ; exit 1 ) + $(E) "[RUN] Testing bm_fullstack_streaming_ping_pong" + $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong || ( echo test bm_fullstack_streaming_ping_pong failed ; exit 1 ) + $(E) "[RUN] Testing bm_fullstack_streaming_pump" + $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump || ( echo test bm_fullstack_streaming_pump failed ; exit 1 ) $(E) "[RUN] Testing bm_fullstack_unary_ping_pong" $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong || ( echo test bm_fullstack_unary_ping_pong failed ; exit 1 ) $(E) "[RUN] Testing bm_metadata" @@ -12724,15 +12729,15 @@ endif endif -BM_FULLSTACK_SRC = \ - test/cpp/microbenchmarks/bm_fullstack.cc \ +BM_FULLSTACK_STREAMING_PING_PONG_SRC = \ + test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc \ -BM_FULLSTACK_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_SRC)))) +BM_FULLSTACK_STREAMING_PING_PONG_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_STREAMING_PING_PONG_SRC)))) ifeq ($(NO_SECURE),true) # You can't build secure targets if you don't have OpenSSL. -$(BINDIR)/$(CONFIG)/bm_fullstack: openssl_dep_error +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong: openssl_dep_error else @@ -12743,26 +12748,69 @@ ifeq ($(NO_PROTOBUF),true) # You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. -$(BINDIR)/$(CONFIG)/bm_fullstack: protobuf_dep_error +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_fullstack: $(PROTOBUF_DEP) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong: $(PROTOBUF_DEP) $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack + $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -deps_bm_fullstack: $(BM_FULLSTACK_OBJS:.o=.dep) +deps_bm_fullstack_streaming_ping_pong: $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS:.o=.dep) ifneq ($(NO_SECURE),true) ifneq ($(NO_DEPS),true) --include $(BM_FULLSTACK_OBJS:.o=.dep) +-include $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS:.o=.dep) +endif +endif + + +BM_FULLSTACK_STREAMING_PUMP_SRC = \ + test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc \ + +BM_FULLSTACK_STREAMING_PUMP_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_STREAMING_PUMP_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump: $(PROTOBUF_DEP) $(BM_FULLSTACK_STREAMING_PUMP_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_STREAMING_PUMP_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_fullstack_streaming_pump: $(BM_FULLSTACK_STREAMING_PUMP_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_FULLSTACK_STREAMING_PUMP_OBJS:.o=.dep) endif endif diff --git a/build.yaml b/build.yaml index aa927459740..8cc8c6721d8 100644 --- a/build.yaml +++ b/build.yaml @@ -3116,11 +3116,35 @@ targets: - mac - linux - posix -- name: bm_fullstack +- name: bm_fullstack_streaming_ping_pong build: test language: c++ src: - - test/cpp/microbenchmarks/bm_fullstack.cc + - test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc + deps: + - grpc_benchmark + - benchmark + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + excluded_poll_engines: + - poll + - poll-cv + platforms: + - mac + - linux + - posix + timeout_seconds: 1200 +- name: bm_fullstack_streaming_pump + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc deps: - grpc_benchmark - benchmark diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc deleted file mode 100644 index 5908b4d11ac..00000000000 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ /dev/null @@ -1,408 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* Benchmark gRPC end2end in various configurations */ - -#include - -#include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" -#include "src/proto/grpc/testing/echo.grpc.pb.h" -#include "test/cpp/microbenchmarks/fullstack_fixtures.h" -#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" -#include "third_party/benchmark/include/benchmark/benchmark.h" - -namespace grpc { -namespace testing { - -// force library initialization -auto& force_library_initialization = Library::get(); - -/******************************************************************************* - * BENCHMARKING KERNELS - */ - -static void* tag(intptr_t x) { return reinterpret_cast(x); } - -// Repeatedly makes Streaming Bidi calls (exchanging a configurable number of -// messages in each call) in a loop on a single channel -// -// First parmeter (i.e state.range(0)): Message size (in bytes) to use -// Second parameter (i.e state.range(1)): Number of ping pong messages. -// Note: One ping-pong means two messages (one from client to server and -// the other from server to client): -template -static void BM_StreamingPingPong(benchmark::State& state) { - const int msg_size = state.range(0); - const int max_ping_pongs = state.range(1); - - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoResponse send_response; - EchoResponse recv_response; - EchoRequest send_request; - EchoRequest recv_request; - - if (msg_size > 0) { - send_request.set_message(std::string(msg_size, 'a')); - send_response.set_message(std::string(msg_size, 'b')); - } - - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - - while (state.KeepRunning()) { - ServerContext svr_ctx; - ServerContextMutator svr_ctx_mut(&svr_ctx); - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - - ClientContext cli_ctx; - ClientContextMutator cli_ctx_mut(&cli_ctx); - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - - // Establish async stream between client side and server side - void* t; - bool ok; - int need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - - // Send 'max_ping_pongs' number of ping pong messages - int ping_pong_cnt = 0; - while (ping_pong_cnt < max_ping_pongs) { - request_rw->Write(send_request, tag(0)); // Start client send - response_rw.Read(&recv_request, tag(1)); // Start server recv - request_rw->Read(&recv_response, tag(2)); // Start client recv - - need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - - // If server recv is complete, start the server send operation - if (i == 1) { - response_rw.Write(send_response, tag(3)); - } - - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - - ping_pong_cnt++; - } - - request_rw->WritesDone(tag(0)); - response_rw.Finish(Status::OK, tag(1)); - - Status recv_status; - request_rw->Finish(&recv_status, tag(2)); - - need_tags = (1 << 0) | (1 << 1) | (1 << 2); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - - GPR_ASSERT(recv_status.ok()); - } - } - - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(msg_size * state.iterations() * max_ping_pongs * 2); -} - -// Repeatedly sends ping pong messages in a single streaming Bidi call in a loop -// First parmeter (i.e state.range(0)): Message size (in bytes) to use -template -static void BM_StreamingPingPongMsgs(benchmark::State& state) { - const int msg_size = state.range(0); - - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoResponse send_response; - EchoResponse recv_response; - EchoRequest send_request; - EchoRequest recv_request; - - if (msg_size > 0) { - send_request.set_message(std::string(msg_size, 'a')); - send_response.set_message(std::string(msg_size, 'b')); - } - - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - - ServerContext svr_ctx; - ServerContextMutator svr_ctx_mut(&svr_ctx); - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - - ClientContext cli_ctx; - ClientContextMutator cli_ctx_mut(&cli_ctx); - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - - // Establish async stream between client side and server side - void* t; - bool ok; - int need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - - while (state.KeepRunning()) { - request_rw->Write(send_request, tag(0)); // Start client send - response_rw.Read(&recv_request, tag(1)); // Start server recv - request_rw->Read(&recv_response, tag(2)); // Start client recv - - need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - - // If server recv is complete, start the server send operation - if (i == 1) { - response_rw.Write(send_response, tag(3)); - } - - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - - request_rw->WritesDone(tag(0)); - response_rw.Finish(Status::OK, tag(1)); - Status recv_status; - request_rw->Finish(&recv_status, tag(2)); - - need_tags = (1 << 0) | (1 << 1) | (1 << 2); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - - GPR_ASSERT(recv_status.ok()); - } - - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(msg_size * state.iterations() * 2); -} - -template -static void BM_PumpStreamClientToServer(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoRequest send_request; - EchoRequest recv_request; - if (state.range(0) > 0) { - send_request.set_message(std::string(state.range(0), 'a')); - } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - response_rw.Read(&recv_request, tag(0)); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - request_rw->Write(send_request, tag(1)); - while (true) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - if (t == tag(0)) { - response_rw.Read(&recv_request, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); - } - } - } - request_rw->WritesDone(tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); -} - -template -static void BM_PumpStreamServerToClient(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoResponse send_response; - EchoResponse recv_response; - if (state.range(0) > 0) { - send_response.set_message(std::string(state.range(0), 'a')); - } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - request_rw->Read(&recv_response, tag(0)); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - response_rw.Write(send_response, tag(1)); - while (true) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - if (t == tag(0)) { - request_rw->Read(&recv_response, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); - } - } - } - response_rw.Finish(Status::OK, tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); -} - -/******************************************************************************* - * CONFIGURATIONS - */ - -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) - ->Range(0, 128 * 1024 * 1024); - -// Generate Args for StreamingPingPong benchmarks. Currently generates args for -// only "small streams" (i.e streams with 0, 1 or 2 messages) -static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) { - int msg_size = 0; - - b->Args({0, 0}); // spl case: 0 ping-pong msgs (msg_size doesn't matter here) - - for (msg_size = 0; msg_size <= 128 * 1024 * 1024; - msg_size == 0 ? msg_size++ : msg_size *= 8) { - b->Args({msg_size, 1}); - b->Args({msg_size, 2}); - } -} - -BENCHMARK_TEMPLATE(BM_StreamingPingPong, InProcessCHTTP2, NoOpMutator, - NoOpMutator) - ->Apply(StreamingPingPongArgs); -BENCHMARK_TEMPLATE(BM_StreamingPingPong, TCP, NoOpMutator, NoOpMutator) - ->Apply(StreamingPingPongArgs); - -BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, InProcessCHTTP2, NoOpMutator, - NoOpMutator) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, TCP, NoOpMutator, NoOpMutator) - ->Range(0, 128 * 1024 * 1024); - -} // namespace testing -} // namespace grpc - -BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc new file mode 100644 index 00000000000..4f2a15e5b98 --- /dev/null +++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc @@ -0,0 +1,197 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Benchmark gRPC end2end in various configurations */ + +#include + +#include "src/core/lib/profiling/timers.h" +#include "src/cpp/client/create_channel_internal.h" +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" +#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +namespace grpc { +namespace testing { + +// force library initialization +auto& force_library_initialization = Library::get(); + +/******************************************************************************* + * BENCHMARKING KERNELS + */ + +static void* tag(intptr_t x) { return reinterpret_cast(x); } + +template +static void BM_PumpStreamClientToServer(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoRequest send_request; + EchoRequest recv_request; + if (state.range(0) > 0) { + send_request.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + response_rw.Read(&recv_request, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + request_rw->Write(send_request, tag(1)); + while (true) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + if (t == tag(0)) { + response_rw.Read(&recv_request, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + request_rw->WritesDone(tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +template +static void BM_PumpStreamServerToClient(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_response.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + request_rw->Read(&recv_response, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + response_rw.Write(send_response, tag(1)); + while (true) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + if (t == tag(0)) { + request_rw->Read(&recv_response, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + response_rw.Finish(Status::OK, tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +/******************************************************************************* + * CONFIGURATIONS + */ + +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) + ->Range(0, 128 * 1024 * 1024); + +} // namespace testing +} // namespace grpc + +BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc new file mode 100644 index 00000000000..4f2a15e5b98 --- /dev/null +++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc @@ -0,0 +1,197 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Benchmark gRPC end2end in various configurations */ + +#include + +#include "src/core/lib/profiling/timers.h" +#include "src/cpp/client/create_channel_internal.h" +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" +#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +namespace grpc { +namespace testing { + +// force library initialization +auto& force_library_initialization = Library::get(); + +/******************************************************************************* + * BENCHMARKING KERNELS + */ + +static void* tag(intptr_t x) { return reinterpret_cast(x); } + +template +static void BM_PumpStreamClientToServer(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoRequest send_request; + EchoRequest recv_request; + if (state.range(0) > 0) { + send_request.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + response_rw.Read(&recv_request, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + request_rw->Write(send_request, tag(1)); + while (true) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + if (t == tag(0)) { + response_rw.Read(&recv_request, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + request_rw->WritesDone(tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +template +static void BM_PumpStreamServerToClient(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_response.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + request_rw->Read(&recv_response, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + response_rw.Write(send_response, tag(1)); + while (true) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + if (t == tag(0)) { + request_rw->Read(&recv_response, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + response_rw.Finish(Status::OK, tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +/******************************************************************************* + * CONFIGURATIONS + */ + +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) + ->Range(0, 128 * 1024 * 1024); + +} // namespace testing +} // namespace grpc + +BENCHMARK_MAIN(); diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index a5ee8a61cb2..fd09ed53360 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2497,9 +2497,30 @@ "headers": [], "is_filegroup": false, "language": "c++", - "name": "bm_fullstack", + "name": "bm_fullstack_streaming_ping_pong", "src": [ - "test/cpp/microbenchmarks/bm_fullstack.cc" + "test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_fullstack_streaming_pump", + "src": [ + "test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc" ], "third_party": false, "type": "target" diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index e01ee5975b7..3d01aedd46b 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2668,7 +2668,34 @@ "flaky": false, "gtest": false, "language": "c++", - "name": "bm_fullstack", + "name": "bm_fullstack_streaming_ping_pong", + "platforms": [ + "linux", + "mac", + "posix" + ], + "timeout_seconds": 1200 + }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "excluded_poll_engines": [ + "poll", + "poll-cv" + ], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_fullstack_streaming_pump", "platforms": [ "linux", "mac", From 52bd441b25879ec1b164d684c3cde76c74200bba Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 10:52:22 -0800 Subject: [PATCH 030/188] Get back trickle --- CMakeLists.txt | 42 ++++++ Makefile | 48 +++++++ build.yaml | 24 ++++ .../microbenchmarks/bm_fullstack_trickle.cc | 130 ++++++++++++++++++ .../generated/sources_and_headers.json | 21 +++ tools/run_tests/generated/tests.json | 27 ++++ 6 files changed, 292 insertions(+) create mode 100644 test/cpp/microbenchmarks/bm_fullstack_trickle.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d85fd5516e..dbc2b4b8bf6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -592,6 +592,9 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_fullstack_streaming_pump) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_fullstack_trickle) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_fullstack_unary_ping_pong) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -7803,6 +7806,45 @@ endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_executable(bm_fullstack_trickle + test/cpp/microbenchmarks/bm_fullstack_trickle.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_fullstack_trickle + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_fullstack_trickle + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark + benchmark + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + add_executable(bm_fullstack_unary_ping_pong test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc third_party/googletest/src/gtest-all.cc diff --git a/Makefile b/Makefile index 1187c8de065..c2d06feedd3 100644 --- a/Makefile +++ b/Makefile @@ -1051,6 +1051,7 @@ bm_cq: $(BINDIR)/$(CONFIG)/bm_cq bm_error: $(BINDIR)/$(CONFIG)/bm_error bm_fullstack_streaming_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump +bm_fullstack_trickle: $(BINDIR)/$(CONFIG)/bm_fullstack_trickle bm_fullstack_unary_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong bm_metadata: $(BINDIR)/$(CONFIG)/bm_metadata channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test @@ -1467,6 +1468,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_error \ $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong \ $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump \ + $(BINDIR)/$(CONFIG)/bm_fullstack_trickle \ $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ @@ -1580,6 +1582,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_error \ $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong \ $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump \ + $(BINDIR)/$(CONFIG)/bm_fullstack_trickle \ $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ @@ -1919,6 +1922,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong || ( echo test bm_fullstack_streaming_ping_pong failed ; exit 1 ) $(E) "[RUN] Testing bm_fullstack_streaming_pump" $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump || ( echo test bm_fullstack_streaming_pump failed ; exit 1 ) + $(E) "[RUN] Testing bm_fullstack_trickle" + $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_trickle || ( echo test bm_fullstack_trickle failed ; exit 1 ) $(E) "[RUN] Testing bm_fullstack_unary_ping_pong" $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong || ( echo test bm_fullstack_unary_ping_pong failed ; exit 1 ) $(E) "[RUN] Testing bm_metadata" @@ -12815,6 +12820,49 @@ endif endif +BM_FULLSTACK_TRICKLE_SRC = \ + test/cpp/microbenchmarks/bm_fullstack_trickle.cc \ + +BM_FULLSTACK_TRICKLE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_TRICKLE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_fullstack_trickle: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_fullstack_trickle: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_fullstack_trickle: $(PROTOBUF_DEP) $(BM_FULLSTACK_TRICKLE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_TRICKLE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_trickle + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_trickle.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_fullstack_trickle: $(BM_FULLSTACK_TRICKLE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_FULLSTACK_TRICKLE_OBJS:.o=.dep) +endif +endif + + BM_FULLSTACK_UNARY_PING_PONG_SRC = \ test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc \ diff --git a/build.yaml b/build.yaml index 8cc8c6721d8..a39ee9f44b6 100644 --- a/build.yaml +++ b/build.yaml @@ -3164,6 +3164,30 @@ targets: - linux - posix timeout_seconds: 1200 +- name: bm_fullstack_trickle + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_fullstack_trickle.cc + deps: + - grpc_benchmark + - benchmark + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + excluded_poll_engines: + - poll + - poll-cv + platforms: + - mac + - linux + - posix + timeout_seconds: 1200 - name: bm_fullstack_unary_ping_pong build: test language: c++ diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc new file mode 100644 index 00000000000..f5443af3190 --- /dev/null +++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc @@ -0,0 +1,130 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Benchmark gRPC end2end in various configurations */ + +namespace grpc { namespace testing{ + +static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) { + while (true) { + switch (fixture->cq()->AsyncNext( + t, ok, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(100, GPR_TIMESPAN)))) { + case CompletionQueue::TIMEOUT: + fixture->Step(); + break; + case CompletionQueue::SHUTDOWN: + GPR_ASSERT(false); + break; + case CompletionQueue::GOT_EVENT: + return; + } + } +} + +static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture( + new TrickledCHTTP2(&service, state.range(1))); + { + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_response.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + TrickleCQNext(fixture.get(), &t, &ok); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + request_rw->Read(&recv_response, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + response_rw.Write(send_response, tag(1)); + while (true) { + TrickleCQNext(fixture.get(), &t, &ok); + if (t == tag(0)) { + request_rw->Read(&recv_response, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + response_rw.Finish(Status::OK, tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + TrickleCQNext(fixture.get(), &t, &ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +/******************************************************************************* + * CONFIGURATIONS + */ + +static void TrickleArgs(benchmark::internal::Benchmark* b) { + for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { + for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) { + double expected_time = + static_cast(14 + i) / (125.0 * static_cast(j)); + if (expected_time > 0.01) continue; + b->Args({i, j}); + } + } +} + +BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs); + +BENCHMARK_MAIN(); diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index fd09ed53360..06fc3799e3f 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2525,6 +2525,27 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_fullstack_trickle", + "src": [ + "test/cpp/microbenchmarks/bm_fullstack_trickle.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "benchmark", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 3d01aedd46b..f24694e3c99 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2703,6 +2703,33 @@ ], "timeout_seconds": 1200 }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "excluded_poll_engines": [ + "poll", + "poll-cv" + ], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_fullstack_trickle", + "platforms": [ + "linux", + "mac", + "posix" + ], + "timeout_seconds": 1200 + }, { "args": [ "--benchmark_min_time=0" From cc928d6b7187d40356e8dbbd6160778493670920 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 11:06:40 -0800 Subject: [PATCH 031/188] Cleanup --- .../bm_fullstack_streaming_ping_pong.cc | 2 +- .../bm_fullstack_streaming_pump.cc | 2 +- .../microbenchmarks/bm_fullstack_trickle.cc | 91 ++++++++++++++++++- .../bm_fullstack_unary_ping_pong.cc | 2 +- test/cpp/microbenchmarks/fullstack_fixtures.h | 30 +++--- 5 files changed, 112 insertions(+), 15 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc index 4f2a15e5b98..dc0e7d769ab 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc @@ -38,8 +38,8 @@ #include "src/core/lib/profiling/timers.h" #include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" -#include "test/cpp/microbenchmarks/fullstack_fixtures.h" #include "test/cpp/microbenchmarks/fullstack_context_mutators.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" #include "third_party/benchmark/include/benchmark/benchmark.h" namespace grpc { diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc index 4f2a15e5b98..dc0e7d769ab 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc @@ -38,8 +38,8 @@ #include "src/core/lib/profiling/timers.h" #include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" -#include "test/cpp/microbenchmarks/fullstack_fixtures.h" #include "test/cpp/microbenchmarks/fullstack_context_mutators.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" #include "third_party/benchmark/include/benchmark/benchmark.h" namespace grpc { diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc index f5443af3190..5011f06368e 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc @@ -33,7 +33,94 @@ /* Benchmark gRPC end2end in various configurations */ -namespace grpc { namespace testing{ +#include "src/core/lib/profiling/timers.h" +#include "src/cpp/client/create_channel_internal.h" +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" +extern "C" { +#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" +#include "src/core/ext/transport/chttp2/transport/internal.h" +#include "test/core/util/trickle_endpoint.h" +} + +namespace grpc { +namespace testing { + +static void* tag(intptr_t x) { return reinterpret_cast(x); } + +class TrickledCHTTP2 : public EndpointPairFixture { + public: + TrickledCHTTP2(Service* service, size_t megabits_per_second) + : EndpointPairFixture(service, MakeEndpoints(megabits_per_second)) {} + + void AddToLabel(std::ostream& out, benchmark::State& state) { + out << " writes/iter:" + << ((double)stats_.num_writes / (double)state.iterations()) + << " cli_transport_stalls/iter:" + << ((double) + client_stats_.streams_stalled_due_to_transport_flow_control / + (double)state.iterations()) + << " cli_stream_stalls/iter:" + << ((double)client_stats_.streams_stalled_due_to_stream_flow_control / + (double)state.iterations()) + << " svr_transport_stalls/iter:" + << ((double) + server_stats_.streams_stalled_due_to_transport_flow_control / + (double)state.iterations()) + << " svr_stream_stalls/iter:" + << ((double)server_stats_.streams_stalled_due_to_stream_flow_control / + (double)state.iterations()); + } + + void Step() { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + size_t client_backlog = + grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.client); + size_t server_backlog = + grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server); + grpc_exec_ctx_finish(&exec_ctx); + + UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_, + client_backlog); + UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_, + server_backlog); + } + + private: + grpc_passthru_endpoint_stats stats_; + struct Stats { + int streams_stalled_due_to_stream_flow_control = 0; + int streams_stalled_due_to_transport_flow_control = 0; + }; + Stats client_stats_; + Stats server_stats_; + + grpc_endpoint_pair MakeEndpoints(size_t kilobits) { + grpc_endpoint_pair p; + grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(), + &stats_); + double bytes_per_second = 125.0 * kilobits; + p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second); + p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second); + return p; + } + + void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) { + if (backlog == 0) { + if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) { + s->streams_stalled_due_to_stream_flow_control++; + } + if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) { + s->streams_stalled_due_to_transport_flow_control++; + } + } + } +}; + +// force library initialization +auto& force_library_initialization = Library::get(); static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) { while (true) { @@ -126,5 +213,7 @@ static void TrickleArgs(benchmark::internal::Benchmark* b) { } BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs); +} +} BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc index ae26a080eb0..e51d272b104 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc @@ -38,8 +38,8 @@ #include "src/core/lib/profiling/timers.h" #include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" -#include "test/cpp/microbenchmarks/fullstack_fixtures.h" #include "test/cpp/microbenchmarks/fullstack_context_mutators.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" #include "third_party/benchmark/include/benchmark/benchmark.h" namespace grpc { diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h index 62ba288bbb7..dc297010599 100644 --- a/test/cpp/microbenchmarks/fullstack_fixtures.h +++ b/test/cpp/microbenchmarks/fullstack_fixtures.h @@ -134,7 +134,8 @@ class UDS : public FullstackFixture { class EndpointPairFixture : public BaseFixture { public: - EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) { + EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) + : endpoint_pair_(endpoints) { ServerBuilder b; cq_ = b.AddCompletionQueue(true); b.RegisterService(service); @@ -143,11 +144,12 @@ class EndpointPairFixture : public BaseFixture { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - /* add server endpoint to server_ */ + /* add server endpoint to server_ + * */ { const grpc_channel_args* server_args = grpc_server_get_channel_args(server_->c_server()); - grpc_transport* transport = grpc_create_chttp2_transport( + server_transport_ = grpc_create_chttp2_transport( &exec_ctx, server_args, endpoints.server, 0 /* is_client */); grpc_pollset** pollsets; @@ -158,9 +160,9 @@ class EndpointPairFixture : public BaseFixture { grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]); } - grpc_server_setup_transport(&exec_ctx, server_->c_server(), transport, - NULL, server_args); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); + grpc_server_setup_transport(&exec_ctx, server_->c_server(), + server_transport_, NULL, server_args); + grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL); } /* create channel */ @@ -170,12 +172,13 @@ class EndpointPairFixture : public BaseFixture { ApplyCommonChannelArguments(&args); grpc_channel_args c_args = args.c_channel_args(); - grpc_transport* transport = + client_transport_ = grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1); - GPR_ASSERT(transport); - grpc_channel* channel = grpc_channel_create( - &exec_ctx, "target", &c_args, GRPC_CLIENT_DIRECT_CHANNEL, transport); - grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL); + GPR_ASSERT(client_transport_); + grpc_channel* channel = + grpc_channel_create(&exec_ctx, "target", &c_args, + GRPC_CLIENT_DIRECT_CHANNEL, client_transport_); + grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL); channel_ = CreateChannelInternal("", channel); } @@ -195,6 +198,11 @@ class EndpointPairFixture : public BaseFixture { ServerCompletionQueue* cq() { return cq_.get(); } std::shared_ptr channel() { return channel_; } + protected: + grpc_endpoint_pair endpoint_pair_; + grpc_transport* client_transport_; + grpc_transport* server_transport_; + private: std::unique_ptr server_; std::unique_ptr cq_; From 0d8b11fe210102e6d8ef30bb27dc05d5729005a1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 11:10:09 -0800 Subject: [PATCH 032/188] Update test list --- tools/run_tests/run_microbenchmark.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 3a9461ecd3b..9153b5c1f6f 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -199,7 +199,10 @@ argp.add_argument('-c', '--collect', default=sorted(collectors.keys()), help='Which collectors should be run against each benchmark') argp.add_argument('-b', '--benchmarks', - default=['bm_fullstack', + default=['bm_fullstack_trickle', + 'bm_fullstack_unary_ping_pong', + 'bm_fullstack_streaming_ping_pong', + 'bm_fullstack_streaming_pump', 'bm_closure', 'bm_cq', 'bm_call_create', From 0dc57ad46f376356c3d4d155e974b0c37e9609df Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 13:15:13 -0800 Subject: [PATCH 033/188] Move trickle to the end: it takes a while, and its less important (in the face of failures) --- tools/run_tests/run_microbenchmark.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 9153b5c1f6f..4fb1d5fc7c2 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -199,8 +199,7 @@ argp.add_argument('-c', '--collect', default=sorted(collectors.keys()), help='Which collectors should be run against each benchmark') argp.add_argument('-b', '--benchmarks', - default=['bm_fullstack_trickle', - 'bm_fullstack_unary_ping_pong', + default=['bm_fullstack_unary_ping_pong', 'bm_fullstack_streaming_ping_pong', 'bm_fullstack_streaming_pump', 'bm_closure', @@ -208,7 +207,9 @@ argp.add_argument('-b', '--benchmarks', 'bm_call_create', 'bm_error', 'bm_chttp2_hpack', - 'bm_metadata'], + 'bm_metadata', + 'bm_fullstack_trickle', + ], nargs='+', type=str, help='Which microbenchmarks should be run') From 1128d462c395d064a620c9e62cf29f152cbce33a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 16:46:54 -0800 Subject: [PATCH 034/188] Pollset benchmark --- CMakeLists.txt | 40 +++++++++++ Makefile | 48 +++++++++++++ build.yaml | 18 +++++ test/cpp/microbenchmarks/bm_metadata.cc | 2 +- test/cpp/microbenchmarks/bm_pollset.cc | 70 +++++++++++++++++++ .../generated/sources_and_headers.json | 19 +++++ tools/run_tests/generated/tests.json | 22 ++++++ 7 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 test/cpp/microbenchmarks/bm_pollset.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index dbc2b4b8bf6..784e15da6be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -600,6 +600,9 @@ endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_metadata) endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_pollset) +endif() add_dependencies(buildtests_cxx channel_arguments_test) add_dependencies(buildtests_cxx channel_filter_test) add_dependencies(buildtests_cxx cli_call_test) @@ -7916,6 +7919,43 @@ target_link_libraries(bm_metadata ${_gRPC_GFLAGS_LIBRARIES} ) +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(bm_pollset + test/cpp/microbenchmarks/bm_pollset.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_pollset + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_pollset + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark + benchmark + grpc_test_util + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) diff --git a/Makefile b/Makefile index c2d06feedd3..727bd64c6e1 100644 --- a/Makefile +++ b/Makefile @@ -1054,6 +1054,7 @@ bm_fullstack_streaming_pump: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump bm_fullstack_trickle: $(BINDIR)/$(CONFIG)/bm_fullstack_trickle bm_fullstack_unary_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong bm_metadata: $(BINDIR)/$(CONFIG)/bm_metadata +bm_pollset: $(BINDIR)/$(CONFIG)/bm_pollset channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test channel_filter_test: $(BINDIR)/$(CONFIG)/channel_filter_test cli_call_test: $(BINDIR)/$(CONFIG)/cli_call_test @@ -1471,6 +1472,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_fullstack_trickle \ $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ + $(BINDIR)/$(CONFIG)/bm_pollset \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ $(BINDIR)/$(CONFIG)/cli_call_test \ @@ -1585,6 +1587,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_fullstack_trickle \ $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ + $(BINDIR)/$(CONFIG)/bm_pollset \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ $(BINDIR)/$(CONFIG)/cli_call_test \ @@ -1928,6 +1931,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong || ( echo test bm_fullstack_unary_ping_pong failed ; exit 1 ) $(E) "[RUN] Testing bm_metadata" $(Q) $(BINDIR)/$(CONFIG)/bm_metadata || ( echo test bm_metadata failed ; exit 1 ) + $(E) "[RUN] Testing bm_pollset" + $(Q) $(BINDIR)/$(CONFIG)/bm_pollset || ( echo test bm_pollset failed ; exit 1 ) $(E) "[RUN] Testing channel_arguments_test" $(Q) $(BINDIR)/$(CONFIG)/channel_arguments_test || ( echo test channel_arguments_test failed ; exit 1 ) $(E) "[RUN] Testing channel_filter_test" @@ -12949,6 +12954,49 @@ endif endif +BM_POLLSET_SRC = \ + test/cpp/microbenchmarks/bm_pollset.cc \ + +BM_POLLSET_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_POLLSET_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_pollset: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_pollset: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_pollset: $(PROTOBUF_DEP) $(BM_POLLSET_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_POLLSET_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_pollset + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_pollset.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_pollset: $(BM_POLLSET_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_POLLSET_OBJS:.o=.dep) +endif +endif + + CHANNEL_ARGUMENTS_TEST_SRC = \ test/cpp/common/channel_arguments_test.cc \ diff --git a/build.yaml b/build.yaml index a39ee9f44b6..dae06e6f93e 100644 --- a/build.yaml +++ b/build.yaml @@ -3230,6 +3230,24 @@ targets: - mac - linux - posix +- name: bm_pollset + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_pollset.cc + deps: + - grpc_benchmark + - benchmark + - grpc_test_util + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + platforms: + - mac + - linux + - posix - name: channel_arguments_test gtest: true build: test diff --git a/test/cpp/microbenchmarks/bm_metadata.cc b/test/cpp/microbenchmarks/bm_metadata.cc index 07c55f42e90..b01a376eb46 100644 --- a/test/cpp/microbenchmarks/bm_metadata.cc +++ b/test/cpp/microbenchmarks/bm_metadata.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc new file mode 100644 index 00000000000..adb49f0b79d --- /dev/null +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -0,0 +1,70 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Test out pollset latencies */ + +#include + +extern "C" { +#include "src/core/lib/iomgr/pollset.h" +} + +#include "test/cpp/microbenchmarks/helpers.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +static void shutdown_ps_closure(grpc_exec_ctx* exec_ctx, void* ps, + grpc_error* error) { + grpc_pollset_destroy(ps); +} + +static void BM_CreateDestroyPollset(benchmark::State& state) { + TrackCounters track_counters; + size_t ps_sz = grpc_pollset_size(); + grpc_pollset* ps = gpr_malloc(ps_sz); + gpr_mu* mu; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_closure shutdown_ps_closure; + grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps, + grpc_schedule_on_exec_ctx); + while (state.KeepRunning()) { + memset(ps, 0, ps_sz); + grpc_pollset_init(ps, &mu); + grpc_pollset_shutdown(&exec_ctx, ps, shutdown_ps_closure); + grpc_exec_ctx_flush(&exec_ctx); + } + grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(&state); +} +BENCHMARK(BM_CreateDestroyPollset); + +BENCHMARK_MAIN(); diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 06fc3799e3f..f8b06bb1725 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2586,6 +2586,25 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_pollset", + "src": [ + "test/cpp/microbenchmarks/bm_pollset.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index f24694e3c99..4b01be8494d 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2779,6 +2779,28 @@ "posix" ] }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_pollset", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [], "ci_platforms": [ From f09ec59222e4abd8a534f89139f6b52378776fc4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 16:52:37 -0800 Subject: [PATCH 035/188] Fix mac build --- CMakeLists.txt | 1 + Makefile | 2 + build.yaml | 2 + test/cpp/microbenchmarks/helpers.cc | 65 +++++++++++++++++++ test/cpp/microbenchmarks/helpers.h | 32 +-------- .../generated/sources_and_headers.json | 1 + .../grpc_benchmark/grpc_benchmark.vcxproj | 2 +- .../grpc_benchmark.vcxproj.filters | 5 ++ 8 files changed, 79 insertions(+), 31 deletions(-) create mode 100644 test/cpp/microbenchmarks/helpers.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index dbc2b4b8bf6..c521b7916ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3106,6 +3106,7 @@ endif() if (gRPC_BUILD_TESTS) add_library(grpc_benchmark + test/cpp/microbenchmarks/helpers.cc ) if(WIN32 AND MSVC) diff --git a/Makefile b/Makefile index c2d06feedd3..61bf4ebf94c 100644 --- a/Makefile +++ b/Makefile @@ -4976,6 +4976,7 @@ endif LIBGRPC_BENCHMARK_SRC = \ + test/cpp/microbenchmarks/helpers.cc \ PUBLIC_HEADERS_CXX += \ @@ -18380,6 +18381,7 @@ test/cpp/interop/interop_client.cc: $(OPENSSL_DEP) test/cpp/interop/interop_server.cc: $(OPENSSL_DEP) test/cpp/interop/interop_server_bootstrap.cc: $(OPENSSL_DEP) test/cpp/interop/server_helper.cc: $(OPENSSL_DEP) +test/cpp/microbenchmarks/helpers.cc: $(OPENSSL_DEP) test/cpp/qps/client_async.cc: $(OPENSSL_DEP) test/cpp/qps/client_sync.cc: $(OPENSSL_DEP) test/cpp/qps/driver.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index a39ee9f44b6..e9f622de9ed 100644 --- a/build.yaml +++ b/build.yaml @@ -1221,6 +1221,8 @@ libs: - test/cpp/microbenchmarks/fullstack_context_mutators.h - test/cpp/microbenchmarks/fullstack_fixtures.h - test/cpp/microbenchmarks/helpers.h + src: + - test/cpp/microbenchmarks/helpers.cc deps: - benchmark - grpc++ diff --git a/test/cpp/microbenchmarks/helpers.cc b/test/cpp/microbenchmarks/helpers.cc new file mode 100644 index 00000000000..947e81ffd83 --- /dev/null +++ b/test/cpp/microbenchmarks/helpers.cc @@ -0,0 +1,65 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "test/cpp/microbenchmarks/helpers.h" + +void TrackCounters::Finish(benchmark::State &state) { + std::ostringstream out; + AddToLabel(out, state); + auto label = out.str(); + if (label.length() && label[0] == ' ') { + label = label.substr(1); + } + state.SetLabel(label); +} + +void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) { +#ifdef GPR_LOW_LEVEL_COUNTERS + out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - + mu_locks_at_start_) / + (double)state.iterations()) + << " atm_cas/iter:" + << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) - + atm_cas_at_start_) / + (double)state.iterations()) + << " atm_add/iter:" + << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - + atm_add_at_start_) / + (double)state.iterations()); +#endif + grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); + out << " allocs/iter:" + << ((double)(counters_at_end.total_allocs_absolute - + counters_at_start_.total_allocs_absolute) / + (double)state.iterations()); +} diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h index ea9024dce59..42a8fbaf0bf 100644 --- a/test/cpp/microbenchmarks/helpers.h +++ b/test/cpp/microbenchmarks/helpers.h @@ -74,36 +74,8 @@ extern "C" gpr_atm gpr_counter_atm_add; class TrackCounters { public: - virtual void Finish(benchmark::State& state) { - std::ostringstream out; - AddToLabel(out, state); - auto label = out.str(); - if (label.length() && label[0] == ' ') { - label = label.substr(1); - } - state.SetLabel(label); - } - - virtual void AddToLabel(std::ostream& out, benchmark::State& state) { -#ifdef GPR_LOW_LEVEL_COUNTERS - out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - - mu_locks_at_start_) / - (double)state.iterations()) - << " atm_cas/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) - - atm_cas_at_start_) / - (double)state.iterations()) - << " atm_add/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - - atm_add_at_start_) / - (double)state.iterations()); -#endif - grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); - out << " allocs/iter:" - << ((double)(counters_at_end.total_allocs_absolute - - counters_at_start_.total_allocs_absolute) / - (double)state.iterations()); - } + virtual void Finish(benchmark::State& state); + virtual void AddToLabel(std::ostream& out, benchmark::State& state); private: #ifdef GPR_LOW_LEVEL_COUNTERS diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 06fc3799e3f..2dbbe03784e 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -5794,6 +5794,7 @@ "src": [ "test/cpp/microbenchmarks/fullstack_context_mutators.h", "test/cpp/microbenchmarks/fullstack_fixtures.h", + "test/cpp/microbenchmarks/helpers.cc", "test/cpp/microbenchmarks/helpers.h" ], "third_party": false, diff --git a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj index 62e10bf1777..228227a531e 100644 --- a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj +++ b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj @@ -165,7 +165,7 @@ - + diff --git a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters index 3e003edd5ee..8e865bcc1bb 100644 --- a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters +++ b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters @@ -1,5 +1,10 @@ + + + test\cpp\microbenchmarks + + test\cpp\microbenchmarks From ab314ee9a6dd633c9e5d5e8bf2e91cb14c4f40a7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 17:18:20 -0800 Subject: [PATCH 036/188] Add some pollset benchmarks --- test/cpp/microbenchmarks/bm_pollset.cc | 104 +++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 6 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc index adb49f0b79d..f7c3b7951d2 100644 --- a/test/cpp/microbenchmarks/bm_pollset.cc +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -34,23 +34,27 @@ /* Test out pollset latencies */ #include +#include extern "C" { +#include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/pollset.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" } #include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -static void shutdown_ps_closure(grpc_exec_ctx* exec_ctx, void* ps, - grpc_error* error) { - grpc_pollset_destroy(ps); +auto& force_library_initialization = Library::get(); + +static void shutdown_ps(grpc_exec_ctx* exec_ctx, void* ps, grpc_error* error) { + grpc_pollset_destroy(static_cast(ps)); } static void BM_CreateDestroyPollset(benchmark::State& state) { TrackCounters track_counters; size_t ps_sz = grpc_pollset_size(); - grpc_pollset* ps = gpr_malloc(ps_sz); + grpc_pollset* ps = static_cast(gpr_malloc(ps_sz)); gpr_mu* mu; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_closure shutdown_ps_closure; @@ -59,12 +63,100 @@ static void BM_CreateDestroyPollset(benchmark::State& state) { while (state.KeepRunning()) { memset(ps, 0, ps_sz); grpc_pollset_init(ps, &mu); - grpc_pollset_shutdown(&exec_ctx, ps, shutdown_ps_closure); + gpr_mu_lock(mu); + grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure); + gpr_mu_unlock(mu); grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(&state); + gpr_free(ps); + track_counters.Finish(state); } BENCHMARK(BM_CreateDestroyPollset); +static void BM_PollEmptyPollset(benchmark::State& state) { + TrackCounters track_counters; + size_t ps_sz = grpc_pollset_size(); + grpc_pollset* ps = static_cast(gpr_zalloc(ps_sz)); + gpr_mu* mu; + grpc_pollset_init(ps, &mu); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_timespec now = gpr_time_0(GPR_CLOCK_MONOTONIC); + gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); + gpr_mu_lock(mu); + while (state.KeepRunning()) { + grpc_pollset_worker* worker; + GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline)); + } + grpc_closure shutdown_ps_closure; + grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps, + grpc_schedule_on_exec_ctx); + grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure); + gpr_mu_unlock(mu); + grpc_exec_ctx_finish(&exec_ctx); + gpr_free(ps); + track_counters.Finish(state); +} +BENCHMARK(BM_PollEmptyPollset); + +template +grpc_closure* MakeClosure(F f, grpc_closure_scheduler* scheduler) { + struct C : public grpc_closure { + C(F f, grpc_closure_scheduler* scheduler) : f_(f) { + grpc_closure_init(this, C::cbfn, this, scheduler); + } + static void cbfn(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { + static_cast(arg)->f_(); + } + F f_; + }; + return new C(f, scheduler); +} + +static void BM_PollAfterWakeup(benchmark::State& state) { + TrackCounters track_counters; + size_t ps_sz = grpc_pollset_size(); + grpc_pollset* ps = static_cast(gpr_zalloc(ps_sz)); + gpr_mu* mu; + grpc_pollset_init(ps, &mu); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_timespec now = gpr_time_0(GPR_CLOCK_MONOTONIC); + gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); + grpc_wakeup_fd wakeup_fd; + GRPC_ERROR_UNREF(grpc_wakeup_fd_init(&wakeup_fd)); + grpc_fd* wakeup = grpc_fd_create(wakeup_fd.read_fd, "wakeup_read"); + grpc_pollset_add_fd(&exec_ctx, ps, wakeup); + bool done = false; + grpc_closure* continue_closure = MakeClosure( + [&]() { + GRPC_ERROR_UNREF(grpc_wakeup_fd_consume_wakeup(&wakeup_fd)); + if (!state.KeepRunning()) { + done = true; + return; + } + GRPC_ERROR_UNREF(grpc_wakeup_fd_wakeup(&wakeup_fd)); + grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure); + }, + grpc_schedule_on_exec_ctx); + GRPC_ERROR_UNREF(grpc_wakeup_fd_wakeup(&wakeup_fd)); + grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure); + gpr_mu_lock(mu); + while (!done) { + grpc_pollset_worker* worker; + GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline)); + } + grpc_fd_orphan(&exec_ctx, wakeup, NULL, NULL, "done"); + wakeup_fd.read_fd = 0; + grpc_closure shutdown_ps_closure; + grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps, + grpc_schedule_on_exec_ctx); + grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure); + gpr_mu_unlock(mu); + grpc_exec_ctx_finish(&exec_ctx); + grpc_wakeup_fd_destroy(&wakeup_fd); + gpr_free(ps); + track_counters.Finish(state); +} +BENCHMARK(BM_PollAfterWakeup); + BENCHMARK_MAIN(); From 055abd65fb3db3f679faf378c09d9f0a6af82d07 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Mar 2017 17:20:36 -0800 Subject: [PATCH 037/188] Fix linux --- test/cpp/microbenchmarks/bm_pollset.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc index f7c3b7951d2..9d7fac39c77 100644 --- a/test/cpp/microbenchmarks/bm_pollset.cc +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -45,6 +45,8 @@ extern "C" { #include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" +#include + auto& force_library_initialization = Library::get(); static void shutdown_ps(grpc_exec_ctx* exec_ctx, void* ps, grpc_error* error) { From d0084c220e3f3b5a35adbed430942237ef327323 Mon Sep 17 00:00:00 2001 From: yang-g Date: Mon, 6 Mar 2017 11:23:35 -0800 Subject: [PATCH 038/188] Include x.pb.h instead of x.grpc.pb.h for message-only protos --- test/cpp/interop/http2_client.cc | 2 +- test/cpp/interop/http2_client.h | 2 +- test/cpp/interop/interop_client.cc | 4 ++-- test/cpp/interop/interop_client.h | 2 +- test/cpp/interop/interop_server.cc | 4 ++-- test/cpp/interop/reconnect_interop_client.cc | 4 ++-- test/cpp/interop/reconnect_interop_server.cc | 4 ++-- test/cpp/qps/client.h | 2 +- test/cpp/qps/driver.h | 2 +- test/cpp/qps/histogram.h | 2 +- test/cpp/qps/server.h | 4 ++-- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/test/cpp/interop/http2_client.cc b/test/cpp/interop/http2_client.cc index 01c07823cfe..38a437f39f8 100644 --- a/test/cpp/interop/http2_client.cc +++ b/test/cpp/interop/http2_client.cc @@ -41,7 +41,7 @@ #include #include "src/core/lib/transport/byte_stream.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/interop/http2_client.h" diff --git a/test/cpp/interop/http2_client.h b/test/cpp/interop/http2_client.h index 12df5d26bc2..e57d695205b 100644 --- a/test/cpp/interop/http2_client.h +++ b/test/cpp/interop/http2_client.h @@ -38,7 +38,7 @@ #include #include -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" namespace grpc { diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc index b7f2723c39b..55ba324cc7d 100644 --- a/test/cpp/interop/interop_client.cc +++ b/test/cpp/interop/interop_client.cc @@ -46,8 +46,8 @@ #include #include "src/core/lib/transport/byte_stream.h" -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/empty.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/interop/client_helper.h" #include "test/cpp/interop/interop_client.h" diff --git a/test/cpp/interop/interop_client.h b/test/cpp/interop/interop_client.h index 74f4db6b789..efcb7d28608 100644 --- a/test/cpp/interop/interop_client.h +++ b/test/cpp/interop/interop_client.h @@ -38,7 +38,7 @@ #include #include -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" namespace grpc { diff --git a/test/cpp/interop/interop_server.cc b/test/cpp/interop/interop_server.cc index 5a810b45ef4..1cbca179287 100644 --- a/test/cpp/interop/interop_server.cc +++ b/test/cpp/interop/interop_server.cc @@ -48,8 +48,8 @@ #include "src/core/lib/support/string.h" #include "src/core/lib/transport/byte_stream.h" -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/empty.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/interop/server_helper.h" #include "test/cpp/util/test_config.h" diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 1c2f6066377..01d985068dc 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -40,8 +40,8 @@ #include #include #include -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/empty.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/util/create_test_channel.h" #include "test/cpp/util/test_config.h" diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc index 634d0a90fce..8d1b884af91 100644 --- a/test/cpp/interop/reconnect_interop_server.cc +++ b/test/cpp/interop/reconnect_interop_server.cc @@ -47,8 +47,8 @@ #include #include -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/empty.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/core/util/reconnect_server.h" #include "test/cpp/util/test_config.h" diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index baa9304cc2d..25a19a5a740 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -46,7 +46,7 @@ #include #include -#include "src/proto/grpc/testing/payloads.grpc.pb.h" +#include "src/proto/grpc/testing/payloads.pb.h" #include "src/proto/grpc/testing/services.grpc.pb.h" #include "test/cpp/qps/histogram.h" diff --git a/test/cpp/qps/driver.h b/test/cpp/qps/driver.h index e72d30a4eff..dd32a16c879 100644 --- a/test/cpp/qps/driver.h +++ b/test/cpp/qps/driver.h @@ -36,7 +36,7 @@ #include -#include "src/proto/grpc/testing/control.grpc.pb.h" +#include "src/proto/grpc/testing/control.pb.h" #include "test/cpp/qps/histogram.h" namespace grpc { diff --git a/test/cpp/qps/histogram.h b/test/cpp/qps/histogram.h index acb415f0a10..470a3943013 100644 --- a/test/cpp/qps/histogram.h +++ b/test/cpp/qps/histogram.h @@ -35,7 +35,7 @@ #define TEST_QPS_HISTOGRAM_H #include -#include "src/proto/grpc/testing/stats.grpc.pb.h" +#include "src/proto/grpc/testing/stats.pb.h" namespace grpc { namespace testing { diff --git a/test/cpp/qps/server.h b/test/cpp/qps/server.h index 821d5935beb..8fbf37a0957 100644 --- a/test/cpp/qps/server.h +++ b/test/cpp/qps/server.h @@ -38,8 +38,8 @@ #include #include -#include "src/proto/grpc/testing/control.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/control.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "test/core/end2end/data/ssl_test_data.h" #include "test/core/util/port.h" #include "test/cpp/qps/usage_timer.h" From 111db946b7c8451f910b45bbca4700407e68fda6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Mar 2017 14:36:17 -0800 Subject: [PATCH 039/188] Add a test of creating a call on a lame channel in C++ --- test/cpp/microbenchmarks/bm_call_create.cc | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index e2e5bbbe00e..7b228f076a6 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -55,6 +56,8 @@ extern "C" { #include "src/core/lib/transport/transport_impl.h" } +#include "src/cpp/client/create_channel_internal.h" +#include "src/proto/grpc/testing/echo.grpc.pb.h" #include "third_party/benchmark/include/benchmark/benchmark.h" static struct Init { @@ -105,6 +108,31 @@ static void BM_CallCreateDestroy(benchmark::State &state) { BENCHMARK_TEMPLATE(BM_CallCreateDestroy, InsecureChannel); BENCHMARK_TEMPLATE(BM_CallCreateDestroy, LameChannel); +static void *tag(int i) { + return reinterpret_cast(static_cast(i)); +} + +static void BM_LameChannelCallCreateCpp(benchmark::State &state) { + auto stub = + grpc::testing::EchoTestService::NewStub(grpc::CreateChannelInternal( + "", grpc_lame_client_channel_create( + "localhost:1234", GRPC_STATUS_UNAUTHENTICATED, "blah"))); + grpc::CompletionQueue cq; + grpc::testing::EchoRequest send_request; + grpc::testing::EchoResponse recv_response; + grpc::Status recv_status; + while (state.KeepRunning()) { + grpc::ClientContext cli_ctx; + auto reader = stub->AsyncEcho(&cli_ctx, send_request, &cq); + reader->Finish(&recv_response, &recv_status, tag(0)); + void *t; + bool ok; + GPR_ASSERT(cq.Next(&t, &ok)); + GPR_ASSERT(ok); + } +} +BENCHMARK(BM_LameChannelCallCreateCpp); + static void FilterDestroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { gpr_free(arg); From e067b6e119e94b097092458dca99e8b253caf64b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Mar 2017 14:46:37 -0800 Subject: [PATCH 040/188] Resolve sanity failures --- CMakeLists.txt | 5 +++-- Makefile | 4 ++-- build.yaml | 5 +++-- test/core/end2end/BUILD | 4 ++-- test/core/end2end/fixtures/h2_http_proxy.c | 2 +- .../fixtures/{http_proxy.c => http_proxy_fixture.c} | 2 +- .../fixtures/{http_proxy.h => http_proxy_fixture.h} | 0 tools/run_tests/generated/sources_and_headers.json | 9 +++++---- vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj | 4 ++-- .../grpc_test_util/grpc_test_util.vcxproj.filters | 4 ++-- .../grpc_test_util_unsecure.vcxproj | 4 ++-- .../grpc_test_util_unsecure.vcxproj.filters | 4 ++-- .../vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj | 3 +++ 13 files changed, 28 insertions(+), 22 deletions(-) rename test/core/end2end/fixtures/{http_proxy.c => http_proxy_fixture.c} (99%) rename test/core/end2end/fixtures/{http_proxy.h => http_proxy_fixture.h} (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index dbc2b4b8bf6..c95085f3f15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1447,7 +1447,7 @@ add_library(grpc_test_util test/core/security/oauth2_utils.c test/core/end2end/cq_verifier.c test/core/end2end/fake_resolver.c - test/core/end2end/fixtures/http_proxy.c + test/core/end2end/fixtures/http_proxy_fixture.c test/core/end2end/fixtures/proxy.c test/core/iomgr/endpoint_tests.c test/core/util/debugger_macros.c @@ -1654,7 +1654,7 @@ if (gRPC_BUILD_TESTS) add_library(grpc_test_util_unsecure test/core/end2end/cq_verifier.c test/core/end2end/fake_resolver.c - test/core/end2end/fixtures/http_proxy.c + test/core/end2end/fixtures/http_proxy_fixture.c test/core/end2end/fixtures/proxy.c test/core/iomgr/endpoint_tests.c test/core/util/debugger_macros.c @@ -3139,6 +3139,7 @@ target_link_libraries(grpc_benchmark ${_gRPC_ALLTARGETS_LIBRARIES} benchmark grpc++ + grpc_test_util grpc ${_gRPC_GFLAGS_LIBRARIES} ) diff --git a/Makefile b/Makefile index c2d06feedd3..3701f1d48c3 100644 --- a/Makefile +++ b/Makefile @@ -3325,7 +3325,7 @@ LIBGRPC_TEST_UTIL_SRC = \ test/core/security/oauth2_utils.c \ test/core/end2end/cq_verifier.c \ test/core/end2end/fake_resolver.c \ - test/core/end2end/fixtures/http_proxy.c \ + test/core/end2end/fixtures/http_proxy_fixture.c \ test/core/end2end/fixtures/proxy.c \ test/core/iomgr/endpoint_tests.c \ test/core/util/debugger_macros.c \ @@ -3525,7 +3525,7 @@ endif LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ test/core/end2end/cq_verifier.c \ test/core/end2end/fake_resolver.c \ - test/core/end2end/fixtures/http_proxy.c \ + test/core/end2end/fixtures/http_proxy_fixture.c \ test/core/end2end/fixtures/proxy.c \ test/core/iomgr/endpoint_tests.c \ test/core/util/debugger_macros.c \ diff --git a/build.yaml b/build.yaml index a39ee9f44b6..a4333b47733 100644 --- a/build.yaml +++ b/build.yaml @@ -586,7 +586,7 @@ filegroups: headers: - test/core/end2end/cq_verifier.h - test/core/end2end/fake_resolver.h - - test/core/end2end/fixtures/http_proxy.h + - test/core/end2end/fixtures/http_proxy_fixture.h - test/core/end2end/fixtures/proxy.h - test/core/iomgr/endpoint_tests.h - test/core/util/debugger_macros.h @@ -602,7 +602,7 @@ filegroups: src: - test/core/end2end/cq_verifier.c - test/core/end2end/fake_resolver.c - - test/core/end2end/fixtures/http_proxy.c + - test/core/end2end/fixtures/http_proxy_fixture.c - test/core/end2end/fixtures/proxy.c - test/core/iomgr/endpoint_tests.c - test/core/util/debugger_macros.c @@ -1224,6 +1224,7 @@ libs: deps: - benchmark - grpc++ + - grpc_test_util - grpc - name: grpc_cli_libs build: private diff --git a/test/core/end2end/BUILD b/test/core/end2end/BUILD index a40fb8e083f..0cef7aa01df 100644 --- a/test/core/end2end/BUILD +++ b/test/core/end2end/BUILD @@ -63,8 +63,8 @@ cc_library( cc_library( name = 'http_proxy', - hdrs = ['fixtures/http_proxy.h'], - srcs = ['fixtures/http_proxy.c'], + hdrs = ['fixtures/http_proxy_fixture.h'], + srcs = ['fixtures/http_proxy_fixture.c'], copts = ['-std=c99'], deps = ['//:gpr', '//:grpc', '//test/core/util:grpc_test_util'] ) diff --git a/test/core/end2end/fixtures/h2_http_proxy.c b/test/core/end2end/fixtures/h2_http_proxy.c index 44b223664ab..55c65fa70ef 100644 --- a/test/core/end2end/fixtures/h2_http_proxy.c +++ b/test/core/end2end/fixtures/h2_http_proxy.c @@ -49,7 +49,7 @@ #include "src/core/lib/support/env.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/server.h" -#include "test/core/end2end/fixtures/http_proxy.h" +#include "test/core/end2end/fixtures/http_proxy_fixture.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/test/core/end2end/fixtures/http_proxy.c b/test/core/end2end/fixtures/http_proxy_fixture.c similarity index 99% rename from test/core/end2end/fixtures/http_proxy.c rename to test/core/end2end/fixtures/http_proxy_fixture.c index 9ccb1263ee7..bcd1c9914b2 100644 --- a/test/core/end2end/fixtures/http_proxy.c +++ b/test/core/end2end/fixtures/http_proxy_fixture.c @@ -31,7 +31,7 @@ * */ -#include "test/core/end2end/fixtures/http_proxy.h" +#include "test/core/end2end/fixtures/http_proxy_fixture.h" #include "src/core/lib/iomgr/sockaddr.h" diff --git a/test/core/end2end/fixtures/http_proxy.h b/test/core/end2end/fixtures/http_proxy_fixture.h similarity index 100% rename from test/core/end2end/fixtures/http_proxy.h rename to test/core/end2end/fixtures/http_proxy_fixture.h diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 06fc3799e3f..9162c2af60f 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -5781,7 +5781,8 @@ "deps": [ "benchmark", "grpc", - "grpc++" + "grpc++", + "grpc_test_util" ], "headers": [ "test/cpp/microbenchmarks/fullstack_context_mutators.h", @@ -8004,7 +8005,7 @@ "headers": [ "test/core/end2end/cq_verifier.h", "test/core/end2end/fake_resolver.h", - "test/core/end2end/fixtures/http_proxy.h", + "test/core/end2end/fixtures/http_proxy_fixture.h", "test/core/end2end/fixtures/proxy.h", "test/core/iomgr/endpoint_tests.h", "test/core/util/debugger_macros.h", @@ -8026,8 +8027,8 @@ "test/core/end2end/cq_verifier.h", "test/core/end2end/fake_resolver.c", "test/core/end2end/fake_resolver.h", - "test/core/end2end/fixtures/http_proxy.c", - "test/core/end2end/fixtures/http_proxy.h", + "test/core/end2end/fixtures/http_proxy_fixture.c", + "test/core/end2end/fixtures/http_proxy_fixture.h", "test/core/end2end/fixtures/proxy.c", "test/core/end2end/fixtures/proxy.h", "test/core/iomgr/endpoint_tests.c", diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index d08ceb68287..e7c9fb71f33 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -182,7 +182,7 @@ - + @@ -317,7 +317,7 @@ - + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 3beaa80994f..3d36948aaef 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -22,7 +22,7 @@ test\core\end2end - + test\core\end2end\fixtures @@ -518,7 +518,7 @@ test\core\end2end - + test\core\end2end\fixtures diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj index 5a58cae83ff..af13acef455 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj @@ -149,7 +149,7 @@ - + @@ -168,7 +168,7 @@ - + diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters index 88c875cb011..4da043ea908 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters @@ -7,7 +7,7 @@ test\core\end2end - + test\core\end2end\fixtures @@ -54,7 +54,7 @@ test\core\end2end - + test\core\end2end\fixtures diff --git a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj index 62e10bf1777..d19df9d3508 100644 --- a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj +++ b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj @@ -175,6 +175,9 @@ {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} From 8918aaeccd4cd22cc6e549dcb3ddc0d661993e97 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Mar 2017 15:59:08 -0800 Subject: [PATCH 041/188] Document status ordering rules This documents a rule that's existed in a hard to find internal document that's existed since Feb 2016 by abhikumar@google.com. Since that rule is critical to untangling some gRPC C core behavior, we should document it publically. --- doc/status_ordering.md | 16 ++++++++++++++++ .../end2end/tests/streaming_error_response.c | 3 +++ 2 files changed, 19 insertions(+) create mode 100644 doc/status_ordering.md diff --git a/doc/status_ordering.md b/doc/status_ordering.md new file mode 100644 index 00000000000..fccfa863a3e --- /dev/null +++ b/doc/status_ordering.md @@ -0,0 +1,16 @@ +Ordering Status and Reads in the gRPC API +----------------------------------------- + +Rules for implementors: +1. Reads and Writes Must not succeed after Status has been delivered. +2. OK Status is only delivered after all buffered messages are read. +3. Reads May continue to succeed after a failing write. + However, once a write fails, all subsequent writes Must fail, + and similarly, once a read fails, all subsequent reads Must fail. +4. When an error status is known to the library, if the user asks for status, + the library Should discard messages received in the library but not delivered + to the user and then deliver the status. If the user does not ask for status + but continues reading, the library Should deliver buffered messages before + delivering status. The library MAY choose to implement the stricter version + where errors cause all buffered messages to be dropped, but this is not a + requirement. diff --git a/test/core/end2end/tests/streaming_error_response.c b/test/core/end2end/tests/streaming_error_response.c index 42055907c85..2b9c404b15a 100644 --- a/test/core/end2end/tests/streaming_error_response.c +++ b/test/core/end2end/tests/streaming_error_response.c @@ -31,6 +31,9 @@ * */ +/** \file Verify that status ordering rules are obeyed. + \ref doc/status_ordering.md */ + #include "test/core/end2end/end2end_tests.h" #include From b699640a773d74e07bbcf5ded4a86d14e49e1a04 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Mar 2017 16:10:20 -0800 Subject: [PATCH 042/188] Factor out benchmark parsing code to enable diffs more easily --- tools/profiling/microbenchmarks/bm2bq.py | 137 +---------------- tools/profiling/microbenchmarks/bm_json.py | 167 +++++++++++++++++++++ 2 files changed, 172 insertions(+), 132 deletions(-) create mode 100644 tools/profiling/microbenchmarks/bm_json.py diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 280f217e690..ffb158a28d0 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -36,7 +36,7 @@ import sys import json import csv -import os +import bm_json columns = [ ('jenkins_build', 'integer'), @@ -82,137 +82,10 @@ with open(sys.argv[1]) as f: writer = csv.DictWriter(sys.stdout, [c for c,t in columns]) -bm_specs = { - 'BM_UnaryPingPong': { - 'tpl': ['fixture', 'client_mutator', 'server_mutator'], - 'dyn': ['request_size', 'response_size'], - }, - 'BM_PumpStreamClientToServer': { - 'tpl': ['fixture'], - 'dyn': ['request_size'], - }, - 'BM_PumpStreamServerToClient': { - 'tpl': ['fixture'], - 'dyn': ['request_size'], - }, - 'BM_StreamingPingPong': { - 'tpl': ['fixture', 'client_mutator', 'server_mutator'], - 'dyn': ['request_size', 'request_count'], - }, - 'BM_StreamingPingPongMsgs': { - 'tpl': ['fixture', 'client_mutator', 'server_mutator'], - 'dyn': ['request_size'], - }, - 'BM_PumpStreamServerToClient_Trickle': { - 'tpl': [], - 'dyn': ['request_size', 'bandwidth_kilobits'], - }, - 'BM_ErrorStringOnNewError': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorStringRepeatedly': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorGetStatus': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorGetStatusCode': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorHttpError': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_HasClearGrpcStatus': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_IsolatedFilter' : { - 'tpl': ['fixture', 'client_mutator'], - 'dyn': [], - }, - 'BM_HpackEncoderEncodeHeader' : { - 'tpl': ['fixture'], - 'dyn': ['end_of_stream', 'request_size'], - }, - 'BM_HpackParserParseHeader' : { - 'tpl': ['fixture'], - 'dyn': [], - }, -} - -def numericalize(s): - if not s: return '' - if s[-1] == 'k': - return int(s[:-1]) * 1024 - if s[-1] == 'M': - return int(s[:-1]) * 1024 * 1024 - if 0 <= (ord(s[-1]) - ord('0')) <= 9: - return int(s) - assert 'not a number: %s' % s - -def parse_name(name): - if '<' not in name and '/' not in name and name not in bm_specs: - return {'name': name} - rest = name - out = {} - tpl_args = [] - dyn_args = [] - if '<' in rest: - tpl_bit = rest[rest.find('<') + 1 : rest.rfind('>')] - arg = '' - nesting = 0 - for c in tpl_bit: - if c == '<': - nesting += 1 - arg += c - elif c == '>': - nesting -= 1 - arg += c - elif c == ',': - if nesting == 0: - tpl_args.append(arg.strip()) - arg = '' - else: - arg += c - else: - arg += c - tpl_args.append(arg.strip()) - rest = rest[:rest.find('<')] + rest[rest.rfind('>') + 1:] - if '/' in rest: - s = rest.split('/') - rest = s[0] - dyn_args = s[1:] - name = rest - assert name in bm_specs, 'bm_specs needs to be expanded for %s' % name - assert len(dyn_args) == len(bm_specs[name]['dyn']) - assert len(tpl_args) == len(bm_specs[name]['tpl']) - out['name'] = name - out.update(dict((k, numericalize(v)) for k, v in zip(bm_specs[name]['dyn'], dyn_args))) - out.update(dict(zip(bm_specs[name]['tpl'], tpl_args))) - return out - -for bm in js['benchmarks']: - context = js['context'] - if 'label' in bm: - labels_list = [s.split(':') for s in bm['label'].strip().split(' ') if len(s) and s[0] != '#'] - for el in labels_list: - el[0] = el[0].replace('/iter', '_per_iteration') - labels = dict(labels_list) - else: - labels = {} - row = { - 'jenkins_build': os.environ.get('BUILD_NUMBER', ''), - 'jenkins_job': os.environ.get('JOB_NAME', ''), - } - row.update(context) - row.update(bm) - row.update(parse_name(row['name'])) - row.update(labels) +for row in bm_json.expand_json(js): if 'label' in row: del row['label'] + del row['cpp_name'] writer.writerow(row) + + diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py new file mode 100644 index 00000000000..dd07719b2a6 --- /dev/null +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -0,0 +1,167 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os + +_BM_SPECS = { + 'BM_UnaryPingPong': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size', 'response_size'], + }, + 'BM_PumpStreamClientToServer': { + 'tpl': ['fixture'], + 'dyn': ['request_size'], + }, + 'BM_PumpStreamServerToClient': { + 'tpl': ['fixture'], + 'dyn': ['request_size'], + }, + 'BM_StreamingPingPong': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size', 'request_count'], + }, + 'BM_StreamingPingPongMsgs': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size'], + }, + 'BM_PumpStreamServerToClient_Trickle': { + 'tpl': [], + 'dyn': ['request_size', 'bandwidth_kilobits'], + }, + 'BM_ErrorStringOnNewError': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_ErrorStringRepeatedly': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_ErrorGetStatus': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_ErrorGetStatusCode': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_ErrorHttpError': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_HasClearGrpcStatus': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_IsolatedFilter' : { + 'tpl': ['fixture', 'client_mutator'], + 'dyn': [], + }, + 'BM_HpackEncoderEncodeHeader' : { + 'tpl': ['fixture'], + 'dyn': ['end_of_stream', 'request_size'], + }, + 'BM_HpackParserParseHeader' : { + 'tpl': ['fixture'], + 'dyn': [], + }, +} + +def numericalize(s): + if not s: return '' + if s[-1] == 'k': + return int(s[:-1]) * 1024 + if s[-1] == 'M': + return int(s[:-1]) * 1024 * 1024 + if 0 <= (ord(s[-1]) - ord('0')) <= 9: + return int(s) + assert 'not a number: %s' % s + +def parse_name(name): + cpp_name = name + if '<' not in name and '/' not in name and name not in _BM_SPECS: + return {'name': name, 'cpp_name': name} + rest = name + out = {} + tpl_args = [] + dyn_args = [] + if '<' in rest: + tpl_bit = rest[rest.find('<') + 1 : rest.rfind('>')] + arg = '' + nesting = 0 + for c in tpl_bit: + if c == '<': + nesting += 1 + arg += c + elif c == '>': + nesting -= 1 + arg += c + elif c == ',': + if nesting == 0: + tpl_args.append(arg.strip()) + arg = '' + else: + arg += c + else: + arg += c + tpl_args.append(arg.strip()) + rest = rest[:rest.find('<')] + rest[rest.rfind('>') + 1:] + if '/' in rest: + s = rest.split('/') + rest = s[0] + dyn_args = s[1:] + name = rest + assert name in _BM_SPECS, '_BM_SPECS needs to be expanded for %s' % name + assert len(dyn_args) == len(_BM_SPECS[name]['dyn']) + assert len(tpl_args) == len(_BM_SPECS[name]['tpl']) + out['name'] = name + out['cpp_name'] = cpp_name + out.update(dict((k, numericalize(v)) for k, v in zip(_BM_SPECS[name]['dyn'], dyn_args))) + out.update(dict(zip(_BM_SPECS[name]['tpl'], tpl_args))) + return out + +def expand_json(js): + for bm in js['benchmarks']: + context = js['context'] + if 'label' in bm: + labels_list = [s.split(':') for s in bm['label'].strip().split(' ') if len(s) and s[0] != '#'] + for el in labels_list: + el[0] = el[0].replace('/iter', '_per_iteration') + labels = dict(labels_list) + else: + labels = {} + row = { + 'jenkins_build': os.environ.get('BUILD_NUMBER', ''), + 'jenkins_job': os.environ.get('JOB_NAME', ''), + } + row.update(context) + row.update(bm) + row.update(parse_name(row['name'])) + row.update(labels) + yield row + From 3651eece9b387e2d4199a92ca4a22c3c50c39b93 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Mar 2017 16:15:48 -0800 Subject: [PATCH 043/188] Fix merge error --- tools/profiling/microbenchmarks/bm2bq.py | 140 ----------------------- 1 file changed, 140 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 17977786036..ae59332f1b3 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -88,148 +88,8 @@ else: writer = csv.DictWriter(sys.stdout, [c for c,t in columns]) -<<<<<<< HEAD for row in bm_json.expand_json(js, js2): -======= -bm_specs = { - 'BM_UnaryPingPong': { - 'tpl': ['fixture', 'client_mutator', 'server_mutator'], - 'dyn': ['request_size', 'response_size'], - }, - 'BM_PumpStreamClientToServer': { - 'tpl': ['fixture'], - 'dyn': ['request_size'], - }, - 'BM_PumpStreamServerToClient': { - 'tpl': ['fixture'], - 'dyn': ['request_size'], - }, - 'BM_StreamingPingPong': { - 'tpl': ['fixture', 'client_mutator', 'server_mutator'], - 'dyn': ['request_size', 'request_count'], - }, - 'BM_StreamingPingPongMsgs': { - 'tpl': ['fixture', 'client_mutator', 'server_mutator'], - 'dyn': ['request_size'], - }, - 'BM_PumpStreamServerToClient_Trickle': { - 'tpl': [], - 'dyn': ['request_size', 'bandwidth_kilobits'], - }, - 'BM_ErrorStringOnNewError': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorStringRepeatedly': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorGetStatus': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorGetStatusCode': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorHttpError': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_HasClearGrpcStatus': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_IsolatedFilter' : { - 'tpl': ['fixture', 'client_mutator'], - 'dyn': [], - }, - 'BM_HpackEncoderEncodeHeader' : { - 'tpl': ['fixture'], - 'dyn': ['end_of_stream', 'request_size'], - }, - 'BM_HpackParserParseHeader' : { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_CallCreateDestroy' : { - 'tpl': ['fixture'], - 'dyn': [], - }, -} - -def numericalize(s): - if not s: return '' - if s[-1] == 'k': - return int(s[:-1]) * 1024 - if s[-1] == 'M': - return int(s[:-1]) * 1024 * 1024 - if 0 <= (ord(s[-1]) - ord('0')) <= 9: - return int(s) - assert 'not a number: %s' % s - -def parse_name(name): - if '<' not in name and '/' not in name and name not in bm_specs: - return {'name': name} - rest = name - out = {} - tpl_args = [] - dyn_args = [] - if '<' in rest: - tpl_bit = rest[rest.find('<') + 1 : rest.rfind('>')] - arg = '' - nesting = 0 - for c in tpl_bit: - if c == '<': - nesting += 1 - arg += c - elif c == '>': - nesting -= 1 - arg += c - elif c == ',': - if nesting == 0: - tpl_args.append(arg.strip()) - arg = '' - else: - arg += c - else: - arg += c - tpl_args.append(arg.strip()) - rest = rest[:rest.find('<')] + rest[rest.rfind('>') + 1:] - if '/' in rest: - s = rest.split('/') - rest = s[0] - dyn_args = s[1:] - name = rest - assert name in bm_specs, 'bm_specs needs to be expanded for %s' % name - assert len(dyn_args) == len(bm_specs[name]['dyn']) - assert len(tpl_args) == len(bm_specs[name]['tpl']) - out['name'] = name - out.update(dict((k, numericalize(v)) for k, v in zip(bm_specs[name]['dyn'], dyn_args))) - out.update(dict(zip(bm_specs[name]['tpl'], tpl_args))) - return out - -for bm in js['benchmarks']: - context = js['context'] - if 'label' in bm: - labels_list = [s.split(':') for s in bm['label'].strip().split(' ') if len(s) and s[0] != '#'] - for el in labels_list: - el[0] = el[0].replace('/iter', '_per_iteration') - labels = dict(labels_list) - else: - labels = {} - row = { - 'jenkins_build': os.environ.get('BUILD_NUMBER', ''), - 'jenkins_job': os.environ.get('JOB_NAME', ''), - } - row.update(context) - row.update(bm) - row.update(parse_name(row['name'])) - row.update(labels) ->>>>>>> cf54c151f1853af1fe1df2058ccbc2ca3a26c43c if 'label' in row: del row['label'] del row['cpp_name'] writer.writerow(row) - - From 21490a1e2dbbff6f6b616232cdabccfbf0b7831c Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 6 Mar 2017 18:50:39 -0800 Subject: [PATCH 044/188] Fix flush read --- .../cronet/transport/cronet_transport.c | 81 +++++++++++++------ 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 01a03533daf..ff31f2ffc56 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -54,6 +54,7 @@ #include "third_party/objective_c/Cronet/bidirectional_stream_c.h" #define GRPC_HEADER_SIZE_IN_BYTES 5 +#define GRPC_FLUSH_READ_SIZE 4096 #define CRONET_LOG(...) \ do { \ @@ -155,7 +156,9 @@ struct op_state { bool flush_read; bool flush_cronet_when_ready; bool pending_write_for_trailer; - bool unprocessed_send_message; + bool pending_send_message; + bool pending_recv_trailing_metadata; + bool pending_read_from_cronet; grpc_error *cancel_error; /* data structure for storing data coming from server */ struct read_state rs; @@ -248,11 +251,27 @@ static const char *op_id_string(enum e_op_id i) { return "UNKNOWN"; } -static void free_read_buffer(stream_obj *s) { +static void null_and_maybe_free_read_buffer(stream_obj *s) { if (s->state.rs.read_buffer && s->state.rs.read_buffer != s->state.rs.grpc_header_bytes) { gpr_free(s->state.rs.read_buffer); - s->state.rs.read_buffer = NULL; + } + s->state.rs.read_buffer = NULL; +} + +static void maybe_flush_read(stream_obj *s) { + if (s->state.pending_recv_trailing_metadata && s->state.fail_state) { + if (!s->state.flush_read) { + CRONET_LOG(GPR_DEBUG, "%p: Flush read", s); + s->state.flush_read = true; + null_and_maybe_free_read_buffer(s); + s->state.rs.read_buffer = gpr_malloc(GRPC_FLUSH_READ_SIZE); + if (!s->state.pending_read_from_cronet) { + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); + bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, GRPC_FLUSH_READ_SIZE); + s->state.pending_read_from_cronet = true; + } + } } } @@ -279,7 +298,11 @@ static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { storage->head = new_op; storage->num_pending_ops++; if (op->send_message) { - s->state.unprocessed_send_message = true; + s->state.pending_send_message = true; + } + if (op->recv_trailing_metadata) { + s->state.pending_recv_trailing_metadata = true; + maybe_flush_read(s); } CRONET_LOG(GPR_DEBUG, "adding new op %p. %d in the queue.", new_op, storage->num_pending_ops); @@ -367,7 +390,7 @@ static void on_failed(bidirectional_stream *stream, int net_error) { gpr_free(s->state.ws.write_buffer); s->state.ws.write_buffer = NULL; } - free_read_buffer(s); + null_and_maybe_free_read_buffer(s); gpr_mu_unlock(&s->mu); execute_from_storage(s); } @@ -390,7 +413,7 @@ static void on_canceled(bidirectional_stream *stream) { gpr_free(s->state.ws.write_buffer); s->state.ws.write_buffer = NULL; } - free_read_buffer(s); + null_and_maybe_free_read_buffer(s); gpr_mu_unlock(&s->mu); execute_from_storage(s); } @@ -405,7 +428,7 @@ static void on_succeeded(bidirectional_stream *stream) { bidirectional_stream_destroy(s->cbs); s->state.state_callback_received[OP_SUCCEEDED] = true; s->cbs = NULL; - free_read_buffer(s); + null_and_maybe_free_read_buffer(s); gpr_mu_unlock(&s->mu); execute_from_storage(s); } @@ -473,6 +496,7 @@ static void on_response_headers_received( CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, s->state.rs.remaining_bytes); + s->state.pending_read_from_cronet = true; } gpr_mu_unlock(&s->mu); grpc_exec_ctx_finish(&exec_ctx); @@ -504,10 +528,12 @@ static void on_read_completed(bidirectional_stream *stream, char *data, CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); gpr_mu_lock(&s->mu); + s->state.pending_read_from_cronet = false; s->state.state_callback_received[OP_RECV_MESSAGE] = true; if (count > 0 && s->state.flush_read) { CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); - bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, 4096); + bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, GRPC_FLUSH_READ_SIZE); + s->state.pending_read_from_cronet = true; gpr_mu_unlock(&s->mu); } else if (count > 0) { s->state.rs.received_bytes += count; @@ -518,16 +544,14 @@ static void on_read_completed(bidirectional_stream *stream, char *data, bidirectional_stream_read( s->cbs, s->state.rs.read_buffer + s->state.rs.received_bytes, s->state.rs.remaining_bytes); + s->state.pending_read_from_cronet = true; gpr_mu_unlock(&s->mu); } else { gpr_mu_unlock(&s->mu); execute_from_storage(s); } } else { - if (s->state.flush_read) { - gpr_free(s->state.rs.read_buffer); - s->state.rs.read_buffer = NULL; - } + null_and_maybe_free_read_buffer(s); s->state.rs.read_stream_closed = true; gpr_mu_unlock(&s->mu); execute_from_storage(s); @@ -564,6 +588,7 @@ static void on_response_trailers_received( if (0 == strcmp(trailers->headers[i].key, "grpc-status") && 0 != strcmp(trailers->headers[i].value, "0")) { s->state.fail_state = true; + maybe_flush_read(s); } } s->state.state_callback_received[OP_RECV_TRAILING_METADATA] = true; @@ -778,7 +803,7 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; /* we haven't sent message yet */ - else if (stream_state->unprocessed_send_message && + else if (stream_state->pending_send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; /* we haven't got on_write_completed for the send yet */ @@ -900,7 +925,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, } else if (stream_op->send_message && op_can_be_run(stream_op, s, &oas->state, OP_SEND_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_MESSAGE", oas); - stream_state->unprocessed_send_message = false; + stream_state->pending_send_message = false; if (stream_state->state_callback_received[OP_FAILED]) { result = NO_ACTION_POSSIBLE; CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed."); @@ -1009,6 +1034,13 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; result = ACTION_TAKEN_NO_CALLBACK; + } else if (stream_state->flush_read) { + CRONET_LOG(GPR_DEBUG, "flush read"); + grpc_closure_sched(exec_ctx, stream_op->recv_message_ready, + GRPC_ERROR_NONE); + stream_state->state_op_done[OP_RECV_MESSAGE] = true; + oas->state.state_op_done[OP_RECV_MESSAGE] = true; + result = ACTION_TAKEN_NO_CALLBACK; } else if (stream_state->rs.length_field_received == false) { if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && stream_state->rs.remaining_bytes == 0) { @@ -1029,6 +1061,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, true; /* Indicates that at least one read request has been made */ bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, stream_state->rs.remaining_bytes); + stream_state->pending_read_from_cronet = true; result = ACTION_TAKEN_WITH_CALLBACK; } else { stream_state->rs.remaining_bytes = 0; @@ -1047,11 +1080,13 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes; stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; stream_state->rs.received_bytes = 0; + stream_state->rs.length_field_received = false; CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); stream_state->state_op_done[OP_READ_REQ_MADE] = true; /* Indicates that at least one read request has been made */ bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, stream_state->rs.remaining_bytes); + stream_state->pending_read_from_cronet = true; result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_state->rs.remaining_bytes == 0) { @@ -1064,6 +1099,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, true; /* Indicates that at least one read request has been made */ bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, stream_state->rs.remaining_bytes); + stream_state->pending_read_from_cronet = true; result = ACTION_TAKEN_WITH_CALLBACK; } else { result = NO_ACTION_POSSIBLE; @@ -1075,7 +1111,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, uint8_t *dst_p = GRPC_SLICE_START_PTR(read_data_slice); memcpy(dst_p, stream_state->rs.read_buffer, (size_t)stream_state->rs.length_field); - free_read_buffer(s); + null_and_maybe_free_read_buffer(s); grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer); grpc_slice_buffer_add(&stream_state->rs.read_slice_buffer, read_data_slice); @@ -1096,6 +1132,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, stream_state->rs.remaining_bytes); + stream_state->pending_read_from_cronet = true; result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_op->recv_trailing_metadata && @@ -1153,15 +1190,6 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, make a note */ if (stream_op->recv_message) stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE] = true; - } else if (stream_state->fail_state && !stream_state->flush_read) { - CRONET_LOG(GPR_DEBUG, "running: %p flush read", oas); - if (stream_state->rs.read_buffer && - stream_state->rs.read_buffer != stream_state->rs.grpc_header_bytes) { - gpr_free(stream_state->rs.read_buffer); - stream_state->rs.read_buffer = NULL; - } - stream_state->rs.read_buffer = gpr_malloc(4096); - stream_state->flush_read = true; } else { result = NO_ACTION_POSSIBLE; } @@ -1190,7 +1218,9 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->state.fail_state = s->state.flush_read = false; s->state.cancel_error = NULL; s->state.flush_cronet_when_ready = s->state.pending_write_for_trailer = false; - s->state.unprocessed_send_message = false; + s->state.pending_send_message = false; + s->state.pending_recv_trailing_metadata = false; + s->state.pending_read_from_cronet = false; s->curr_gs = gs; s->curr_ct = (grpc_cronet_transport *)gt; @@ -1240,6 +1270,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, void *and_free_memory) { stream_obj *s = (stream_obj *)gs; + null_and_maybe_free_read_buffer(s); GRPC_ERROR_UNREF(s->state.cancel_error); } From b3f11af3a550d65877ab747a094634c70a6076a8 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 6 Mar 2017 18:51:08 -0800 Subject: [PATCH 045/188] Improve the way to reject :authority header --- .../cronet/transport/cronet_transport.c | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index ff31f2ffc56..fae4d724d7e 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -1239,32 +1239,22 @@ static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx, static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_transport_stream_op *op) { CRONET_LOG(GPR_DEBUG, "perform_stream_op"); - stream_obj *s = (stream_obj *)gs; - add_to_storage(s, op); if (op->send_initial_metadata && header_has_authority(op->send_initial_metadata->list.head)) { /* Cronet does not support :authority header field. We cancel the call when - this field is present in metadata */ - bidirectional_stream_header_array header_array; - bidirectional_stream_header *header; - bidirectional_stream cbs; - CRONET_LOG(GPR_DEBUG, - ":authority header is provided but not supported;" - " cancel operations"); - /* Notify application that operation is cancelled by forging trailers */ - header_array.count = 1; - header_array.capacity = 1; - header_array.headers = gpr_malloc(sizeof(bidirectional_stream_header)); - header = (bidirectional_stream_header *)header_array.headers; - header->key = "grpc-status"; - header->value = "1"; /* Return status GRPC_STATUS_CANCELLED */ - cbs.annotation = (void *)s; - s->state.state_op_done[OP_CANCEL_ERROR] = true; - on_response_trailers_received(&cbs, &header_array); - gpr_free(header_array.headers); - } else { - execute_from_storage(s); + this field is present in metadata */ + if (op->recv_initial_metadata_ready) { + grpc_closure_sched(exec_ctx, op->recv_initial_metadata_ready, GRPC_ERROR_CANCELLED); + } + if (op->recv_message_ready) { + grpc_closure_sched(exec_ctx, op->recv_message_ready, GRPC_ERROR_CANCELLED); + } + grpc_closure_sched(exec_ctx, op->on_complete, GRPC_ERROR_CANCELLED); + return; } + stream_obj *s = (stream_obj *)gs; + add_to_storage(s, op); + execute_from_storage(s); } static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, From 39a797387f1a7bbfd5cad235a90d1059bfddfdba Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 6 Mar 2017 18:52:24 -0800 Subject: [PATCH 046/188] clang-format --- .../transport/cronet/transport/cronet_transport.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index fae4d724d7e..1759fe7ad2d 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -268,7 +268,8 @@ static void maybe_flush_read(stream_obj *s) { s->state.rs.read_buffer = gpr_malloc(GRPC_FLUSH_READ_SIZE); if (!s->state.pending_read_from_cronet) { CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); - bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, GRPC_FLUSH_READ_SIZE); + bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, + GRPC_FLUSH_READ_SIZE); s->state.pending_read_from_cronet = true; } } @@ -532,7 +533,8 @@ static void on_read_completed(bidirectional_stream *stream, char *data, s->state.state_callback_received[OP_RECV_MESSAGE] = true; if (count > 0 && s->state.flush_read) { CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); - bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, GRPC_FLUSH_READ_SIZE); + bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, + GRPC_FLUSH_READ_SIZE); s->state.pending_read_from_cronet = true; gpr_mu_unlock(&s->mu); } else if (count > 0) { @@ -1244,10 +1246,12 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, /* Cronet does not support :authority header field. We cancel the call when this field is present in metadata */ if (op->recv_initial_metadata_ready) { - grpc_closure_sched(exec_ctx, op->recv_initial_metadata_ready, GRPC_ERROR_CANCELLED); + grpc_closure_sched(exec_ctx, op->recv_initial_metadata_ready, + GRPC_ERROR_CANCELLED); } if (op->recv_message_ready) { - grpc_closure_sched(exec_ctx, op->recv_message_ready, GRPC_ERROR_CANCELLED); + grpc_closure_sched(exec_ctx, op->recv_message_ready, + GRPC_ERROR_CANCELLED); } grpc_closure_sched(exec_ctx, op->on_complete, GRPC_ERROR_CANCELLED); return; From f26236a159300c92633fafd0a906691165bc3289 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Mar 2017 09:43:51 -0800 Subject: [PATCH 047/188] Silence accept4 message when its irrelevant --- src/core/lib/iomgr/tcp_server_posix.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index 36f878fdd4d..b93e9c410ac 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -114,6 +114,8 @@ struct grpc_tcp_server { /* is this server shutting down? */ bool shutdown; + /* have listeners been shutdown? */ + bool shutdown_listeners; /* use SO_REUSEPORT */ bool so_reuseport; /* expand wildcard addresses to a list of all local addresses */ @@ -422,7 +424,12 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure); return; default: - gpr_log(GPR_ERROR, "Failed accept4: %s", strerror(errno)); + if (!sp->server->shutdown_listeners) { + gpr_log(GPR_ERROR, "Failed accept4: %s", strerror(errno)); + } else { + /* if we have shutdown listeners, accept4 could fail, and we + needn't notify users */ + } goto error; } } @@ -438,11 +445,6 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { grpc_fd *fdobj = grpc_fd_create(fd, name); - if (read_notifier_pollset == NULL) { - gpr_log(GPR_ERROR, "Read notifier pollset is not set on the fd"); - goto error; - } - grpc_pollset_add_fd(exec_ctx, read_notifier_pollset, fdobj); // Create acceptor. @@ -941,6 +943,7 @@ void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { void grpc_tcp_server_shutdown_listeners(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { gpr_mu_lock(&s->mu); + s->shutdown_listeners = true; /* shutdown all fd's */ if (s->active_ports) { grpc_tcp_listener *sp; From 702a445dc199bd3308bbb70b94ef50386cedd215 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Mar 2017 09:53:00 -0800 Subject: [PATCH 048/188] Fix initialization --- src/core/lib/iomgr/tcp_server_posix.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index b93e9c410ac..a17f6d48eff 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -163,7 +163,7 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, grpc_tcp_server **server) { gpr_once_init(&check_init, init); - grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server)); + grpc_tcp_server *s = gpr_zalloc(sizeof(grpc_tcp_server)); s->so_reuseport = has_so_reuseport; s->resource_quota = grpc_resource_quota_create(NULL); s->expand_wildcard_addrs = false; From f0d6b88626223e2f8b77072fb628fdcf4a5f87ab Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 6 Mar 2017 14:20:26 -0800 Subject: [PATCH 049/188] Avoid repetitive division calculations --- test/core/support/cpu_test.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/core/support/cpu_test.c b/test/core/support/cpu_test.c index ca0fe0ccb50..7b9bf6c5e1d 100644 --- a/test/core/support/cpu_test.c +++ b/test/core/support/cpu_test.c @@ -81,9 +81,12 @@ static void worker_thread(void *arg) { uint32_t cpu; unsigned r = 12345678; unsigned i, j; - for (i = 0; i < 1000 / grpc_test_slowdown_factor(); i++) { + /* Avoid repetitive division calculations */ + int64_t max_i = 1000 / grpc_test_slowdown_factor(); + int64_t max_j = 1000000 / grpc_test_slowdown_factor(); + for (i = 0; i < max_i; i++) { /* run for a bit - just calculate something random. */ - for (j = 0; j < 1000000 / grpc_test_slowdown_factor(); j++) { + for (j = 0; j < max_j; j++) { r = (r * 17) & ((r - i) | (r * i)); } cpu = gpr_cpu_current_cpu(); From e3e3b2aec8fc7ac45f3b8f7d0adfdb7cdce2bd1f Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 7 Mar 2017 16:12:56 -0800 Subject: [PATCH 050/188] changes for picking up well known protos Add dependency and protoc argument for picking up well known protos from the //third_party/protobuf repo --- bazel/cc_grpc_library.bzl | 7 ++++++- bazel/generate_cc.bzl | 17 ++++++++++++++++- bazel/grpc_build_system.bzl | 4 +++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/bazel/cc_grpc_library.bzl b/bazel/cc_grpc_library.bzl index b9d7eb5902b..ab1add476e1 100644 --- a/bazel/cc_grpc_library.bzl +++ b/bazel/cc_grpc_library.bzl @@ -2,7 +2,7 @@ load("//:bazel/generate_cc.bzl", "generate_cc") -def cc_grpc_library(name, srcs, deps, proto_only, use_external = False, **kwargs): +def cc_grpc_library(name, srcs, deps, proto_only, well_known_protos, use_external = False, **kwargs): """Generates C++ grpc classes from a .proto file. Assumes the generated classes will be used in cc_api_version = 2. @@ -12,6 +12,9 @@ def cc_grpc_library(name, srcs, deps, proto_only, use_external = False, **kwargs srcs: a single proto_library, which wraps the .proto files with services. deps: a list of C++ proto_library (or cc_proto_library) which provides the compiled code of any message that the services depend on. + well_known_protos: The target from protobuf library that exports well + known protos. Currently it will only work if the value is + "@submodule_protobuf//:well_known_protos" use_external: When True the grpc deps are prefixed with //external. This allows grpc to be used as a dependency in other bazel projects. **kwargs: rest of arguments, e.g., compatible_with and visibility. @@ -35,6 +38,7 @@ def cc_grpc_library(name, srcs, deps, proto_only, use_external = False, **kwargs generate_cc( name = codegen_target, srcs = [proto_target], + well_known_protos = well_known_protos, **kwargs ) @@ -49,6 +53,7 @@ def cc_grpc_library(name, srcs, deps, proto_only, use_external = False, **kwargs name = codegen_grpc_target, srcs = [proto_target], plugin = plugin, + well_known_protos = well_known_protos, **kwargs ) diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl index d49cbe8d728..f3961f0a419 100644 --- a/bazel/generate_cc.bzl +++ b/bazel/generate_cc.bzl @@ -31,8 +31,20 @@ def generate_cc_impl(ctx): arguments += ["-I{0}={0}".format(include.path) for include in includes] arguments += [proto.path for proto in protos] + # create a list of well known proto files if the argument is non-None + well_known_proto_files = [] + if ctx.attr.well_known_protos: + f = ctx.attr.well_known_protos.files.to_list()[0].dirname + if f != "external/submodule_protobuf/src/google/protobuf": + print("Error: Only @submodule_protobuf//:well_known_protos is supported") + else: + # f points to "external/submodule_protobuf/src/google/protobuf" + # add -I argument to protoc so it knows where to look for the proto files. + arguments += ["-I{0}".format(f + "/../..")] + well_known_proto_files = [f for f in ctx.attr.well_known_protos.files] + ctx.action( - inputs = protos + includes + additional_input, + inputs = protos + includes + additional_input + well_known_proto_files, outputs = out_files, executable = ctx.executable._protoc, arguments = arguments, @@ -56,6 +68,9 @@ generate_cc = rule( mandatory = False, allow_empty = True, ), + "well_known_protos" : attr.label( + mandatory = False, + ), "_protoc": attr.label( default = Label("//external:protocol_compiler"), executable = True, diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index 855d2d7b723..8b524bd0e52 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -58,11 +58,13 @@ def grpc_proto_plugin(name, srcs = [], deps = []): load("//:bazel/cc_grpc_library.bzl", "cc_grpc_library") -def grpc_proto_library(name, srcs = [], deps = [], well_known_deps = [], has_services = True, use_external = False): +def grpc_proto_library(name, srcs = [], deps = [], well_known_protos = None, + has_services = True, use_external = False): cc_grpc_library( name = name, srcs = srcs, deps = deps, + well_known_protos = well_known_protos, proto_only = not has_services, use_external = use_external, ) From 144446434507dcc6d4583a04e723904df5cd2718 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Mar 2017 16:21:55 -0800 Subject: [PATCH 051/188] Fix locking --- src/core/lib/iomgr/tcp_server_posix.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index a17f6d48eff..5f286a6723c 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -424,12 +424,14 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure); return; default: + gpr_mu_lock(&sp->server->mu); if (!sp->server->shutdown_listeners) { gpr_log(GPR_ERROR, "Failed accept4: %s", strerror(errno)); } else { /* if we have shutdown listeners, accept4 could fail, and we needn't notify users */ } + gpr_mu_unlock(&sp->server->mu); goto error; } } From eb24a7bd847409091893e0f75f97fecaff99ebcf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Mar 2017 17:04:46 -0800 Subject: [PATCH 052/188] Use new diff script --- tools/profiling/microbenchmarks/bm_diff.py | 0 tools/run_tests/run_microbenchmark.py | 18 +++++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 tools/profiling/microbenchmarks/bm_diff.py diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 3a9461ecd3b..0b5fc5ec468 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -228,8 +228,10 @@ for bm_name in args.benchmarks: for collect in args.collect: collectors[collect](bm_name, args) if args.diff_perf: - for bm_name in args.benchmarks: - run_summary(bm_name, 'opt', '%s.new' % bm_name) + if 'summary' not in args.collect: + for bm_name in args.benchmarks: + run_summary(bm_name, 'opt', bm_name) + run_summary(bm_name, 'counters', bm_name) where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() subprocess.check_call(['git', 'checkout', args.diff_perf]) comparables = [] @@ -238,15 +240,21 @@ if args.diff_perf: for bm_name in args.benchmarks: try: run_summary(bm_name, 'opt', '%s.old' % bm_name) + run_summary(bm_name, 'counters', '%s.old' % bm_name) comparables.append(bm_name) except subprocess.CalledProcessError, e: pass finally: subprocess.check_call(['git', 'checkout', where_am_i]) for bm_name in comparables: - subprocess.check_call(['third_party/benchmark/tools/compare_bench.py', - '%s.new.opt.json' % bm_name, - '%s.old.opt.json' % bm_name]) + diff = subprocess.check_output(['tools/profiling/microbenchmarks/bm_diff.py', + '%s.opt.json' % bm_name, + '%s.counters.json' % bm_name, + '%s.old.opt.json' % bm_name, + '%s.old.counters.json' % bm_name]).strip() + if diff: + heading('Performance diff: %s' % bm_name) + text(diff) index_html += "\n\n" with open('reports/index.html', 'w') as f: From 1fabdfb870d29c1fad40a882d1029494d9fcb425 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 7 Mar 2017 17:18:31 -0800 Subject: [PATCH 053/188] Add more Kokoro job configs --- tools/internal_ci/linux/grpc_master.sh | 2 + .../linux/grpc_master_sanitizers.cfg | 39 +++++++++++++++ .../linux/grpc_master_sanitizers.sh | 48 ++++++++++++++++++ tools/internal_ci/linux/grpc_portability.cfg | 39 +++++++++++++++ tools/internal_ci/linux/grpc_portability.sh | 49 +++++++++++++++++++ .../windows/grpc_portability_master.bat | 43 ++++++++++++++++ .../windows/grpc_portability_master.cfg | 39 +++++++++++++++ 7 files changed, 259 insertions(+) create mode 100644 tools/internal_ci/linux/grpc_master_sanitizers.cfg create mode 100644 tools/internal_ci/linux/grpc_master_sanitizers.sh create mode 100644 tools/internal_ci/linux/grpc_portability.cfg create mode 100644 tools/internal_ci/linux/grpc_portability.sh create mode 100644 tools/internal_ci/windows/grpc_portability_master.bat create mode 100644 tools/internal_ci/windows/grpc_portability_master.cfg diff --git a/tools/internal_ci/linux/grpc_master.sh b/tools/internal_ci/linux/grpc_master.sh index d01d6375e9a..9ecf1239594 100755 --- a/tools/internal_ci/linux/grpc_master.sh +++ b/tools/internal_ci/linux/grpc_master.sh @@ -45,6 +45,8 @@ ulimit -n 2000 git submodule update --init +# download docker images from dockerhub +export DOCKERHUB_ORGANIZATION=grpctesting tools/run_tests/run_tests.py -l c -t -x sponge_log.xml || FAILED="true" # kill port_server.py to prevent the build from hanging diff --git a/tools/internal_ci/linux/grpc_master_sanitizers.cfg b/tools/internal_ci/linux/grpc_master_sanitizers.cfg new file mode 100644 index 00000000000..64ad23e2142 --- /dev/null +++ b/tools/internal_ci/linux/grpc_master_sanitizers.cfg @@ -0,0 +1,39 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_master_sanitizers.sh" +timeout_mins: 1440 +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/tools/internal_ci/linux/grpc_master_sanitizers.sh b/tools/internal_ci/linux/grpc_master_sanitizers.sh new file mode 100644 index 00000000000..b174c9b1749 --- /dev/null +++ b/tools/internal_ci/linux/grpc_master_sanitizers.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +git submodule update --init + +# download docker images from dockerhub +export DOCKERHUB_ORGANIZATION=grpctesting +tools/jenkins/run_jenkins_matrix.sh -f sanitizers linux + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_portability.cfg b/tools/internal_ci/linux/grpc_portability.cfg new file mode 100644 index 00000000000..e6e9a9085b4 --- /dev/null +++ b/tools/internal_ci/linux/grpc_portability.cfg @@ -0,0 +1,39 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_portability.sh" +timeout_mins: 360 +action { + define_artifacts { + regex: "**report**.xml" + } +} diff --git a/tools/internal_ci/linux/grpc_portability.sh b/tools/internal_ci/linux/grpc_portability.sh new file mode 100644 index 00000000000..a7460e5a50b --- /dev/null +++ b/tools/internal_ci/linux/grpc_portability.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +git submodule update --init + +# download docker images from dockerhub +export DOCKERHUB_ORGANIZATION=grpctesting +tools/jenkins/run_jenkins_matrix.sh -f portability linux + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi + diff --git a/tools/internal_ci/windows/grpc_portability_master.bat b/tools/internal_ci/windows/grpc_portability_master.bat new file mode 100644 index 00000000000..b98c70146c4 --- /dev/null +++ b/tools/internal_ci/windows/grpc_portability_master.bat @@ -0,0 +1,43 @@ +@rem Copyright 2017, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +@rem make sure msys binaries are preferred over cygwin binaries +@rem set path to python 2.7 +set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% + +@rem enter repo root +cd /d %~dp0\..\..\.. + +git submodule update --init + +python tools/run_tests/run_tests_matrix.py -f portability windows -j 1 --inner_jobs 8 || goto :error +goto :EOF + +:error +exit /b %errorlevel% diff --git a/tools/internal_ci/windows/grpc_portability_master.cfg b/tools/internal_ci/windows/grpc_portability_master.cfg new file mode 100644 index 00000000000..fa6fd8b276e --- /dev/null +++ b/tools/internal_ci/windows/grpc_portability_master.cfg @@ -0,0 +1,39 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_portability_master.bat" +timeout_mins: 360 +action { + define_artifacts { + regex: "**sponge_log.xml" + } +} From 76fc48bc5cff0886493ff39ecbdd3db8a84ceb91 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Mar 2017 17:21:16 -0800 Subject: [PATCH 054/188] Start of diff script --- tools/profiling/microbenchmarks/bm_diff.py | 47 ++++++++++++++++++++++ 1 file changed, 47 insertions(+) mode change 100644 => 100755 tools/profiling/microbenchmarks/bm_diff.py diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py old mode 100644 new mode 100755 index e69de29bb2d..4dbcc768f76 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python2.7 + +import sys +import json +import bm_json + +with open(sys.argv[1]) as f: + js_new_ctr = json.loads(f.read()) +with open(sys.argv[2]) as f: + js_new_opt = json.loads(f.read()) +with open(sys.argv[3]) as f: + js_old_ctr = json.loads(f.read()) +with open(sys.argv[4]) as f: + js_old_opt = json.loads(f.read()) + +new = {} +old = {} + +for row in bm_json.expand_json(js_new_ctr, js_new_opt): + new[row['cpp_name']] = row +for row in bm_json.expand_json(js_old_ctr, js_old_opt): + old[row['cpp_name']] = row + +def min_change(pct): + return lambda n, o: abs((n-o)/o - 1) > pct/100 + +_INTERESTING = ( + ('cpu_time', min_change(5)), + ('real_time', min_change(5)), +) + +for bm in sorted(new.keys()): + if bm not in old: continue + hdr = False + n = new[bm] + o = old[bm] + print n + print o + for fld, chk in _INTERESTING: + if fld not in n or fld not in o: continue + if chk(n[fld], o[fld]): + if not hdr: + print '%s shows changes:' % bm + hdr = True + print ' %s changed %r --> %r' % (fld, o[fld], n[fld]) + sys.exit(0) + From 8809afd0abdbbc83d45d225955dff064903334cf Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 7 Mar 2017 17:29:00 -0800 Subject: [PATCH 055/188] Update regex to find sponge_log.xml --- tools/internal_ci/linux/grpc_master.cfg | 2 +- tools/internal_ci/linux/grpc_master_sanitizers.cfg | 2 +- tools/internal_ci/linux/grpc_portability.cfg | 2 +- tools/internal_ci/linux/grpc_portability_build_only.cfg | 2 +- tools/internal_ci/linux/grpc_portability_build_only.sh | 1 - tools/internal_ci/linux/grpc_pull_request_sanity.cfg | 2 +- tools/internal_ci/windows/grpc_master.cfg | 2 +- tools/internal_ci/windows/grpc_portability_master.cfg | 2 +- 8 files changed, 7 insertions(+), 8 deletions(-) diff --git a/tools/internal_ci/linux/grpc_master.cfg b/tools/internal_ci/linux/grpc_master.cfg index 7536a91a679..6c94c3b4d8f 100644 --- a/tools/internal_ci/linux/grpc_master.cfg +++ b/tools/internal_ci/linux/grpc_master.cfg @@ -34,6 +34,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_master.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/*sponge_log.xml" } } diff --git a/tools/internal_ci/linux/grpc_master_sanitizers.cfg b/tools/internal_ci/linux/grpc_master_sanitizers.cfg index 64ad23e2142..a2a9407128b 100644 --- a/tools/internal_ci/linux/grpc_master_sanitizers.cfg +++ b/tools/internal_ci/linux/grpc_master_sanitizers.cfg @@ -34,6 +34,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_master_sanitizers.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/*sponge_log.xml" } } diff --git a/tools/internal_ci/linux/grpc_portability.cfg b/tools/internal_ci/linux/grpc_portability.cfg index e6e9a9085b4..0c25661d624 100644 --- a/tools/internal_ci/linux/grpc_portability.cfg +++ b/tools/internal_ci/linux/grpc_portability.cfg @@ -34,6 +34,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_portability.sh" timeout_mins: 360 action { define_artifacts { - regex: "**report**.xml" + regex: "**/*sponge_log.xml" } } diff --git a/tools/internal_ci/linux/grpc_portability_build_only.cfg b/tools/internal_ci/linux/grpc_portability_build_only.cfg index ce5be5abe97..4d3dda40822 100644 --- a/tools/internal_ci/linux/grpc_portability_build_only.cfg +++ b/tools/internal_ci/linux/grpc_portability_build_only.cfg @@ -34,6 +34,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_portability_build_only.sh" timeout_mins: 180 action { define_artifacts { - regex: "**report**.xml" + regex: "**/*sponge_log.xml" } } diff --git a/tools/internal_ci/linux/grpc_portability_build_only.sh b/tools/internal_ci/linux/grpc_portability_build_only.sh index ebdc0e82d76..92678c15f80 100644 --- a/tools/internal_ci/linux/grpc_portability_build_only.sh +++ b/tools/internal_ci/linux/grpc_portability_build_only.sh @@ -37,5 +37,4 @@ git submodule update --init # download docker images from dockerhub export DOCKERHUB_ORGANIZATION=grpctesting - tools/jenkins/run_jenkins_matrix.sh -f portability linux --build_only diff --git a/tools/internal_ci/linux/grpc_pull_request_sanity.cfg b/tools/internal_ci/linux/grpc_pull_request_sanity.cfg index 511f2d6b35b..1abf6ac600b 100644 --- a/tools/internal_ci/linux/grpc_pull_request_sanity.cfg +++ b/tools/internal_ci/linux/grpc_pull_request_sanity.cfg @@ -34,6 +34,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_sanity.sh" timeout_mins: 30 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/*sponge_log.xml" } } diff --git a/tools/internal_ci/windows/grpc_master.cfg b/tools/internal_ci/windows/grpc_master.cfg index f90af113085..21a9d6a985d 100644 --- a/tools/internal_ci/windows/grpc_master.cfg +++ b/tools/internal_ci/windows/grpc_master.cfg @@ -34,6 +34,6 @@ build_file: "grpc/tools/internal_ci/windows/grpc_master.bat" timeout_mins: 360 action { define_artifacts { - regex: "**sponge_log.xml" + regex: "**/*sponge_log.xml" } } diff --git a/tools/internal_ci/windows/grpc_portability_master.cfg b/tools/internal_ci/windows/grpc_portability_master.cfg index fa6fd8b276e..10d8e985910 100644 --- a/tools/internal_ci/windows/grpc_portability_master.cfg +++ b/tools/internal_ci/windows/grpc_portability_master.cfg @@ -34,6 +34,6 @@ build_file: "grpc/tools/internal_ci/windows/grpc_portability_master.bat" timeout_mins: 360 action { define_artifacts { - regex: "**sponge_log.xml" + regex: "**/*sponge_log.xml" } } From b38aac4c76013c19e406ba2ac5e23699dbcd1c63 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Wed, 8 Mar 2017 08:05:42 +0000 Subject: [PATCH 056/188] Bump protobuf to 3.2.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bf87de07f89..12969958aee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ coverage>=4.0 cython>=0.23 enum34>=1.0.4 futures>=2.2.0 -protobuf>=3.0.0 +protobuf>=3.2.0 six>=1.10 wheel>=0.29 From 4751c28b0b371eaa6fd17583e06e740989b62b86 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 10 Jan 2017 14:29:00 -0800 Subject: [PATCH 057/188] Add channel arguments to server builder --- CMakeLists.txt | 6 ++ Makefile | 6 ++ build.yaml | 2 + include/grpc++/impl/channel_argument_option.h | 52 +++++++++++++ include/grpc++/server_builder.h | 8 ++ src/cpp/server/channel_argument_option.cc | 78 +++++++++++++++++++ tools/doxygen/Doxyfile.c++ | 1 + tools/doxygen/Doxyfile.c++.internal | 2 + .../generated/sources_and_headers.json | 3 + vsprojects/vcxproj/grpc++/grpc++.vcxproj | 3 + .../vcxproj/grpc++/grpc++.vcxproj.filters | 6 ++ .../grpc++_unsecure/grpc++_unsecure.vcxproj | 3 + .../grpc++_unsecure.vcxproj.filters | 6 ++ 13 files changed, 176 insertions(+) create mode 100644 include/grpc++/impl/channel_argument_option.h create mode 100644 src/cpp/server/channel_argument_option.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 94d578eebda..d4c2792523c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2078,6 +2078,7 @@ add_library(grpc++ src/cpp/common/rpc_method.cc src/cpp/common/version_cc.cc src/cpp/server/async_generic_service.cc + src/cpp/server/channel_argument_option.cc src/cpp/server/create_default_thread_pool.cc src/cpp/server/dynamic_thread_pool.cc src/cpp/server/health/default_health_check_service.cc @@ -2143,6 +2144,7 @@ foreach(_hdr include/grpc++/grpc++.h include/grpc++/health_check_service_interface.h include/grpc++/impl/call.h + include/grpc++/impl/channel_argument_option.h include/grpc++/impl/client_unary_call.h include/grpc++/impl/codegen/core_codegen.h include/grpc++/impl/grpc_library.h @@ -2264,6 +2266,7 @@ add_library(grpc++_cronet src/cpp/common/rpc_method.cc src/cpp/common/version_cc.cc src/cpp/server/async_generic_service.cc + src/cpp/server/channel_argument_option.cc src/cpp/server/create_default_thread_pool.cc src/cpp/server/dynamic_thread_pool.cc src/cpp/server/health/default_health_check_service.cc @@ -2513,6 +2516,7 @@ foreach(_hdr include/grpc++/grpc++.h include/grpc++/health_check_service_interface.h include/grpc++/impl/call.h + include/grpc++/impl/channel_argument_option.h include/grpc++/impl/client_unary_call.h include/grpc++/impl/codegen/core_codegen.h include/grpc++/impl/grpc_library.h @@ -2935,6 +2939,7 @@ add_library(grpc++_unsecure src/cpp/common/rpc_method.cc src/cpp/common/version_cc.cc src/cpp/server/async_generic_service.cc + src/cpp/server/channel_argument_option.cc src/cpp/server/create_default_thread_pool.cc src/cpp/server/dynamic_thread_pool.cc src/cpp/server/health/default_health_check_service.cc @@ -3000,6 +3005,7 @@ foreach(_hdr include/grpc++/grpc++.h include/grpc++/health_check_service_interface.h include/grpc++/impl/call.h + include/grpc++/impl/channel_argument_option.h include/grpc++/impl/client_unary_call.h include/grpc++/impl/codegen/core_codegen.h include/grpc++/impl/grpc_library.h diff --git a/Makefile b/Makefile index 57e877beb01..9542f17de9e 100644 --- a/Makefile +++ b/Makefile @@ -3919,6 +3919,7 @@ LIBGRPC++_SRC = \ src/cpp/common/rpc_method.cc \ src/cpp/common/version_cc.cc \ src/cpp/server/async_generic_service.cc \ + src/cpp/server/channel_argument_option.cc \ src/cpp/server/create_default_thread_pool.cc \ src/cpp/server/dynamic_thread_pool.cc \ src/cpp/server/health/default_health_check_service.cc \ @@ -3951,6 +3952,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/grpc++.h \ include/grpc++/health_check_service_interface.h \ include/grpc++/impl/call.h \ + include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/core_codegen.h \ include/grpc++/impl/grpc_library.h \ @@ -4118,6 +4120,7 @@ LIBGRPC++_CRONET_SRC = \ src/cpp/common/rpc_method.cc \ src/cpp/common/version_cc.cc \ src/cpp/server/async_generic_service.cc \ + src/cpp/server/channel_argument_option.cc \ src/cpp/server/create_default_thread_pool.cc \ src/cpp/server/dynamic_thread_pool.cc \ src/cpp/server/health/default_health_check_service.cc \ @@ -4333,6 +4336,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/grpc++.h \ include/grpc++/health_check_service_interface.h \ include/grpc++/impl/call.h \ + include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/core_codegen.h \ include/grpc++/impl/grpc_library.h \ @@ -4801,6 +4805,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/cpp/common/rpc_method.cc \ src/cpp/common/version_cc.cc \ src/cpp/server/async_generic_service.cc \ + src/cpp/server/channel_argument_option.cc \ src/cpp/server/create_default_thread_pool.cc \ src/cpp/server/dynamic_thread_pool.cc \ src/cpp/server/health/default_health_check_service.cc \ @@ -4833,6 +4838,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/grpc++.h \ include/grpc++/health_check_service_interface.h \ include/grpc++/impl/call.h \ + include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/core_codegen.h \ include/grpc++/impl/grpc_library.h \ diff --git a/build.yaml b/build.yaml index 9ff37d59e17..5d96524f271 100644 --- a/build.yaml +++ b/build.yaml @@ -776,6 +776,7 @@ filegroups: - include/grpc++/grpc++.h - include/grpc++/health_check_service_interface.h - include/grpc++/impl/call.h + - include/grpc++/impl/channel_argument_option.h - include/grpc++/impl/client_unary_call.h - include/grpc++/impl/codegen/core_codegen.h - include/grpc++/impl/grpc_library.h @@ -832,6 +833,7 @@ filegroups: - src/cpp/common/rpc_method.cc - src/cpp/common/version_cc.cc - src/cpp/server/async_generic_service.cc + - src/cpp/server/channel_argument_option.cc - src/cpp/server/create_default_thread_pool.cc - src/cpp/server/dynamic_thread_pool.cc - src/cpp/server/health/default_health_check_service.cc diff --git a/include/grpc++/impl/channel_argument_option.h b/include/grpc++/impl/channel_argument_option.h new file mode 100644 index 00000000000..057acc2cebf --- /dev/null +++ b/include/grpc++/impl/channel_argument_option.h @@ -0,0 +1,52 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPCXX_IMPL_CHANNEL_ARGUMENT_OPTION_H +#define GRPCXX_IMPL_CHANNEL_ARGUMENT_OPTION_H + +#include +#include + +#include +#include + +namespace grpc { + +std::unique_ptr MakeChannelArgumentOption( + const grpc::string &name, const grpc::string &value); +std::unique_ptr MakeChannelArgumentOption( + const grpc::string &name, int value); + +} // namespace grpc + +#endif // GRPCXX_IMPL_CHANNEL_ARGUMENT_OPTION_H diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index 2ac2f0a1ef4..d707100a522 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -39,6 +39,7 @@ #include #include +#include #include #include #include @@ -130,6 +131,13 @@ class ServerBuilder { /// Only useful if this is a Synchronous server. ServerBuilder& SetSyncServerOption(SyncServerOption option, int value); + /// Add a channel argument (an escape hatch to tuning core library parameters + /// directly) + template + ServerBuilder& AddChannelArgument(const grpc::string& arg, const T& value) { + return SetOption(MakeChannelArgumentOption(arg, value)); + } + /// Tries to bind \a server to the given \a addr. /// /// It can be invoked multiple times. diff --git a/src/cpp/server/channel_argument_option.cc b/src/cpp/server/channel_argument_option.cc new file mode 100644 index 00000000000..723f968ff86 --- /dev/null +++ b/src/cpp/server/channel_argument_option.cc @@ -0,0 +1,78 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include + +namespace grpc { + +std::unique_ptr MakeChannelArgumentOption( + const grpc::string &name, const grpc::string &value) { + class StringOption final : public ServerBuilderOption { + public: + StringOption(const grpc::string &name, const grpc::string &value) + : name_(name), value_(value) {} + + virtual void UpdateArguments(ChannelArguments *args) override { + args->SetString(name_, value_); + } + virtual void UpdatePlugins( + std::vector> *plugins) override {} + + private: + const grpc::string name_; + const grpc::string value_; + }; + return std::unique_ptr(new StringOption(name, value)); +} + +std::unique_ptr MakeChannelArgumentOption( + const grpc::string &name, int value) { + class IntOption final : public ServerBuilderOption { + public: + IntOption(const grpc::string &name, int value) + : name_(name), value_(value) {} + + virtual void UpdateArguments(ChannelArguments *args) override { + args->SetInt(name_, value_); + } + virtual void UpdatePlugins( + std::vector> *plugins) override {} + + private: + const grpc::string name_; + const int value_; + }; + return std::unique_ptr(new IntOption(name, value)); +} + +} // namespace grpc diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 551ba46b3b8..b0ff5e7f8b6 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -801,6 +801,7 @@ include/grpc++/generic/generic_stub.h \ include/grpc++/grpc++.h \ include/grpc++/health_check_service_interface.h \ include/grpc++/impl/call.h \ +include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 0d7ddfc6d67..5d3e5ed24fc 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -801,6 +801,7 @@ include/grpc++/generic/generic_stub.h \ include/grpc++/grpc++.h \ include/grpc++/health_check_service_interface.h \ include/grpc++/impl/call.h \ +include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ @@ -912,6 +913,7 @@ src/cpp/common/secure_channel_arguments.cc \ src/cpp/common/secure_create_auth_context.cc \ src/cpp/common/version_cc.cc \ src/cpp/server/async_generic_service.cc \ +src/cpp/server/channel_argument_option.cc \ src/cpp/server/create_default_thread_pool.cc \ src/cpp/server/dynamic_thread_pool.cc \ src/cpp/server/dynamic_thread_pool.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 15bcf5621e3..99942609dcf 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8253,6 +8253,7 @@ "include/grpc++/grpc++.h", "include/grpc++/health_check_service_interface.h", "include/grpc++/impl/call.h", + "include/grpc++/impl/channel_argument_option.h", "include/grpc++/impl/client_unary_call.h", "include/grpc++/impl/codegen/core_codegen.h", "include/grpc++/impl/grpc_library.h", @@ -8309,6 +8310,7 @@ "include/grpc++/grpc++.h", "include/grpc++/health_check_service_interface.h", "include/grpc++/impl/call.h", + "include/grpc++/impl/channel_argument_option.h", "include/grpc++/impl/client_unary_call.h", "include/grpc++/impl/codegen/core_codegen.h", "include/grpc++/impl/grpc_library.h", @@ -8358,6 +8360,7 @@ "src/cpp/common/rpc_method.cc", "src/cpp/common/version_cc.cc", "src/cpp/server/async_generic_service.cc", + "src/cpp/server/channel_argument_option.cc", "src/cpp/server/create_default_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.h", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 45f3037e8ab..c85454a6dda 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -270,6 +270,7 @@ + @@ -414,6 +415,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index 95cdb7434c1..d800b297e9f 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -70,6 +70,9 @@ src\cpp\server + + src\cpp\server + src\cpp\server @@ -162,6 +165,9 @@ include\grpc++\impl + + include\grpc++\impl + include\grpc++\impl diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 22ea361a02e..45c0ecd35d0 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -270,6 +270,7 @@ + @@ -398,6 +399,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index c3cef2d4df5..987d56d6a09 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -55,6 +55,9 @@ src\cpp\server + + src\cpp\server + src\cpp\server @@ -147,6 +150,9 @@ include\grpc++\impl + + include\grpc++\impl + include\grpc++\impl From 12352b2c5c4f1876586fec8148d4b2405d0c7e36 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 10 Jan 2017 15:16:14 -0800 Subject: [PATCH 058/188] Add test of server builder, including channel args --- Makefile | 55 +++++ build.yaml | 15 ++ src/cpp/server/server_builder.cc | 8 +- src/cpp/server/server_cc.cc | 2 +- test/cpp/server/server_builder_test.cc | 96 ++++++++ .../generated/sources_and_headers.json | 24 ++ tools/run_tests/generated/tests.json | 22 ++ .../server_builder_test.vcxproj | 223 ++++++++++++++++++ .../server_builder_test.vcxproj.filters | 39 +++ 9 files changed, 482 insertions(+), 2 deletions(-) create mode 100644 test/cpp/server/server_builder_test.cc create mode 100644 vsprojects/vcxproj/test/server_builder_test/server_builder_test.vcxproj create mode 100644 vsprojects/vcxproj/test/server_builder_test/server_builder_test.vcxproj.filters diff --git a/Makefile b/Makefile index 9542f17de9e..e41f1bee8a4 100644 --- a/Makefile +++ b/Makefile @@ -1100,6 +1100,7 @@ round_robin_end2end_test: $(BINDIR)/$(CONFIG)/round_robin_end2end_test secure_auth_context_test: $(BINDIR)/$(CONFIG)/secure_auth_context_test secure_sync_unary_ping_pong_test: $(BINDIR)/$(CONFIG)/secure_sync_unary_ping_pong_test server_builder_plugin_test: $(BINDIR)/$(CONFIG)/server_builder_plugin_test +server_builder_test: $(BINDIR)/$(CONFIG)/server_builder_test server_context_test_spouse_test: $(BINDIR)/$(CONFIG)/server_context_test_spouse_test server_crash_test: $(BINDIR)/$(CONFIG)/server_crash_test server_crash_test_client: $(BINDIR)/$(CONFIG)/server_crash_test_client @@ -1507,6 +1508,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/secure_auth_context_test \ $(BINDIR)/$(CONFIG)/secure_sync_unary_ping_pong_test \ $(BINDIR)/$(CONFIG)/server_builder_plugin_test \ + $(BINDIR)/$(CONFIG)/server_builder_test \ $(BINDIR)/$(CONFIG)/server_context_test_spouse_test \ $(BINDIR)/$(CONFIG)/server_crash_test \ $(BINDIR)/$(CONFIG)/server_crash_test_client \ @@ -1618,6 +1620,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/secure_auth_context_test \ $(BINDIR)/$(CONFIG)/secure_sync_unary_ping_pong_test \ $(BINDIR)/$(CONFIG)/server_builder_plugin_test \ + $(BINDIR)/$(CONFIG)/server_builder_test \ $(BINDIR)/$(CONFIG)/server_context_test_spouse_test \ $(BINDIR)/$(CONFIG)/server_crash_test \ $(BINDIR)/$(CONFIG)/server_crash_test_client \ @@ -1971,6 +1974,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/secure_sync_unary_ping_pong_test || ( echo test secure_sync_unary_ping_pong_test failed ; exit 1 ) $(E) "[RUN] Testing server_builder_plugin_test" $(Q) $(BINDIR)/$(CONFIG)/server_builder_plugin_test || ( echo test server_builder_plugin_test failed ; exit 1 ) + $(E) "[RUN] Testing server_builder_test" + $(Q) $(BINDIR)/$(CONFIG)/server_builder_test || ( echo test server_builder_test failed ; exit 1 ) $(E) "[RUN] Testing server_context_test_spouse_test" $(Q) $(BINDIR)/$(CONFIG)/server_context_test_spouse_test || ( echo test server_context_test_spouse_test failed ; exit 1 ) $(E) "[RUN] Testing server_crash_test" @@ -14833,6 +14838,56 @@ endif endif +SERVER_BUILDER_TEST_SRC = \ + $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc \ + $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc \ + test/cpp/server/server_builder_test.cc \ + +SERVER_BUILDER_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(SERVER_BUILDER_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/server_builder_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/server_builder_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/server_builder_test: $(PROTOBUF_DEP) $(SERVER_BUILDER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(SERVER_BUILDER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/server_builder_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/echo_messages.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a + +$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/echo.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a + +$(OBJDIR)/$(CONFIG)/test/cpp/server/server_builder_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_server_builder_test: $(SERVER_BUILDER_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(SERVER_BUILDER_TEST_OBJS:.o=.dep) +endif +endif +$(OBJDIR)/$(CONFIG)/test/cpp/server/server_builder_test.o: $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc + + SERVER_CONTEXT_TEST_SPOUSE_TEST_SRC = \ test/cpp/test/server_context_test_spouse_test.cc \ diff --git a/build.yaml b/build.yaml index 5d96524f271..e27e3019535 100644 --- a/build.yaml +++ b/build.yaml @@ -3810,6 +3810,21 @@ targets: - grpc - gpr_test_util - gpr +- name: server_builder_test + gtest: true + build: test + language: c++ + src: + - src/proto/grpc/testing/echo_messages.proto + - src/proto/grpc/testing/echo.proto + - test/cpp/server/server_builder_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - gpr_test_util + - grpc++ + - grpc + - gpr - name: server_context_test_spouse_test gtest: true build: test diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc index 00a90bb184c..4eb4b5a1b21 100644 --- a/src/cpp/server/server_builder.cc +++ b/src/cpp/server/server_builder.cc @@ -323,9 +323,14 @@ std::unique_ptr ServerBuilder::BuildAndStart() { } } + bool added_port = false; for (auto port = ports_.begin(); port != ports_.end(); port++) { int r = server->AddListeningPort(port->addr, port->creds.get()); - if (!r) return nullptr; + if (!r) { + if (added_port) server->Shutdown(); + return nullptr; + } + added_port = true; if (port->selected_port != nullptr) { *port->selected_port = r; } @@ -333,6 +338,7 @@ std::unique_ptr ServerBuilder::BuildAndStart() { auto cqs_data = cqs_.empty() ? nullptr : &cqs_[0]; if (!server->Start(cqs_data, cqs_.size())) { + if (added_port) server->Shutdown(); return nullptr; } diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index 9e11a8a9e07..e874892e73c 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -534,7 +534,7 @@ bool Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) { void Server::ShutdownInternal(gpr_timespec deadline) { std::unique_lock lock(mu_); - if (started_ && !shutdown_) { + if (!shutdown_) { shutdown_ = true; /// The completion queue to use for server shutdown completion notification diff --git a/test/cpp/server/server_builder_test.cc b/test/cpp/server/server_builder_test.cc new file mode 100644 index 00000000000..1d9eda17b40 --- /dev/null +++ b/test/cpp/server/server_builder_test.cc @@ -0,0 +1,96 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include + +#include +#include + +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/core/util/port.h" + +namespace grpc { +namespace { + +testing::EchoTestService::Service g_service; + +grpc::string MakePort() { + std::ostringstream s; + int p = grpc_pick_unused_port_or_die(); + s << "localhost:" << p; + return s.str(); +} + +grpc::string g_port = MakePort(); + +TEST(ServerBuilderTest, NoOp) { ServerBuilder b; } + +TEST(ServerBuilderTest, CreateServerNoPorts) { + ServerBuilder().RegisterService(&g_service).BuildAndStart()->Shutdown(); +} + +TEST(ServerBuilderTest, CreateServerOnePort) { + ServerBuilder() + .RegisterService(&g_service) + .AddListeningPort(g_port, InsecureServerCredentials()) + .BuildAndStart() + ->Shutdown(); +} + +TEST(ServerBuilderTest, CreateServerRepeatedPort) { + ServerBuilder() + .RegisterService(&g_service) + .AddListeningPort(g_port, InsecureServerCredentials()) + .AddListeningPort(g_port, InsecureServerCredentials()) + .BuildAndStart() + ->Shutdown(); +} + +TEST(ServerBuilderTest, CreateServerRepeatedPortWithDisallowedReusePort) { + EXPECT_EQ(ServerBuilder() + .RegisterService(&g_service) + .AddListeningPort(g_port, InsecureServerCredentials()) + .AddListeningPort(g_port, InsecureServerCredentials()) + .AddChannelArgument(GRPC_ARG_ALLOW_REUSEPORT, 0) + .BuildAndStart(), + nullptr); +} + +} // namespace +} // namespace grpc + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 99942609dcf..866bff8dac6 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -3455,6 +3455,30 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [ + "src/proto/grpc/testing/echo.grpc.pb.h", + "src/proto/grpc/testing/echo.pb.h", + "src/proto/grpc/testing/echo_messages.grpc.pb.h", + "src/proto/grpc/testing/echo_messages.pb.h" + ], + "is_filegroup": false, + "language": "c++", + "name": "server_builder_test", + "src": [ + "test/cpp/server/server_builder_test.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index eca65ac5334..d7221b11770 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -3350,6 +3350,28 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "server_builder_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/vcxproj/test/server_builder_test/server_builder_test.vcxproj b/vsprojects/vcxproj/test/server_builder_test/server_builder_test.vcxproj new file mode 100644 index 00000000000..ebbfd59f3d3 --- /dev/null +++ b/vsprojects/vcxproj/test/server_builder_test/server_builder_test.vcxproj @@ -0,0 +1,223 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {60523734-00BD-765B-5A5B-19E19A2E31B8} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + server_builder_test + static + Debug + static + Debug + + + server_builder_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + {0BE77741-552A-929B-A497-4EF7ECE17A64} + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/server_builder_test/server_builder_test.vcxproj.filters b/vsprojects/vcxproj/test/server_builder_test/server_builder_test.vcxproj.filters new file mode 100644 index 00000000000..c323b7a6731 --- /dev/null +++ b/vsprojects/vcxproj/test/server_builder_test/server_builder_test.vcxproj.filters @@ -0,0 +1,39 @@ + + + + + src\proto\grpc\testing + + + src\proto\grpc\testing + + + test\cpp\server + + + + + + {828e0ffc-a89a-de93-ae06-706d522188a1} + + + {538db689-e85f-c369-7020-8d78e0ee5049} + + + {351168ef-9b4f-6165-ff4f-0e13781910db} + + + {530a1a67-0a37-50f8-42d0-7ccf0ec34cfc} + + + {67afe178-6a18-fd24-bbe6-656fee5a5f10} + + + {aaa8777b-1bc3-abaa-5e6d-28040c5aa213} + + + {af770080-f515-c773-3ae0-243d5929bbd0} + + + + From 69709e2819f5eeeb47a16f7ed964dd1196b94ff0 Mon Sep 17 00:00:00 2001 From: Robbie Shade Date: Wed, 8 Mar 2017 12:54:28 -0500 Subject: [PATCH 059/188] Fix flaky use-after-free in udp_server --- src/core/lib/iomgr/udp_server.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index 2a1c8d39fae..d1bcd89af1f 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -485,7 +485,11 @@ void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, grpc_schedule_on_exec_ctx); grpc_fd_notify_on_write(exec_ctx, sp->emfd, &sp->write_closure); - s->active_ports++; + /* Registered for both read and write callbacks: increment active_ports + * twice to account for this, and delay free-ing of memory until both + * on_read and on_write have fired. */ + s->active_ports += 2; + sp = sp->next; } From abc886dc64b674d52f4f341681eb6625e283b957 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 8 Mar 2017 10:58:48 -0800 Subject: [PATCH 060/188] Use the right encoding overhead number in remote interop test with Cronet --- src/objective-c/tests/InteropTests.m | 6 ------ .../InteropTestsRemoteWithCronet.m | 4 ++++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m index 5584246ad94..766fa19d857 100644 --- a/src/objective-c/tests/InteropTests.m +++ b/src/objective-c/tests/InteropTests.m @@ -169,8 +169,6 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } -#ifndef GRPC_COMPILE_WITH_CRONET -// TODO (mxyan): Fix this test - (void)testResponsesOverMaxSizeFailWithActionableMessage { XCTAssertNotNil(self.class.host); __weak XCTestExpectation *expectation = [self expectationWithDescription:@"ResponseOverMaxSize"]; @@ -191,7 +189,6 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } -#endif - (void)testResponsesOver4MBAreAcceptedIfOptedIn { XCTAssertNotNil(self.class.host); @@ -327,8 +324,6 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } -#ifndef GRPC_COMPILE_WITH_CRONET -// TODO(makdharma@): Fix this test - (void)testEmptyStreamRPC { XCTAssertNotNil(self.class.host); __weak XCTestExpectation *expectation = [self expectationWithDescription:@"EmptyStream"]; @@ -342,7 +337,6 @@ }]; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } -#endif - (void)testCancelAfterBeginRPC { XCTAssertNotNil(self.class.host); diff --git a/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m b/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m index fab8ad8d25f..7bc303ac10d 100644 --- a/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m +++ b/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m @@ -47,4 +47,8 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; return kRemoteSSLHost; } +- (int32_t)encodingOverhead { + return 12; // bytes +} + @end From 1d88979a57752ef38d3d6022deca6a8c0839ede1 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 8 Mar 2017 20:12:40 +0100 Subject: [PATCH 061/188] Delete TestResult.xml --- src/csharp/Grpc.Core.Tests/TestResult.xml | 41 ----------------------- 1 file changed, 41 deletions(-) delete mode 100644 src/csharp/Grpc.Core.Tests/TestResult.xml diff --git a/src/csharp/Grpc.Core.Tests/TestResult.xml b/src/csharp/Grpc.Core.Tests/TestResult.xml deleted file mode 100644 index 13da80739ce..00000000000 --- a/src/csharp/Grpc.Core.Tests/TestResult.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 3464f7069477fab1d25bb589d71e624ce06a28a8 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 8 Mar 2017 20:50:57 +0100 Subject: [PATCH 062/188] dont silence realtime output from run_tests.py --use_docker --- tools/run_tests/run_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index d4983403cc0..76651d41157 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1298,7 +1298,9 @@ if args.use_docker: if not args.travis: env['TTY_FLAG'] = '-t' # enables Ctrl-C when not on Jenkins. - run_shell_command('tools/run_tests/dockerize/build_docker_and_run_tests.sh', env=env) + subprocess.check_call('tools/run_tests/dockerize/build_docker_and_run_tests.sh', + shell=True, + env=env) sys.exit(0) _check_arch_option(args.arch) From e2bdbfe075d2accdf9269ab60cafffc8ba420a59 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 8 Mar 2017 11:55:07 -0800 Subject: [PATCH 063/188] Fix misplaced assert in run_tests.py --- tools/run_tests/run_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index d4983403cc0..61de2c93a9c 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1465,10 +1465,9 @@ def _build_and_run( sample_size = int(num_jobs * args.sample_percent/100.0) massaged_one_run = random.sample(massaged_one_run, sample_size) if not isclose(args.sample_percent, 100.0): + assert args.runs_per_test == 1, "Can't do sampling (-p) over multiple runs (-n)." print("Running %d tests out of %d (~%d%%)" % (sample_size, num_jobs, args.sample_percent)) - else: - assert args.runs_per_test == 1, "Can't do sampling (-p) over multiple runs (-n)." if infinite_runs: assert len(massaged_one_run) > 0, 'Must have at least one test for a -n inf run' runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs From 5faabc45c5d74b62953121282cd55f45f41b64e4 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 7 Mar 2017 18:04:28 -0800 Subject: [PATCH 064/188] Fix pyenv dockerfile --- templates/tools/dockerfile/apt_get_pyenv.include | 3 +++ tools/dockerfile/push_testing_images.sh | 2 +- tools/dockerfile/test/python_pyenv_x64/Dockerfile | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/templates/tools/dockerfile/apt_get_pyenv.include b/templates/tools/dockerfile/apt_get_pyenv.include index 816b27904f6..ef0964e5975 100644 --- a/templates/tools/dockerfile/apt_get_pyenv.include +++ b/templates/tools/dockerfile/apt_get_pyenv.include @@ -12,6 +12,9 @@ RUN apt-get update && apt-get install -y ${'\\'} # Install Pyenv and dev Python versions 3.5 and 3.6 RUN curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash +ENV PATH /root/.pyenv/bin:$PATH +RUN eval "$(pyenv init -)" +RUN eval "$(pyenv virtualenv-init -)" RUN pyenv update RUN pyenv install 3.5-dev RUN pyenv install 3.6-dev diff --git a/tools/dockerfile/push_testing_images.sh b/tools/dockerfile/push_testing_images.sh index 9dceb29a877..973e045ffe4 100755 --- a/tools/dockerfile/push_testing_images.sh +++ b/tools/dockerfile/push_testing_images.sh @@ -44,7 +44,7 @@ cd - DOCKERHUB_ORGANIZATION=grpctesting -for DOCKERFILE_DIR in tools/dockerfile/test/fuzzer tools/dockerfile/test/sanity +for DOCKERFILE_DIR in tools/dockerfile/test/* do # Generate image name based on Dockerfile checksum. That works well as long # as can count on dockerfiles being written in a way that changing the logical diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile index ecd785a86d3..a596e42d2ce 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile @@ -92,6 +92,9 @@ RUN apt-get update && apt-get install -y \ # Install Pyenv and dev Python versions 3.5 and 3.6 RUN curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash +ENV PATH /root/.pyenv/bin:$PATH +RUN eval "$(pyenv init -)" +RUN eval "$(pyenv virtualenv-init -)" RUN pyenv update RUN pyenv install 3.5-dev RUN pyenv install 3.6-dev From 332bcc377cb2b03064e8ab807edb60b0808e2667 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Mar 2017 12:47:46 -0800 Subject: [PATCH 065/188] Fix sanity --- CMakeLists.txt | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index e1fc86dfd46..145935f1dab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -653,6 +653,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx secure_sync_unary_ping_pong_test) endif() add_dependencies(buildtests_cxx server_builder_plugin_test) +add_dependencies(buildtests_cxx server_builder_test) add_dependencies(buildtests_cxx server_context_test_spouse_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx server_crash_test) @@ -9668,6 +9669,55 @@ target_link_libraries(server_builder_plugin_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +add_executable(server_builder_test + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.grpc.pb.h + test/cpp/server/server_builder_test.cc + third_party/googletest/src/gtest-all.cc +) + +protobuf_generate_grpc_cpp( + src/proto/grpc/testing/echo_messages.proto +) +protobuf_generate_grpc_cpp( + src/proto/grpc/testing/echo.proto +) + +target_include_directories(server_builder_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(server_builder_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_executable(server_context_test_spouse_test test/cpp/test/server_context_test_spouse_test.cc third_party/googletest/src/gtest-all.cc From b432885a565cac6cd9510c8ff05ed9eb2cc0458e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Mar 2017 13:08:40 -0800 Subject: [PATCH 066/188] Always write index.html --- tools/run_tests/run_microbenchmark.py | 71 ++++++++++++++------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index e735ff8bf9a..09f2eb180ed 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -228,38 +228,39 @@ argp.add_argument('--summary_time', help='Minimum time to run benchmarks for the summary collection') args = argp.parse_args() -for bm_name in args.benchmarks: - for collect in args.collect: - collectors[collect](bm_name, args) -if args.diff_perf: - if 'summary' not in args.collect: - for bm_name in args.benchmarks: - run_summary(bm_name, 'opt', bm_name) - run_summary(bm_name, 'counters', bm_name) - where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() - subprocess.check_call(['git', 'checkout', args.diff_perf]) - comparables = [] - subprocess.check_call(['make', 'clean']) - try: - for bm_name in args.benchmarks: - try: - run_summary(bm_name, 'opt', '%s.old' % bm_name) - run_summary(bm_name, 'counters', '%s.old' % bm_name) - comparables.append(bm_name) - except subprocess.CalledProcessError, e: - pass - finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - for bm_name in comparables: - diff = subprocess.check_output(['tools/profiling/microbenchmarks/bm_diff.py', - '%s.opt.json' % bm_name, - '%s.counters.json' % bm_name, - '%s.old.opt.json' % bm_name, - '%s.old.counters.json' % bm_name]).strip() - if diff: - heading('Performance diff: %s' % bm_name) - text(diff) - -index_html += "\n\n" -with open('reports/index.html', 'w') as f: - f.write(index_html) +try: + for bm_name in args.benchmarks: + for collect in args.collect: + collectors[collect](bm_name, args) + if args.diff_perf: + if 'summary' not in args.collect: + for bm_name in args.benchmarks: + run_summary(bm_name, 'opt', bm_name) + run_summary(bm_name, 'counters', bm_name) + where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + subprocess.check_call(['git', 'checkout', args.diff_perf]) + comparables = [] + subprocess.check_call(['make', 'clean']) + try: + for bm_name in args.benchmarks: + try: + run_summary(bm_name, 'opt', '%s.old' % bm_name) + run_summary(bm_name, 'counters', '%s.old' % bm_name) + comparables.append(bm_name) + except subprocess.CalledProcessError, e: + pass + finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + for bm_name in comparables: + diff = subprocess.check_output(['tools/profiling/microbenchmarks/bm_diff.py', + '%s.opt.json' % bm_name, + '%s.counters.json' % bm_name, + '%s.old.opt.json' % bm_name, + '%s.old.counters.json' % bm_name]).strip() + if diff: + heading('Performance diff: %s' % bm_name) + text(diff) +finally: + index_html += "\n\n" + with open('reports/index.html', 'w') as f: + f.write(index_html) From 26995eb2ee277f098ea98f992e8257038f2dd3f8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Mar 2017 13:10:28 -0800 Subject: [PATCH 067/188] Get filename right --- tools/run_tests/run_microbenchmark.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 09f2eb180ed..137e0beb7eb 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -178,13 +178,15 @@ def run_summary(bm_name, cfg, base_json_name): def collect_summary(bm_name, args): heading('Summary: %s [no counters]' % bm_name) - text(run_summary(bm_name, 'opt', 'out')) + text(run_summary(bm_name, 'opt', bm_name)) heading('Summary: %s [with counters]' % bm_name) - text(run_summary(bm_name, 'counters', 'out')) + text(run_summary(bm_name, 'counters', bm_name)) if args.bigquery_upload: - with open('out.csv', 'w') as f: - f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.counters.json', 'out.opt.json'])) - subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', 'out.csv']) + with open('%s.csv' % bm_name, 'w') as f: + f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', + '%s.counters.json' % bm_name, + '%s.opt.json' % bm_name])) + subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name]) collectors = { 'latency': collect_latency, From 5adc93e0ba993b3f43d7982aa738e98f1f777abb Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Mar 2017 13:17:53 -0800 Subject: [PATCH 068/188] Add more interesting metrics --- tools/profiling/microbenchmarks/bm_diff.py | 10 +++++++--- tools/run_tests/run_microbenchmark.py | 6 +++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 4dbcc768f76..c9e721369ae 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -25,8 +25,13 @@ def min_change(pct): return lambda n, o: abs((n-o)/o - 1) > pct/100 _INTERESTING = ( - ('cpu_time', min_change(5)), - ('real_time', min_change(5)), + ('cpu_time', min_change(10)), + ('real_time', min_change(10)), + ('locks_per_iteration', min_change(5)), + ('allocs_per_iteration', min_change(5)), + ('writes_per_iteration', min_change(5)), + ('atm_cas_per_iteration', min_change(1)), + ('atm_add_per_iteration', min_change(5)), ) for bm in sorted(new.keys()): @@ -44,4 +49,3 @@ for bm in sorted(new.keys()): hdr = True print ' %s changed %r --> %r' % (fld, o[fld], n[fld]) sys.exit(0) - diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 137e0beb7eb..f41bde113a8 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -255,10 +255,10 @@ try: subprocess.check_call(['git', 'checkout', where_am_i]) for bm_name in comparables: diff = subprocess.check_output(['tools/profiling/microbenchmarks/bm_diff.py', - '%s.opt.json' % bm_name, '%s.counters.json' % bm_name, - '%s.old.opt.json' % bm_name, - '%s.old.counters.json' % bm_name]).strip() + '%s.opt.json' % bm_name, + '%s.old.counters.json' % bm_name, + '%s.old.opt.json' % bm_name]).strip() if diff: heading('Performance diff: %s' % bm_name) text(diff) From 47c56a677ee2abe3089fe53a7f81cd0e772491d7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Mar 2017 13:45:53 -0800 Subject: [PATCH 069/188] A better benchmark diff script --- tools/profiling/microbenchmarks/bm_diff.py | 40 +++++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index c9e721369ae..5e1f68bc37d 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -3,6 +3,7 @@ import sys import json import bm_json +import tabulate with open(sys.argv[1]) as f: js_new_ctr = json.loads(f.read()) @@ -21,8 +22,11 @@ for row in bm_json.expand_json(js_new_ctr, js_new_opt): for row in bm_json.expand_json(js_old_ctr, js_old_opt): old[row['cpp_name']] = row +def changed_ratio(n, o): + return float(n-o)/float(o) + def min_change(pct): - return lambda n, o: abs((n-o)/o - 1) > pct/100 + return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 _INTERESTING = ( ('cpu_time', min_change(10)), @@ -34,18 +38,36 @@ _INTERESTING = ( ('atm_add_per_iteration', min_change(5)), ) +changed = [] +for fld, chk in _INTERESTING: + for bm in new.keys(): + if bm not in old: continue + n = new[bm] + o = old[bm] + if fld not in n or fld not in o: continue + if chk(n[fld], o[fld]): + changed.append((fld, chk)) + break + +headers = ['Benchmark'] + [c[0] for c in changed] + ['Details'] +rows = [] for bm in sorted(new.keys()): if bm not in old: continue - hdr = False + row = [bm] + any_changed = False n = new[bm] o = old[bm] - print n - print o + details = '' for fld, chk in _INTERESTING: if fld not in n or fld not in o: continue if chk(n[fld], o[fld]): - if not hdr: - print '%s shows changes:' % bm - hdr = True - print ' %s changed %r --> %r' % (fld, o[fld], n[fld]) - sys.exit(0) + row.append(changed_ratio(n[fld], o[fld])) + if details: details += ', ' + details += '%s:%r-->%r' % (fld, o[fld], n[fld]) + any_changed = True + else: + row.append('') + if any_changed: + row.append(details) + rows.append(row) +print tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') From b97e82b179fa1aeb077cf0575a982866afe17203 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 8 Mar 2017 14:20:42 -0800 Subject: [PATCH 070/188] Change Kokoro scripts to use run_tests_matrix.py --- tools/internal_ci/linux/grpc_master_sanitizers.sh | 10 +--------- tools/internal_ci/linux/grpc_portability.sh | 11 +---------- .../internal_ci/linux/grpc_portability_build_only.sh | 2 +- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/tools/internal_ci/linux/grpc_master_sanitizers.sh b/tools/internal_ci/linux/grpc_master_sanitizers.sh index b174c9b1749..2cd0efe1588 100644 --- a/tools/internal_ci/linux/grpc_master_sanitizers.sh +++ b/tools/internal_ci/linux/grpc_master_sanitizers.sh @@ -37,12 +37,4 @@ git submodule update --init # download docker images from dockerhub export DOCKERHUB_ORGANIZATION=grpctesting -tools/jenkins/run_jenkins_matrix.sh -f sanitizers linux - -# kill port_server.py to prevent the build from hanging -ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 - -if [ "$FAILED" != "" ] -then - exit 1 -fi +tools/run_tests/run_tests_matrix.sh -f sanitizers linux diff --git a/tools/internal_ci/linux/grpc_portability.sh b/tools/internal_ci/linux/grpc_portability.sh index a7460e5a50b..c7ac3bbcbee 100644 --- a/tools/internal_ci/linux/grpc_portability.sh +++ b/tools/internal_ci/linux/grpc_portability.sh @@ -37,13 +37,4 @@ git submodule update --init # download docker images from dockerhub export DOCKERHUB_ORGANIZATION=grpctesting -tools/jenkins/run_jenkins_matrix.sh -f portability linux - -# kill port_server.py to prevent the build from hanging -ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 - -if [ "$FAILED" != "" ] -then - exit 1 -fi - +tools/run_tests/run_tests_matrix.sh -f portability linux diff --git a/tools/internal_ci/linux/grpc_portability_build_only.sh b/tools/internal_ci/linux/grpc_portability_build_only.sh index 92678c15f80..edd6f916872 100644 --- a/tools/internal_ci/linux/grpc_portability_build_only.sh +++ b/tools/internal_ci/linux/grpc_portability_build_only.sh @@ -37,4 +37,4 @@ git submodule update --init # download docker images from dockerhub export DOCKERHUB_ORGANIZATION=grpctesting -tools/jenkins/run_jenkins_matrix.sh -f portability linux --build_only +tools/run_tests/run_tests_matrix.sh -f portability linux --build_only From 88a41bfa9d1260c7ad4d4b1610394d65b8311fbe Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 8 Mar 2017 14:54:11 -0800 Subject: [PATCH 071/188] Add comment to encodingOverhead --- src/objective-c/tests/InteropTests.m | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m index 766fa19d857..d964f53e8e2 100644 --- a/src/objective-c/tests/InteropTests.m +++ b/src/objective-c/tests/InteropTests.m @@ -90,6 +90,9 @@ return nil; } +// This number indicates how many bytes of overhead does Protocol Buffers encoding add onto the +// message. The number varies as different message.proto is used on different servers. The actual +// number for each interop server is overridden in corresponding derived test classes. - (int32_t)encodingOverhead { return 0; } From 55337bb317ab22e662ffebba71078a4e8def7127 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Thu, 2 Mar 2017 15:12:33 -0800 Subject: [PATCH 072/188] Error arena optimization --- src/core/lib/iomgr/error.c | 419 ++++++++++++++++++--------- src/core/lib/iomgr/error.h | 11 +- src/core/lib/iomgr/error_internal.h | 24 +- src/core/lib/transport/error_utils.c | 22 +- test/core/iomgr/error_test.c | 157 +++++++--- test/cpp/microbenchmarks/bm_error.cc | 20 -- 6 files changed, 430 insertions(+), 223 deletions(-) diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index 6c4d0157c20..42c2f3e2dde 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -35,6 +35,7 @@ #include +#include #include #include #include @@ -47,46 +48,7 @@ #include "src/core/lib/iomgr/error_internal.h" #include "src/core/lib/profiling/timers.h" - -static void destroy_integer(void *key) {} - -static void *copy_integer(void *key) { return key; } - -static long compare_integers(void *key1, void *key2) { - return GPR_ICMP((uintptr_t)key1, (uintptr_t)key2); -} - -static void destroy_string(void *str) { gpr_free(str); } - -static void *copy_string(void *str) { return gpr_strdup(str); } - -static void destroy_err(void *err) { GRPC_ERROR_UNREF(err); } - -static void *copy_err(void *err) { return GRPC_ERROR_REF(err); } - -static void destroy_time(void *tm) { gpr_free(tm); } - -static gpr_timespec *box_time(gpr_timespec tm) { - gpr_timespec *out = gpr_malloc(sizeof(*out)); - *out = tm; - return out; -} - -static void *copy_time(void *tm) { return box_time(*(gpr_timespec *)tm); } - -static const gpr_avl_vtable avl_vtable_ints = {destroy_integer, copy_integer, - compare_integers, - destroy_integer, copy_integer}; - -static const gpr_avl_vtable avl_vtable_strs = {destroy_integer, copy_integer, - compare_integers, destroy_string, - copy_string}; - -static const gpr_avl_vtable avl_vtable_times = { - destroy_integer, copy_integer, compare_integers, destroy_time, copy_time}; - -static const gpr_avl_vtable avl_vtable_errs = { - destroy_integer, copy_integer, compare_integers, destroy_err, copy_err}; +#include "src/core/lib/slice/slice_internal.h" static const char *error_int_name(grpc_error_ints key) { switch (key) { @@ -120,6 +82,8 @@ static const char *error_int_name(grpc_error_ints key) { return "limit"; case GRPC_ERROR_INT_OCCURRED_DURING_WRITE: return "occurred_during_write"; + case GRPC_ERROR_INT_MAX: + GPR_UNREACHABLE_CODE(return "unknown"); } GPR_UNREACHABLE_CODE(return "unknown"); } @@ -150,6 +114,8 @@ static const char *error_str_name(grpc_error_strs key) { return "filename"; case GRPC_ERROR_STR_QUEUED_BUFFERS: return "queued_buffers"; + case GRPC_ERROR_STR_MAX: + GPR_UNREACHABLE_CODE(return "unknown"); } GPR_UNREACHABLE_CODE(return "unknown"); } @@ -158,6 +124,8 @@ static const char *error_time_name(grpc_error_times key) { switch (key) { case GRPC_ERROR_TIME_CREATED: return "created"; + case GRPC_ERROR_TIME_MAX: + GPR_UNREACHABLE_CODE(return "unknown"); } GPR_UNREACHABLE_CODE(return "unknown"); } @@ -184,12 +152,36 @@ grpc_error *grpc_error_ref(grpc_error *err) { } #endif +static void unref_errs(grpc_error *err) { + uint8_t slot = err->first_err; + while (slot != UINT8_MAX) { + linked_error *lerr = (linked_error *)(err->arena + slot); + GRPC_ERROR_UNREF(lerr->err); + GPR_ASSERT(err->last_err == slot ? lerr->next == UINT8_MAX + : lerr->next != UINT8_MAX); + slot = lerr->next; + } +} + +static void unref_slice(grpc_slice slice) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_slice_unref_internal(&exec_ctx, slice); + grpc_exec_ctx_finish(&exec_ctx); +} + +static void unref_strs(grpc_error *err) { + for (size_t which = 0; which < GRPC_ERROR_STR_MAX; ++which) { + uint8_t slot = err->strs[which]; + if (slot != UINT8_MAX) { + unref_slice(*(grpc_slice *)(err->arena + slot)); + } + } +} + static void error_destroy(grpc_error *err) { GPR_ASSERT(!grpc_error_is_special(err)); - gpr_avl_unref(err->ints); - gpr_avl_unref(err->strs); - gpr_avl_unref(err->errs); - gpr_avl_unref(err->times); + unref_errs(err); + unref_strs(err); gpr_free((void *)gpr_atm_acq_load(&err->error_string)); gpr_free(err); } @@ -213,69 +205,188 @@ void grpc_error_unref(grpc_error *err) { } #endif +static uint8_t get_placement(grpc_error **err, size_t size) { + GPR_ASSERT(*err); + uint8_t slots = (uint8_t)(size / sizeof(intptr_t)); + if ((*err)->arena_size + slots > (*err)->arena_capacity) { + (*err)->arena_capacity = (uint8_t)(3 * (*err)->arena_capacity / 2); + *err = realloc( + *err, sizeof(grpc_error) + (*err)->arena_capacity * sizeof(intptr_t)); + } + uint8_t placement = (*err)->arena_size; + (*err)->arena_size = (uint8_t)((*err)->arena_size + slots); + return placement; +} + +static void internal_set_int(grpc_error **err, grpc_error_ints which, + intptr_t value) { + // GPR_ASSERT((*err)->ints[which] == UINT8_MAX); // TODO, enforce this + uint8_t slot = (*err)->ints[which]; + if (slot == UINT8_MAX) { + slot = get_placement(err, sizeof(value)); + } + (*err)->ints[which] = slot; + (*err)->arena[slot] = value; +} + +static void internal_set_str(grpc_error **err, grpc_error_strs which, + grpc_slice value) { + // GPR_ASSERT((*err)->strs[which] == UINT8_MAX); // TODO, enforce this + uint8_t slot = (*err)->strs[which]; + if (slot == UINT8_MAX) { + slot = get_placement(err, sizeof(value)); + } else { + unref_slice(*(grpc_slice *)((*err)->arena + slot)); + } + (*err)->strs[which] = slot; + memcpy((*err)->arena + slot, &value, sizeof(value)); +} + +static void internal_set_time(grpc_error **err, grpc_error_times which, + gpr_timespec value) { + // GPR_ASSERT((*err)->times[which] == UINT8_MAX); // TODO, enforce this + uint8_t slot = (*err)->times[which]; + if (slot == UINT8_MAX) { + slot = get_placement(err, sizeof(value)); + } + (*err)->times[which] = slot; + memcpy((*err)->arena + slot, &value, sizeof(value)); +} + +static void internal_add_error(grpc_error **err, grpc_error *new) { + linked_error new_last = {new, UINT8_MAX}; + uint8_t slot = get_placement(err, sizeof(linked_error)); + if ((*err)->first_err == UINT8_MAX) { + GPR_ASSERT((*err)->last_err == UINT8_MAX); + (*err)->last_err = slot; + (*err)->first_err = slot; + } else { + GPR_ASSERT((*err)->last_err != UINT8_MAX); + linked_error *old_last = (linked_error *)((*err)->arena + (*err)->last_err); + old_last->next = slot; + (*err)->last_err = slot; + } + memcpy((*err)->arena + slot, &new_last, sizeof(linked_error)); +} + +#define SLOTS_PER_INT (sizeof(intptr_t) / sizeof(intptr_t)) +#define SLOTS_PER_STR (sizeof(grpc_slice) / sizeof(intptr_t)) +#define SLOTS_PER_TIME (sizeof(gpr_timespec) / sizeof(intptr_t)) +#define SLOTS_PER_LINKED_ERROR (sizeof(linked_error) / sizeof(intptr_t)) + +// size of storing one int and two slices and a timespec. For line, desc, file, +// and time created +#define DEFAULT_ERROR_CAPACITY \ + (SLOTS_PER_INT + (SLOTS_PER_STR * 2) + SLOTS_PER_TIME) + +// It is very common to include and extra int and string in an error +#define SURPLUS_CAPACITY (2 * SLOTS_PER_INT + SLOTS_PER_TIME) + grpc_error *grpc_error_create(const char *file, int line, const char *desc, grpc_error **referencing, size_t num_referencing) { GPR_TIMER_BEGIN("grpc_error_create", 0); - grpc_error *err = gpr_malloc(sizeof(*err)); + uint8_t initial_arena_capacity = (uint8_t)( + DEFAULT_ERROR_CAPACITY + + (uint8_t)(num_referencing * SLOTS_PER_LINKED_ERROR) + SURPLUS_CAPACITY); + grpc_error *err = + gpr_malloc(sizeof(*err) + initial_arena_capacity * sizeof(intptr_t)); if (err == NULL) { // TODO(ctiller): make gpr_malloc return NULL return GRPC_ERROR_OOM; } #ifdef GRPC_ERROR_REFCOUNT_DEBUG gpr_log(GPR_DEBUG, "%p create [%s:%d]", err, file, line); #endif - err->ints = gpr_avl_add(gpr_avl_create(&avl_vtable_ints), - (void *)(uintptr_t)GRPC_ERROR_INT_FILE_LINE, - (void *)(uintptr_t)line); - err->strs = gpr_avl_add( - gpr_avl_add(gpr_avl_create(&avl_vtable_strs), - (void *)(uintptr_t)GRPC_ERROR_STR_FILE, gpr_strdup(file)), - (void *)(uintptr_t)GRPC_ERROR_STR_DESCRIPTION, gpr_strdup(desc)); - err->errs = gpr_avl_create(&avl_vtable_errs); - err->next_err = 0; - for (size_t i = 0; i < num_referencing; i++) { + + err->arena_size = 0; + err->arena_capacity = initial_arena_capacity; + err->first_err = UINT8_MAX; + err->last_err = UINT8_MAX; + + memset(err->ints, UINT8_MAX, GRPC_ERROR_INT_MAX); + memset(err->strs, UINT8_MAX, GRPC_ERROR_STR_MAX); + memset(err->times, UINT8_MAX, GRPC_ERROR_TIME_MAX); + + internal_set_int(&err, GRPC_ERROR_INT_FILE_LINE, line); + internal_set_str(&err, GRPC_ERROR_STR_FILE, + grpc_slice_from_static_string(file)); + internal_set_str( + &err, GRPC_ERROR_STR_DESCRIPTION, + grpc_slice_from_copied_buffer( + desc, + strlen(desc) + + 1)); // TODO, pull this up. // TODO(ncteisen), pull this up. + + for (size_t i = 0; i < num_referencing; ++i) { if (referencing[i] == GRPC_ERROR_NONE) continue; - err->errs = gpr_avl_add(err->errs, (void *)(err->next_err++), - GRPC_ERROR_REF(referencing[i])); + internal_add_error( + &err, + GRPC_ERROR_REF( + referencing[i])); // TODO(ncteisen), change ownership semantics } - err->times = gpr_avl_add(gpr_avl_create(&avl_vtable_times), - (void *)(uintptr_t)GRPC_ERROR_TIME_CREATED, - box_time(gpr_now(GPR_CLOCK_REALTIME))); + + internal_set_time(&err, GRPC_ERROR_TIME_CREATED, gpr_now(GPR_CLOCK_REALTIME)); + gpr_atm_no_barrier_store(&err->error_string, 0); gpr_ref_init(&err->refs, 1); GPR_TIMER_END("grpc_error_create", 0); return err; } +static void ref_strs(grpc_error *err) { + for (size_t i = 0; i < GRPC_ERROR_STR_MAX; ++i) { + uint8_t slot = err->strs[i]; + if (slot != UINT8_MAX) { + grpc_slice_ref_internal(*(grpc_slice *)(err->arena + slot)); + } + } +} + +static void ref_errs(grpc_error *err) { + uint8_t slot = err->first_err; + while (slot != UINT8_MAX) { + linked_error *lerr = (linked_error *)(err->arena + slot); + GRPC_ERROR_REF(lerr->err); + slot = lerr->next; + } +} + static grpc_error *copy_error_and_unref(grpc_error *in) { GPR_TIMER_BEGIN("copy_error_and_unref", 0); grpc_error *out; if (grpc_error_is_special(in)) { - if (in == GRPC_ERROR_NONE) - out = grpc_error_set_int(GRPC_ERROR_CREATE("no error"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_OK); - else if (in == GRPC_ERROR_OOM) - out = GRPC_ERROR_CREATE("oom"); - else if (in == GRPC_ERROR_CANCELLED) - out = - grpc_error_set_int(GRPC_ERROR_CREATE("cancelled"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_CANCELLED); - else - out = GRPC_ERROR_CREATE("unknown"); + out = GRPC_ERROR_CREATE("unknown"); + if (in == GRPC_ERROR_NONE) { + internal_set_str(&out, GRPC_ERROR_STR_DESCRIPTION, + grpc_slice_from_static_string("no error")); + internal_set_int(&out, GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_OK); + } else if (in == GRPC_ERROR_OOM) { + internal_set_str(&out, GRPC_ERROR_STR_DESCRIPTION, + grpc_slice_from_static_string("oom")); + } else if (in == GRPC_ERROR_CANCELLED) { + internal_set_str(&out, GRPC_ERROR_STR_DESCRIPTION, + grpc_slice_from_static_string("cancelled")); + internal_set_int(&out, GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_CANCELLED); + } } else if (gpr_ref_is_unique(&in->refs)) { return in; } else { - out = gpr_malloc(sizeof(*out)); + uint8_t new_arena_capacity = in->arena_capacity; + // the returned err will be added to, so we ensure this is room to avoid + // unneeded allocations. + if (in->arena_capacity - in->arena_size < (uint8_t)SLOTS_PER_STR) { + new_arena_capacity = (uint8_t)(3 * new_arena_capacity / 2); + } + out = gpr_malloc(sizeof(*in) + new_arena_capacity * sizeof(intptr_t)); #ifdef GRPC_ERROR_REFCOUNT_DEBUG gpr_log(GPR_DEBUG, "%p create copying %p", out, in); #endif - out->ints = gpr_avl_ref(in->ints); - out->strs = gpr_avl_ref(in->strs); - out->errs = gpr_avl_ref(in->errs); - out->times = gpr_avl_ref(in->times); + memcpy(out, in, sizeof(*in) + in->arena_size * sizeof(intptr_t)); + out->arena_capacity = new_arena_capacity; gpr_atm_no_barrier_store(&out->error_string, 0); - out->next_err = in->next_err; gpr_ref_init(&out->refs, 1); + ref_strs(out); + ref_errs(out); GRPC_ERROR_UNREF(in); } GPR_TIMER_END("copy_error_and_unref", 0); @@ -286,7 +397,7 @@ grpc_error *grpc_error_set_int(grpc_error *src, grpc_error_ints which, intptr_t value) { GPR_TIMER_BEGIN("grpc_error_set_int", 0); grpc_error *new = copy_error_and_unref(src); - new->ints = gpr_avl_add(new->ints, (void *)(uintptr_t)which, (void *)value); + internal_set_int(&new, which, value); GPR_TIMER_END("grpc_error_set_int", 0); return new; } @@ -304,7 +415,6 @@ static special_error_status_map error_status_map[] = { bool grpc_error_get_int(grpc_error *err, grpc_error_ints which, intptr_t *p) { GPR_TIMER_BEGIN("grpc_error_get_int", 0); - void *pp; if (grpc_error_is_special(err)) { if (which == GRPC_ERROR_INT_GRPC_STATUS) { for (size_t i = 0; i < GPR_ARRAY_SIZE(error_status_map); i++) { @@ -318,8 +428,9 @@ bool grpc_error_get_int(grpc_error *err, grpc_error_ints which, intptr_t *p) { GPR_TIMER_END("grpc_error_get_int", 0); return false; } - if (gpr_avl_maybe_get(err->ints, (void *)(uintptr_t)which, &pp)) { - if (p != NULL) *p = (intptr_t)pp; + uint8_t slot = err->ints[which]; + if (slot != UINT8_MAX) { + if (p != NULL) *p = err->arena[slot]; GPR_TIMER_END("grpc_error_get_int", 0); return true; } @@ -331,8 +442,9 @@ grpc_error *grpc_error_set_str(grpc_error *src, grpc_error_strs which, const char *value) { GPR_TIMER_BEGIN("grpc_error_set_str", 0); grpc_error *new = copy_error_and_unref(src); - new->strs = - gpr_avl_add(new->strs, (void *)(uintptr_t)which, gpr_strdup(value)); + internal_set_str(&new, which, + grpc_slice_from_copied_buffer( + value, strlen(value) + 1)); // TODO, pull this up. GPR_TIMER_END("grpc_error_set_str", 0); return new; } @@ -348,13 +460,19 @@ const char *grpc_error_get_str(grpc_error *err, grpc_error_strs which) { } return NULL; } - return gpr_avl_get(err->strs, (void *)(uintptr_t)which); + uint8_t slot = err->strs[which]; + if (slot != UINT8_MAX) { + return (const char *)GRPC_SLICE_START_PTR( + *(grpc_slice *)(err->arena + slot)); + } else { + return NULL; + } } grpc_error *grpc_error_add_child(grpc_error *src, grpc_error *child) { GPR_TIMER_BEGIN("grpc_error_add_child", 0); grpc_error *new = copy_error_and_unref(src); - new->errs = gpr_avl_add(new->errs, (void *)(new->next_err++), child); + internal_add_error(&new, child); GPR_TIMER_END("grpc_error_add_child", 0); return new; } @@ -374,42 +492,6 @@ typedef struct { size_t cap_kvs; } kv_pairs; -static void append_kv(kv_pairs *kvs, char *key, char *value) { - if (kvs->num_kvs == kvs->cap_kvs) { - kvs->cap_kvs = GPR_MAX(3 * kvs->cap_kvs / 2, 4); - kvs->kvs = gpr_realloc(kvs->kvs, sizeof(*kvs->kvs) * kvs->cap_kvs); - } - kvs->kvs[kvs->num_kvs].key = key; - kvs->kvs[kvs->num_kvs].value = value; - kvs->num_kvs++; -} - -static void collect_kvs(gpr_avl_node *node, char *key(void *k), - char *fmt(void *v), kv_pairs *kvs) { - if (node == NULL) return; - append_kv(kvs, key(node->key), fmt(node->value)); - collect_kvs(node->left, key, fmt, kvs); - collect_kvs(node->right, key, fmt, kvs); -} - -static char *key_int(void *p) { - return gpr_strdup(error_int_name((grpc_error_ints)(uintptr_t)p)); -} - -static char *key_str(void *p) { - return gpr_strdup(error_str_name((grpc_error_strs)(uintptr_t)p)); -} - -static char *key_time(void *p) { - return gpr_strdup(error_time_name((grpc_error_times)(uintptr_t)p)); -} - -static char *fmt_int(void *p) { - char *s; - gpr_asprintf(&s, "%" PRIdPTR, (intptr_t)p); - return s; -} - static void append_chr(char c, char **s, size_t *sz, size_t *cap) { if (*sz == *cap) { *cap = GPR_MAX(8, 3 * *cap / 2); @@ -461,6 +543,40 @@ static void append_esc_str(const char *str, char **s, size_t *sz, size_t *cap) { append_chr('"', s, sz, cap); } +static void append_kv(kv_pairs *kvs, char *key, char *value) { + if (kvs->num_kvs == kvs->cap_kvs) { + kvs->cap_kvs = GPR_MAX(3 * kvs->cap_kvs / 2, 4); + kvs->kvs = gpr_realloc(kvs->kvs, sizeof(*kvs->kvs) * kvs->cap_kvs); + } + kvs->kvs[kvs->num_kvs].key = key; + kvs->kvs[kvs->num_kvs].value = value; + kvs->num_kvs++; +} + +static char *key_int(grpc_error_ints which) { + return gpr_strdup(error_int_name(which)); +} + +static char *fmt_int(intptr_t p) { + char *s; + gpr_asprintf(&s, "%" PRIdPTR, p); + return s; +} + +static void collect_ints_kvs(grpc_error *err, kv_pairs *kvs) { + for (size_t which = 0; which < GRPC_ERROR_INT_MAX; ++which) { + uint8_t slot = err->ints[which]; + if (slot != UINT8_MAX) { + append_kv(kvs, key_int((grpc_error_ints)which), + fmt_int(err->arena[slot])); + } + } +} + +static char *key_str(grpc_error_strs which) { + return gpr_strdup(error_str_name(which)); +} + static char *fmt_str(void *p) { char *s = NULL; size_t sz = 0; @@ -470,8 +586,22 @@ static char *fmt_str(void *p) { return s; } -static char *fmt_time(void *p) { - gpr_timespec tm = *(gpr_timespec *)p; +static void collect_strs_kvs(grpc_error *err, kv_pairs *kvs) { + for (size_t which = 0; which < GRPC_ERROR_STR_MAX; ++which) { + uint8_t slot = err->strs[which]; + if (slot != UINT8_MAX) { + append_kv( + kvs, key_str((grpc_error_strs)which), + fmt_str(GRPC_SLICE_START_PTR(*(grpc_slice *)(err->arena + slot)))); + } + } +} + +static char *key_time(grpc_error_times which) { + return gpr_strdup(error_time_name(which)); +} + +static char *fmt_time(gpr_timespec tm) { char *out; char *pfx = "!!"; switch (tm.clock_type) { @@ -492,24 +622,37 @@ static char *fmt_time(void *p) { return out; } -static void add_errs(gpr_avl_node *n, char **s, size_t *sz, size_t *cap, - bool *first) { - if (n == NULL) return; - add_errs(n->left, s, sz, cap, first); - if (!*first) append_chr(',', s, sz, cap); - *first = false; - const char *e = grpc_error_string(n->value); - append_str(e, s, sz, cap); - add_errs(n->right, s, sz, cap, first); +static void collect_times_kvs(grpc_error *err, kv_pairs *kvs) { + for (size_t which = 0; which < GRPC_ERROR_TIME_MAX; ++which) { + uint8_t slot = err->times[which]; + if (slot != UINT8_MAX) { + append_kv(kvs, key_time((grpc_error_times)which), + fmt_time(*(gpr_timespec *)(err->arena + slot))); + } + } +} + +static void add_errs(grpc_error *err, char **s, size_t *sz, size_t *cap) { + uint8_t slot = err->first_err; + bool first = true; + while (slot != UINT8_MAX) { + linked_error *lerr = (linked_error *)(err->arena + slot); + if (!first) append_chr(',', s, sz, cap); + first = false; + const char *e = grpc_error_string(lerr->err); + append_str(e, s, sz, cap); + GPR_ASSERT(err->last_err == slot ? lerr->next == UINT8_MAX + : lerr->next != UINT8_MAX); + slot = lerr->next; + } } static char *errs_string(grpc_error *err) { char *s = NULL; size_t sz = 0; size_t cap = 0; - bool first = true; append_chr('[', &s, &sz, &cap); - add_errs(err->errs.root, &s, &sz, &cap, &first); + add_errs(err, &s, &sz, &cap); append_chr(']', &s, &sz, &cap); append_chr(0, &s, &sz, &cap); return s; @@ -557,10 +700,10 @@ const char *grpc_error_string(grpc_error *err) { kv_pairs kvs; memset(&kvs, 0, sizeof(kvs)); - collect_kvs(err->ints.root, key_int, fmt_int, &kvs); - collect_kvs(err->strs.root, key_str, fmt_str, &kvs); - collect_kvs(err->times.root, key_time, fmt_time, &kvs); - if (!gpr_avl_is_empty(err->errs)) { + collect_ints_kvs(err, &kvs); + collect_strs_kvs(err, &kvs); + collect_times_kvs(err, &kvs); + if (err->first_err != UINT8_MAX) { append_kv(&kvs, gpr_strdup("referenced_errors"), errs_string(err)); } diff --git a/src/core/lib/iomgr/error.h b/src/core/lib/iomgr/error.h index 2613512acbe..eb953947ae9 100644 --- a/src/core/lib/iomgr/error.h +++ b/src/core/lib/iomgr/error.h @@ -102,6 +102,9 @@ typedef enum { GRPC_ERROR_INT_LIMIT, /// chttp2: did the error occur while a write was in progress GRPC_ERROR_INT_OCCURRED_DURING_WRITE, + + /// Must always be last + GRPC_ERROR_INT_MAX, } grpc_error_ints; typedef enum { @@ -129,11 +132,17 @@ typedef enum { GRPC_ERROR_STR_KEY, /// value associated with the error GRPC_ERROR_STR_VALUE, + + /// Must always be last + GRPC_ERROR_STR_MAX, } grpc_error_strs; typedef enum { /// timestamp of error creation GRPC_ERROR_TIME_CREATED, + + /// Must always be last + GRPC_ERROR_TIME_MAX, } grpc_error_times; /// The following "special" errors can be propagated without allocating memory. @@ -184,8 +193,6 @@ void grpc_error_unref(grpc_error *err); grpc_error *grpc_error_set_int(grpc_error *src, grpc_error_ints which, intptr_t value) GRPC_MUST_USE_RESULT; bool grpc_error_get_int(grpc_error *error, grpc_error_ints which, intptr_t *p); -grpc_error *grpc_error_set_time(grpc_error *src, grpc_error_times which, - gpr_timespec value) GRPC_MUST_USE_RESULT; grpc_error *grpc_error_set_str(grpc_error *src, grpc_error_strs which, const char *value) GRPC_MUST_USE_RESULT; /// Returns NULL if the specified string is not set. diff --git a/src/core/lib/iomgr/error_internal.h b/src/core/lib/iomgr/error_internal.h index 1c89ead4ed6..bdebbe1792a 100644 --- a/src/core/lib/iomgr/error_internal.h +++ b/src/core/lib/iomgr/error_internal.h @@ -35,18 +35,28 @@ #define GRPC_CORE_LIB_IOMGR_ERROR_INTERNAL_H #include -#include +#include // TODO, do we need this? -#include +#include + +typedef struct linked_error linked_error; + +struct linked_error { + grpc_error *err; + uint8_t next; +}; struct grpc_error { gpr_refcount refs; - gpr_avl ints; - gpr_avl strs; - gpr_avl times; - gpr_avl errs; - uintptr_t next_err; + uint8_t ints[GRPC_ERROR_INT_MAX]; + uint8_t strs[GRPC_ERROR_STR_MAX]; + uint8_t times[GRPC_ERROR_TIME_MAX]; + uint8_t first_err; + uint8_t last_err; gpr_atm error_string; + uint8_t arena_size; + uint8_t arena_capacity; + intptr_t arena[0]; }; bool grpc_error_is_special(grpc_error *err); diff --git a/src/core/lib/transport/error_utils.c b/src/core/lib/transport/error_utils.c index da77828d9c1..707aac024b2 100644 --- a/src/core/lib/transport/error_utils.c +++ b/src/core/lib/transport/error_utils.c @@ -44,12 +44,12 @@ static grpc_error *recursively_find_error_with_field(grpc_error *error, } if (grpc_error_is_special(error)) return NULL; // Otherwise, search through its children. - intptr_t key = 0; - while (true) { - grpc_error *child_error = gpr_avl_get(error->errs, (void *)key++); - if (child_error == NULL) break; - grpc_error *result = recursively_find_error_with_field(child_error, which); - if (result != NULL) return result; + uint8_t slot = error->first_err; + while (slot != UINT8_MAX) { + linked_error *lerr = (linked_error *)(error->arena + slot); + grpc_error *result = recursively_find_error_with_field(lerr->err, which); + if (result) return result; + slot = lerr->next; } return NULL; } @@ -112,13 +112,13 @@ bool grpc_error_has_clear_grpc_status(grpc_error *error) { if (grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, NULL)) { return true; } - intptr_t key = 0; - while (true) { - grpc_error *child_error = gpr_avl_get(error->errs, (void *)key++); - if (child_error == NULL) break; - if (grpc_error_has_clear_grpc_status(child_error)) { + uint8_t slot = error->first_err; + while (slot != UINT8_MAX) { + linked_error *lerr = (linked_error *)(error->arena + slot); + if (grpc_error_has_clear_grpc_status(lerr->err)) { return true; } + slot = lerr->next; } return false; } diff --git a/test/core/iomgr/error_test.c b/test/core/iomgr/error_test.c index 8a0e7625e0d..d8289ac1029 100644 --- a/test/core/iomgr/error_test.c +++ b/test/core/iomgr/error_test.c @@ -43,30 +43,29 @@ #include "test/core/util/test_config.h" -static void test_set_get_int() -{ +static void test_set_get_int() { grpc_error* error = GRPC_ERROR_CREATE("Test"); + GPR_ASSERT(error); intptr_t i = 0; GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_FILE_LINE, &i)); - GPR_ASSERT(i); // line set will never be 0 + GPR_ASSERT(i); // line set will never be 0 GPR_ASSERT(!grpc_error_get_int(error, GRPC_ERROR_INT_ERRNO, &i)); GPR_ASSERT(!grpc_error_get_int(error, GRPC_ERROR_INT_SIZE, &i)); - intptr_t errno = 314; - error = grpc_error_set_int(error, GRPC_ERROR_INT_ERRNO, errno); + intptr_t errnumber = 314; + error = grpc_error_set_int(error, GRPC_ERROR_INT_ERRNO, errnumber); GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_ERRNO, &i)); - GPR_ASSERT(i == errno); + GPR_ASSERT(i == errnumber); - intptr_t line = 555; - error = grpc_error_set_int(error, GRPC_ERROR_INT_FILE_LINE, line); - GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_FILE_LINE, &i)); - GPR_ASSERT(i == line); + intptr_t http = 2; + error = grpc_error_set_int(error, GRPC_ERROR_INT_HTTP2_ERROR, http); + GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_HTTP2_ERROR, &i)); + GPR_ASSERT(i == http); GRPC_ERROR_UNREF(error); } -static void test_set_get_str() -{ +static void test_set_get_str() { grpc_error* error = GRPC_ERROR_CREATE("Test"); GPR_ASSERT(!grpc_error_get_str(error, GRPC_ERROR_STR_SYSCALL)); @@ -74,78 +73,146 @@ static void test_set_get_str() const char* c = grpc_error_get_str(error, GRPC_ERROR_STR_FILE); GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "test/core/iomgr/error_test.c")); + GPR_ASSERT(strstr(c, "error_test.c")); // __FILE__ expands differently on + // Windows. All should at least + // contain error_test.c c = grpc_error_get_str(error, GRPC_ERROR_STR_DESCRIPTION); GPR_ASSERT(c); GPR_ASSERT(!strcmp(c, "Test")); - error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "message"); + error = + grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "longer message"); c = grpc_error_get_str(error, GRPC_ERROR_STR_GRPC_MESSAGE); GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "message")); - - error = grpc_error_set_str(error, GRPC_ERROR_STR_DESCRIPTION, "new desc"); - c = grpc_error_get_str(error, GRPC_ERROR_STR_DESCRIPTION); - GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "new desc")); + GPR_ASSERT(!strcmp(c, "longer message")); GRPC_ERROR_UNREF(error); } -static void test_copy_and_unref() -{ +static void test_copy_and_unref() { // error1 has one ref - grpc_error* error1 = GRPC_ERROR_CREATE("Test"); - grpc_error* error2 = grpc_error_set_str(error1, GRPC_ERROR_STR_GRPC_MESSAGE, "message"); + grpc_error* error1 = grpc_error_set_str( + GRPC_ERROR_CREATE("Test"), GRPC_ERROR_STR_GRPC_MESSAGE, "message"); const char* c = grpc_error_get_str(error1, GRPC_ERROR_STR_GRPC_MESSAGE); GPR_ASSERT(c); GPR_ASSERT(!strcmp(c, "message")); - c = grpc_error_get_str(error2, GRPC_ERROR_STR_GRPC_MESSAGE); - GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "message")); - GPR_ASSERT(error1 == error2); // error 1 has two refs GRPC_ERROR_REF(error1); // this gives error3 a ref to the new error, and decrements error1 to one ref - grpc_error* error3 = grpc_error_set_str(error1, GRPC_ERROR_STR_DESCRIPTION, "reset"); - GPR_ASSERT(error3 != error1); // should not be the same because of extra ref + grpc_error* error3 = + grpc_error_set_str(error1, GRPC_ERROR_STR_SYSCALL, "syscall"); + GPR_ASSERT(error3 != error1); // should not be the same because of extra ref c = grpc_error_get_str(error3, GRPC_ERROR_STR_GRPC_MESSAGE); GPR_ASSERT(c); GPR_ASSERT(!strcmp(c, "message")); - c = grpc_error_get_str(error1, GRPC_ERROR_STR_DESCRIPTION); + // error 1 should not have a syscall but 3 should + GPR_ASSERT(!grpc_error_get_str(error1, GRPC_ERROR_STR_SYSCALL)); + c = grpc_error_get_str(error3, GRPC_ERROR_STR_SYSCALL); GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "Test")); - - c = grpc_error_get_str(error3, GRPC_ERROR_STR_DESCRIPTION); - GPR_ASSERT(c); - GPR_ASSERT(!strcmp(c, "reset")); + GPR_ASSERT(!strcmp(c, "syscall")); GRPC_ERROR_UNREF(error1); GRPC_ERROR_UNREF(error3); } -static void print_error_strings() -{ - grpc_error* error = grpc_error_set_int(GRPC_ERROR_CREATE("Error"), - GRPC_ERROR_INT_GRPC_STATUS, - GRPC_STATUS_UNIMPLEMENTED); - error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, 0); +static void test_create_referencing() { + grpc_error* child = grpc_error_set_str( + GRPC_ERROR_CREATE("Child"), GRPC_ERROR_STR_GRPC_MESSAGE, "message"); + grpc_error* parent = GRPC_ERROR_CREATE_REFERENCING("Parent", &child, 1); + GPR_ASSERT(parent); + + GRPC_ERROR_UNREF(child); + GRPC_ERROR_UNREF(parent); +} + +static void test_create_referencing_many() { + grpc_error* children[3]; + children[0] = grpc_error_set_str(GRPC_ERROR_CREATE("Child1"), + GRPC_ERROR_STR_GRPC_MESSAGE, "message"); + children[1] = grpc_error_set_int(GRPC_ERROR_CREATE("Child2"), + GRPC_ERROR_INT_HTTP2_ERROR, 5); + children[2] = grpc_error_set_str(GRPC_ERROR_CREATE("Child3"), + GRPC_ERROR_STR_GRPC_MESSAGE, "message 3"); + + grpc_error* parent = GRPC_ERROR_CREATE_REFERENCING("Parent", children, 3); + GPR_ASSERT(parent); + + for (size_t i = 0; i < 3; ++i) { + GRPC_ERROR_UNREF(children[i]); + } + GRPC_ERROR_UNREF(parent); +} + +static void print_error_string() { + grpc_error* error = + grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_GRPC_STATUS, + GRPC_STATUS_UNIMPLEMENTED); error = grpc_error_set_int(error, GRPC_ERROR_INT_SIZE, 666); error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "message"); - gpr_log(GPR_DEBUG, "%s", grpc_error_string(error)); + // gpr_log(GPR_DEBUG, "%s", grpc_error_string(error)); + GRPC_ERROR_UNREF(error); +} + +static void print_error_string_reference() { + grpc_error* children[2]; + children[0] = grpc_error_set_str( + grpc_error_set_int(GRPC_ERROR_CREATE("1"), GRPC_ERROR_INT_GRPC_STATUS, + GRPC_STATUS_UNIMPLEMENTED), + GRPC_ERROR_STR_GRPC_MESSAGE, "message for child 1"); + children[1] = grpc_error_set_str( + grpc_error_set_int(GRPC_ERROR_CREATE("2sd"), GRPC_ERROR_INT_GRPC_STATUS, + GRPC_STATUS_INTERNAL), + GRPC_ERROR_STR_GRPC_MESSAGE, "message for child 2"); + + grpc_error* parent = GRPC_ERROR_CREATE_REFERENCING("Parent", children, 2); + + gpr_log(GPR_DEBUG, "%s", grpc_error_string(parent)); + + for (size_t i = 0; i < 2; ++i) { + GRPC_ERROR_UNREF(children[i]); + } + GRPC_ERROR_UNREF(parent); +} + +static void test_os_error() { + int fake_errno = 5; + const char* syscall = "syscall name"; + grpc_error* error = GRPC_OS_ERROR(fake_errno, syscall); + + intptr_t i = 0; + GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_ERRNO, &i)); + GPR_ASSERT(i == fake_errno); + + const char* c = grpc_error_get_str(error, GRPC_ERROR_STR_SYSCALL); + GPR_ASSERT(c); + GPR_ASSERT(!strcmp(c, syscall)); + GRPC_ERROR_UNREF(error); +} + +static void test_special() { + grpc_error* error = GRPC_ERROR_NONE; + error = grpc_error_add_child(error, GRPC_ERROR_CREATE("test child")); + intptr_t i; + GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, &i)); + GPR_ASSERT(i == GRPC_STATUS_OK); GRPC_ERROR_UNREF(error); } -int main(int argc, char **argv) { +int main(int argc, char** argv) { grpc_test_init(argc, argv); grpc_init(); test_set_get_int(); test_set_get_str(); test_copy_and_unref(); - print_error_strings(); + print_error_string(); + print_error_string_reference(); + test_os_error(); + test_create_referencing(); + test_create_referencing_many(); + test_special(); grpc_shutdown(); return 0; diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc index 4a5af6884df..66256d75388 100644 --- a/test/cpp/microbenchmarks/bm_error.cc +++ b/test/cpp/microbenchmarks/bm_error.cc @@ -74,26 +74,6 @@ static void BM_ErrorCreateAndSetIntAndStr(benchmark::State& state) { } BENCHMARK(BM_ErrorCreateAndSetIntAndStr); -static void BM_ErrorCreateAndSetIntLoop(benchmark::State& state) { - grpc_error* error = GRPC_ERROR_CREATE("Error"); - int n = 0; - while (state.KeepRunning()) { - error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, n++); - } - GRPC_ERROR_UNREF(error); -} -BENCHMARK(BM_ErrorCreateAndSetIntLoop); - -static void BM_ErrorCreateAndSetStrLoop(benchmark::State& state) { - grpc_error* error = GRPC_ERROR_CREATE("Error"); - const char* str = "hello"; - while (state.KeepRunning()) { - error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, str); - } - GRPC_ERROR_UNREF(error); -} -BENCHMARK(BM_ErrorCreateAndSetStrLoop); - static void BM_ErrorRefUnref(benchmark::State& state) { grpc_error* error = GRPC_ERROR_CREATE("Error"); while (state.KeepRunning()) { From 1a882cfc4a193f7bc15e2e47b4e2bcf43c105440 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 8 Mar 2017 15:42:59 -0800 Subject: [PATCH 073/188] Add more core banned functions --- tools/run_tests/sanity/core_banned_functions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/run_tests/sanity/core_banned_functions.py b/tools/run_tests/sanity/core_banned_functions.py index afac10bf80e..6d9be2e5d4a 100755 --- a/tools/run_tests/sanity/core_banned_functions.py +++ b/tools/run_tests/sanity/core_banned_functions.py @@ -42,6 +42,12 @@ BANNED_EXCEPT = { 'grpc_slice_buffer_reset_and_unref(': ['src/core/lib/slice/slice_buffer.c'], 'grpc_slice_ref(': ['src/core/lib/slice/slice.c'], 'grpc_slice_unref(': ['src/core/lib/slice/slice.c'], + 'grpc_error_create(': ['src/core/lib/iomgr/error.c'], + 'grpc_error_ref(': ['src/core/lib/iomgr/error.c'], + 'grpc_error_unref(': ['src/core/lib/iomgr/error.c'], + 'grpc_os_error(': ['src/core/lib/iomgr/error.c'], + 'grpc_wsa_error(': ['src/core/lib/iomgr/error.c'], + 'grpc_log_if_error(': ['src/core/lib/iomgr/error.c'], } errors = 0 From e0341792d5a33469bbac481e9737bd00e3cb8900 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Mar 2017 15:59:16 -0800 Subject: [PATCH 074/188] Remove most usage of the grpc_call lock --- src/core/lib/surface/call.c | 77 ++++++++++++++----------------------- 1 file changed, 29 insertions(+), 48 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index c2547c5147a..9b01c46bd59 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -143,9 +143,10 @@ struct grpc_call { grpc_channel *channel; grpc_call *parent; grpc_call *first_child; + gpr_atm has_children; gpr_timespec start_time; - /* TODO(ctiller): share with cq if possible? */ - gpr_mu mu; + /* protects first_child, setting has_children, and child next/prev links */ + gpr_mu child_list_mu; /* client or server call */ bool is_client; @@ -161,7 +162,7 @@ struct grpc_call { bool receiving_message; bool requested_final_op; bool received_final_op; - bool sent_any_op; + gpr_atm num_ops_sent; /* have we received initial metadata */ bool has_initial_md_been_received; @@ -275,7 +276,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, GPR_TIMER_BEGIN("grpc_call_create", 0); call = gpr_zalloc(sizeof(grpc_call) + channel_stack->call_stack_size); *out_call = call; - gpr_mu_init(&call->mu); + gpr_mu_init(&call->child_list_mu); call->channel = args->channel; call->cq = args->cq; call->parent = args->parent_call; @@ -313,7 +314,8 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, GPR_ASSERT(call->is_client); GPR_ASSERT(!args->parent_call->is_client); - gpr_mu_lock(&args->parent_call->mu); + gpr_mu_lock(&args->parent_call->child_list_mu); + gpr_atm_no_barrier_store(&args->parent_call->has_children, 1); if (args->propagation_mask & GRPC_PROPAGATE_DEADLINE) { send_deadline = gpr_time_min( @@ -353,7 +355,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, call; } - gpr_mu_unlock(&args->parent_call->mu); + gpr_mu_unlock(&args->parent_call->child_list_mu); } call->send_deadline = send_deadline; @@ -435,7 +437,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, if (c->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, c->receiving_stream); } - gpr_mu_destroy(&c->mu); + gpr_mu_destroy(&c->child_list_mu); for (ii = 0; ii < c->send_extra_metadata_count; ii++) { GRPC_MDELEM_UNREF(exec_ctx, c->send_extra_metadata[ii].md); } @@ -473,7 +475,7 @@ void grpc_call_destroy(grpc_call *c) { GRPC_API_TRACE("grpc_call_destroy(c=%p)", 1, (c)); if (parent) { - gpr_mu_lock(&parent->mu); + gpr_mu_lock(&parent->child_list_mu); if (c == parent->first_child) { parent->first_child = c->sibling_next; if (c == parent->first_child) { @@ -482,15 +484,13 @@ void grpc_call_destroy(grpc_call *c) { c->sibling_prev->sibling_next = c->sibling_next; c->sibling_next->sibling_prev = c->sibling_prev; } - gpr_mu_unlock(&parent->mu); + gpr_mu_unlock(&parent->child_list_mu); GRPC_CALL_INTERNAL_UNREF(&exec_ctx, parent, "child"); } - gpr_mu_lock(&c->mu); GPR_ASSERT(!c->destroy_called); c->destroy_called = 1; - cancel = c->sent_any_op && !c->received_final_op; - gpr_mu_unlock(&c->mu); + cancel = gpr_atm_no_barrier_load(&c->num_ops_sent) && !c->received_final_op; if (cancel) { cancel_with_error(&exec_ctx, c, STATUS_FROM_API_OVERRIDE, GRPC_ERROR_CANCELLED); @@ -555,10 +555,8 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c, "c=%p, status=%d, description=%s, reserved=%p)", 4, (c, (int)status, description, reserved)); GPR_ASSERT(reserved == NULL); - gpr_mu_lock(&c->mu); cancel_with_status(&exec_ctx, c, STATUS_FROM_API_OVERRIDE, status, description); - gpr_mu_unlock(&c->mu); grpc_exec_ctx_finish(&exec_ctx); return GRPC_CALL_OK; } @@ -715,9 +713,7 @@ static void set_incoming_compression_algorithm( grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm( grpc_call *call) { grpc_compression_algorithm algorithm; - gpr_mu_lock(&call->mu); algorithm = call->incoming_compression_algorithm; - gpr_mu_unlock(&call->mu); return algorithm; } @@ -729,9 +725,7 @@ static grpc_compression_algorithm compression_algorithm_for_level_locked( uint32_t grpc_call_test_only_get_message_flags(grpc_call *call) { uint32_t flags; - gpr_mu_lock(&call->mu); flags = call->test_only_last_message_flags; - gpr_mu_unlock(&call->mu); return flags; } @@ -785,9 +779,7 @@ static void set_encodings_accepted_by_peer(grpc_exec_ctx *exec_ctx, uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call *call) { uint32_t encodings_accepted_by_peer; - gpr_mu_lock(&call->mu); encodings_accepted_by_peer = call->encodings_accepted_by_peer; - gpr_mu_unlock(&call->mu); return encodings_accepted_by_peer; } @@ -1083,8 +1075,6 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, grpc_call *call = bctl->call; grpc_error *error = consolidate_batch_errors(bctl); - gpr_mu_lock(&call->mu); - if (bctl->send_initial_metadata) { grpc_metadata_batch_destroy( exec_ctx, @@ -1105,17 +1095,21 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, call->received_final_op = true; /* propagate cancellation to any interested children */ - child_call = call->first_child; - if (child_call != NULL) { - do { - next_child_call = child_call->sibling_next; - if (child_call->cancellation_is_inherited) { - GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel"); - grpc_call_cancel(child_call, NULL); - GRPC_CALL_INTERNAL_UNREF(exec_ctx, child_call, "propagate_cancel"); - } - child_call = next_child_call; - } while (child_call != call->first_child); + if (gpr_atm_no_barrier_load(&call->has_children)) { + gpr_mu_lock(&call->child_list_mu); + child_call = call->first_child; + if (child_call != NULL) { + do { + next_child_call = child_call->sibling_next; + if (child_call->cancellation_is_inherited) { + GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel"); + grpc_call_cancel(child_call, NULL); + GRPC_CALL_INTERNAL_UNREF(exec_ctx, child_call, "propagate_cancel"); + } + child_call = next_child_call; + } while (child_call != call->first_child); + } + gpr_mu_unlock(&call->child_list_mu); } if (call->is_client) { @@ -1130,7 +1124,6 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); error = GRPC_ERROR_NONE; } - gpr_mu_unlock(&call->mu); if (bctl->is_notify_tag_closure) { /* unrefs bctl->error */ @@ -1221,7 +1214,6 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, grpc_error *error) { batch_control *bctl = bctlp; grpc_call *call = bctl->call; - gpr_mu_lock(&bctl->call->mu); if (error != GRPC_ERROR_NONE) { if (call->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, call->receiving_stream); @@ -1233,11 +1225,9 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, } if (call->has_initial_md_been_received || error != GRPC_ERROR_NONE || call->receiving_stream == NULL) { - gpr_mu_unlock(&bctl->call->mu); process_data_after_md(exec_ctx, bctlp); } else { call->saved_receiving_stream_ready_bctlp = bctlp; - gpr_mu_unlock(&bctl->call->mu); } } @@ -1309,8 +1299,6 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx, batch_control *bctl = bctlp; grpc_call *call = bctl->call; - gpr_mu_lock(&call->mu); - add_batch_error(exec_ctx, bctl, GRPC_ERROR_REF(error), false); if (error == GRPC_ERROR_NONE) { grpc_metadata_batch *md = @@ -1336,11 +1324,9 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx, receiving_stream_ready, call->saved_receiving_stream_ready_bctlp, grpc_schedule_on_exec_ctx); call->saved_receiving_stream_ready_bctlp = NULL; - grpc_closure_sched(exec_ctx, saved_rsr_closure, GRPC_ERROR_REF(error)); + grpc_closure_run(exec_ctx, saved_rsr_closure, GRPC_ERROR_REF(error)); } - gpr_mu_unlock(&call->mu); - finish_batch_step(exec_ctx, bctl); } @@ -1393,7 +1379,6 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, bctl->notify_tag = notify_tag; bctl->is_notify_tag_closure = (uint8_t)(is_notify_tag_closure != 0); - gpr_mu_lock(&call->mu); grpc_transport_stream_op *stream_op = &bctl->op; memset(stream_op, 0, sizeof(*stream_op)); stream_op->covered_by_poller = true; @@ -1679,8 +1664,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, grpc_closure_init(&bctl->finish_batch, finish_batch, bctl, grpc_schedule_on_exec_ctx); stream_op->on_complete = &bctl->finish_batch; - call->sent_any_op = true; - gpr_mu_unlock(&call->mu); + gpr_atm_no_barrier_fetch_add(&call->num_ops_sent, 1); execute_op(exec_ctx, call, stream_op); @@ -1711,7 +1695,6 @@ done_with_error: if (bctl->recv_final_op) { call->requested_final_op = 0; } - gpr_mu_unlock(&call->mu); goto done; } @@ -1760,10 +1743,8 @@ uint8_t grpc_call_is_client(grpc_call *call) { return call->is_client; } grpc_compression_algorithm grpc_call_compression_for_level( grpc_call *call, grpc_compression_level level) { - gpr_mu_lock(&call->mu); grpc_compression_algorithm algo = compression_algorithm_for_level_locked(call, level); - gpr_mu_unlock(&call->mu); return algo; } From 676db291d2387603f795dd6bd5853188e0ff7f63 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 8 Mar 2017 16:10:56 -0800 Subject: [PATCH 075/188] Fix barriers --- src/core/lib/surface/call.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 9b01c46bd59..2f3ddd6bcb9 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -315,7 +315,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, GPR_ASSERT(!args->parent_call->is_client); gpr_mu_lock(&args->parent_call->child_list_mu); - gpr_atm_no_barrier_store(&args->parent_call->has_children, 1); + gpr_atm_rel_store(&args->parent_call->has_children, 1); if (args->propagation_mask & GRPC_PROPAGATE_DEADLINE) { send_deadline = gpr_time_min( @@ -1095,7 +1095,7 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, call->received_final_op = true; /* propagate cancellation to any interested children */ - if (gpr_atm_no_barrier_load(&call->has_children)) { + if (gpr_atm_acq_load(&call->has_children)) { gpr_mu_lock(&call->child_list_mu); child_call = call->first_child; if (child_call != NULL) { From 0cb7fef40669789c8720427c70b892a41e27bbdf Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 9 Mar 2017 11:05:36 +0100 Subject: [PATCH 076/188] Revert "Split bm_fullstack into pieces, reuse infrastructure across other microbenchmarks" --- CMakeLists.txt | 191 +-- Makefile | 264 +--- build.yaml | 101 +- include/grpc/impl/codegen/sync.h | 8 - test/core/end2end/BUILD | 4 +- test/core/end2end/fixtures/h2_http_proxy.c | 2 +- .../{http_proxy_fixture.c => http_proxy.c} | 2 +- .../{http_proxy_fixture.h => http_proxy.h} | 0 test/cpp/microbenchmarks/bm_call_create.cc | 10 +- test/cpp/microbenchmarks/bm_chttp2_hpack.cc | 14 +- test/cpp/microbenchmarks/bm_closure.cc | 114 +- test/cpp/microbenchmarks/bm_cq.cc | 23 +- test/cpp/microbenchmarks/bm_error.cc | 28 - test/cpp/microbenchmarks/bm_fullstack.cc | 1079 +++++++++++++++++ .../bm_fullstack_streaming_ping_pong.cc | 197 --- .../bm_fullstack_streaming_pump.cc | 197 --- .../microbenchmarks/bm_fullstack_trickle.cc | 219 ---- .../bm_fullstack_unary_ping_pong.cc | 205 ---- test/cpp/microbenchmarks/bm_metadata.cc | 45 +- .../fullstack_context_mutators.h | 158 --- test/cpp/microbenchmarks/fullstack_fixtures.h | 244 ---- test/cpp/microbenchmarks/helpers.cc | 65 - test/cpp/microbenchmarks/helpers.h | 91 -- .../generated/sources_and_headers.json | 104 +- tools/run_tests/generated/tests.json | 83 +- tools/run_tests/run_microbenchmark.py | 8 +- .../grpc_test_util/grpc_test_util.vcxproj | 4 +- .../grpc_test_util.vcxproj.filters | 4 +- .../grpc_test_util_unsecure.vcxproj | 4 +- .../grpc_test_util_unsecure.vcxproj.filters | 4 +- .../grpc_benchmark/grpc_benchmark.vcxproj | 206 ---- .../grpc_benchmark.vcxproj.filters | 32 - 32 files changed, 1234 insertions(+), 2476 deletions(-) rename test/core/end2end/fixtures/{http_proxy_fixture.c => http_proxy.c} (99%) rename test/core/end2end/fixtures/{http_proxy_fixture.h => http_proxy.h} (100%) create mode 100644 test/cpp/microbenchmarks/bm_fullstack.cc delete mode 100644 test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc delete mode 100644 test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc delete mode 100644 test/cpp/microbenchmarks/bm_fullstack_trickle.cc delete mode 100644 test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc delete mode 100644 test/cpp/microbenchmarks/fullstack_context_mutators.h delete mode 100644 test/cpp/microbenchmarks/fullstack_fixtures.h delete mode 100644 test/cpp/microbenchmarks/helpers.cc delete mode 100644 test/cpp/microbenchmarks/helpers.h delete mode 100644 vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj delete mode 100644 vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index 24b84fd15a8..607de349cb1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -588,16 +588,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_error) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) -add_dependencies(buildtests_cxx bm_fullstack_streaming_ping_pong) -endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) -add_dependencies(buildtests_cxx bm_fullstack_streaming_pump) -endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) -add_dependencies(buildtests_cxx bm_fullstack_trickle) -endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) -add_dependencies(buildtests_cxx bm_fullstack_unary_ping_pong) +add_dependencies(buildtests_cxx bm_fullstack) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_metadata) @@ -1449,7 +1440,7 @@ add_library(grpc_test_util test/core/security/oauth2_utils.c test/core/end2end/cq_verifier.c test/core/end2end/fake_resolver.c - test/core/end2end/fixtures/http_proxy_fixture.c + test/core/end2end/fixtures/http_proxy.c test/core/end2end/fixtures/proxy.c test/core/iomgr/endpoint_tests.c test/core/util/debugger_macros.c @@ -1656,7 +1647,7 @@ if (gRPC_BUILD_TESTS) add_library(grpc_test_util_unsecure test/core/end2end/cq_verifier.c test/core/end2end/fake_resolver.c - test/core/end2end/fixtures/http_proxy_fixture.c + test/core/end2end/fixtures/http_proxy.c test/core/end2end/fixtures/proxy.c test/core/iomgr/endpoint_tests.c test/core/util/debugger_macros.c @@ -3107,50 +3098,6 @@ endif() if (gRPC_BUILD_TESTS) -add_library(grpc_benchmark - test/cpp/microbenchmarks/helpers.cc -) - -if(WIN32 AND MSVC) - set_target_properties(grpc_benchmark PROPERTIES COMPILE_PDB_NAME "grpc_benchmark" - COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" - ) - if (gRPC_INSTALL) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc_benchmark.pdb - DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL - ) - endif() -endif() - - -target_include_directories(grpc_benchmark - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include - PRIVATE ${BORINGSSL_ROOT_DIR}/include - PRIVATE ${PROTOBUF_ROOT_DIR}/src - PRIVATE ${ZLIB_INCLUDE_DIR} - PRIVATE ${BENCHMARK}/include - PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib - PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include - PRIVATE third_party/googletest/include - PRIVATE third_party/googletest - PRIVATE ${_gRPC_PROTO_GENS_DIR} -) - -target_link_libraries(grpc_benchmark - ${_gRPC_PROTOBUF_LIBRARIES} - ${_gRPC_ALLTARGETS_LIBRARIES} - benchmark - grpc++ - grpc_test_util - grpc - ${_gRPC_GFLAGS_LIBRARIES} -) - - -endif (gRPC_BUILD_TESTS) -if (gRPC_BUILD_TESTS) - add_library(grpc_cli_libs test/cpp/util/cli_call.cc test/cpp/util/cli_credentials.cc @@ -7614,7 +7561,6 @@ target_include_directories(bm_call_create target_link_libraries(bm_call_create ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7653,7 +7599,6 @@ target_include_directories(bm_chttp2_hpack target_link_libraries(bm_chttp2_hpack ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7692,7 +7637,6 @@ target_include_directories(bm_closure target_link_libraries(bm_closure ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7731,7 +7675,6 @@ target_include_directories(bm_cq target_link_libraries(bm_cq ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7770,124 +7713,6 @@ target_include_directories(bm_error target_link_libraries(bm_error ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util - grpc++ - grpc - gpr_test_util - gpr - ${_gRPC_GFLAGS_LIBRARIES} -) - -endif() -endif (gRPC_BUILD_TESTS) -if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) - -add_executable(bm_fullstack_streaming_ping_pong - test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc - third_party/googletest/src/gtest-all.cc -) - - -target_include_directories(bm_fullstack_streaming_ping_pong - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include - PRIVATE ${BORINGSSL_ROOT_DIR}/include - PRIVATE ${PROTOBUF_ROOT_DIR}/src - PRIVATE ${BENCHMARK_ROOT_DIR}/include - PRIVATE ${ZLIB_ROOT_DIR} - PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib - PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include - PRIVATE third_party/googletest/include - PRIVATE third_party/googletest - PRIVATE ${_gRPC_PROTO_GENS_DIR} -) - -target_link_libraries(bm_fullstack_streaming_ping_pong - ${_gRPC_PROTOBUF_LIBRARIES} - ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util - grpc++ - grpc - gpr_test_util - gpr - ${_gRPC_GFLAGS_LIBRARIES} -) - -endif() -endif (gRPC_BUILD_TESTS) -if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) - -add_executable(bm_fullstack_streaming_pump - test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc - third_party/googletest/src/gtest-all.cc -) - - -target_include_directories(bm_fullstack_streaming_pump - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include - PRIVATE ${BORINGSSL_ROOT_DIR}/include - PRIVATE ${PROTOBUF_ROOT_DIR}/src - PRIVATE ${BENCHMARK_ROOT_DIR}/include - PRIVATE ${ZLIB_ROOT_DIR} - PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib - PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include - PRIVATE third_party/googletest/include - PRIVATE third_party/googletest - PRIVATE ${_gRPC_PROTO_GENS_DIR} -) - -target_link_libraries(bm_fullstack_streaming_pump - ${_gRPC_PROTOBUF_LIBRARIES} - ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util - grpc++ - grpc - gpr_test_util - gpr - ${_gRPC_GFLAGS_LIBRARIES} -) - -endif() -endif (gRPC_BUILD_TESTS) -if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) - -add_executable(bm_fullstack_trickle - test/cpp/microbenchmarks/bm_fullstack_trickle.cc - third_party/googletest/src/gtest-all.cc -) - - -target_include_directories(bm_fullstack_trickle - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include - PRIVATE ${BORINGSSL_ROOT_DIR}/include - PRIVATE ${PROTOBUF_ROOT_DIR}/src - PRIVATE ${BENCHMARK_ROOT_DIR}/include - PRIVATE ${ZLIB_ROOT_DIR} - PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib - PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include - PRIVATE third_party/googletest/include - PRIVATE third_party/googletest - PRIVATE ${_gRPC_PROTO_GENS_DIR} -) - -target_link_libraries(bm_fullstack_trickle - ${_gRPC_PROTOBUF_LIBRARIES} - ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7903,13 +7728,13 @@ endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) -add_executable(bm_fullstack_unary_ping_pong - test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc +add_executable(bm_fullstack + test/cpp/microbenchmarks/bm_fullstack.cc third_party/googletest/src/gtest-all.cc ) -target_include_directories(bm_fullstack_unary_ping_pong +target_include_directories(bm_fullstack PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${BORINGSSL_ROOT_DIR}/include @@ -7923,10 +7748,9 @@ target_include_directories(bm_fullstack_unary_ping_pong PRIVATE ${_gRPC_PROTO_GENS_DIR} ) -target_link_libraries(bm_fullstack_unary_ping_pong +target_link_libraries(bm_fullstack ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7965,7 +7789,6 @@ target_include_directories(bm_metadata target_link_libraries(bm_metadata ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_benchmark benchmark grpc_test_util grpc diff --git a/Makefile b/Makefile index a49fa928aac..e8643855eb3 100644 --- a/Makefile +++ b/Makefile @@ -1051,10 +1051,7 @@ bm_chttp2_hpack: $(BINDIR)/$(CONFIG)/bm_chttp2_hpack bm_closure: $(BINDIR)/$(CONFIG)/bm_closure bm_cq: $(BINDIR)/$(CONFIG)/bm_cq bm_error: $(BINDIR)/$(CONFIG)/bm_error -bm_fullstack_streaming_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong -bm_fullstack_streaming_pump: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump -bm_fullstack_trickle: $(BINDIR)/$(CONFIG)/bm_fullstack_trickle -bm_fullstack_unary_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong +bm_fullstack: $(BINDIR)/$(CONFIG)/bm_fullstack bm_metadata: $(BINDIR)/$(CONFIG)/bm_metadata channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test channel_filter_test: $(BINDIR)/$(CONFIG)/channel_filter_test @@ -1470,10 +1467,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_closure \ $(BINDIR)/$(CONFIG)/bm_cq \ $(BINDIR)/$(CONFIG)/bm_error \ - $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong \ - $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump \ - $(BINDIR)/$(CONFIG)/bm_fullstack_trickle \ - $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ + $(BINDIR)/$(CONFIG)/bm_fullstack \ $(BINDIR)/$(CONFIG)/bm_metadata \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ @@ -1584,10 +1578,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_closure \ $(BINDIR)/$(CONFIG)/bm_cq \ $(BINDIR)/$(CONFIG)/bm_error \ - $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong \ - $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump \ - $(BINDIR)/$(CONFIG)/bm_fullstack_trickle \ - $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ + $(BINDIR)/$(CONFIG)/bm_fullstack \ $(BINDIR)/$(CONFIG)/bm_metadata \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ @@ -1926,14 +1917,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/bm_cq || ( echo test bm_cq failed ; exit 1 ) $(E) "[RUN] Testing bm_error" $(Q) $(BINDIR)/$(CONFIG)/bm_error || ( echo test bm_error failed ; exit 1 ) - $(E) "[RUN] Testing bm_fullstack_streaming_ping_pong" - $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong || ( echo test bm_fullstack_streaming_ping_pong failed ; exit 1 ) - $(E) "[RUN] Testing bm_fullstack_streaming_pump" - $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump || ( echo test bm_fullstack_streaming_pump failed ; exit 1 ) - $(E) "[RUN] Testing bm_fullstack_trickle" - $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_trickle || ( echo test bm_fullstack_trickle failed ; exit 1 ) - $(E) "[RUN] Testing bm_fullstack_unary_ping_pong" - $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong || ( echo test bm_fullstack_unary_ping_pong failed ; exit 1 ) + $(E) "[RUN] Testing bm_fullstack" + $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack || ( echo test bm_fullstack failed ; exit 1 ) $(E) "[RUN] Testing bm_metadata" $(Q) $(BINDIR)/$(CONFIG)/bm_metadata || ( echo test bm_metadata failed ; exit 1 ) $(E) "[RUN] Testing channel_arguments_test" @@ -3333,7 +3318,7 @@ LIBGRPC_TEST_UTIL_SRC = \ test/core/security/oauth2_utils.c \ test/core/end2end/cq_verifier.c \ test/core/end2end/fake_resolver.c \ - test/core/end2end/fixtures/http_proxy_fixture.c \ + test/core/end2end/fixtures/http_proxy.c \ test/core/end2end/fixtures/proxy.c \ test/core/iomgr/endpoint_tests.c \ test/core/util/debugger_macros.c \ @@ -3533,7 +3518,7 @@ endif LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ test/core/end2end/cq_verifier.c \ test/core/end2end/fake_resolver.c \ - test/core/end2end/fixtures/http_proxy_fixture.c \ + test/core/end2end/fixtures/http_proxy.c \ test/core/end2end/fixtures/proxy.c \ test/core/iomgr/endpoint_tests.c \ test/core/util/debugger_macros.c \ @@ -4983,55 +4968,6 @@ ifneq ($(NO_DEPS),true) endif -LIBGRPC_BENCHMARK_SRC = \ - test/cpp/microbenchmarks/helpers.cc \ - -PUBLIC_HEADERS_CXX += \ - -LIBGRPC_BENCHMARK_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC_BENCHMARK_SRC)))) - - -ifeq ($(NO_SECURE),true) - -# You can't build secure libraries if you don't have OpenSSL. - -$(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: openssl_dep_error - - -else - -ifeq ($(NO_PROTOBUF),true) - -# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay. - -$(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: protobuf_dep_error - - -else - -$(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC_BENCHMARK_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a - $(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBGRPC_BENCHMARK_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a -endif - - - - -endif - -endif - -ifneq ($(NO_SECURE),true) -ifneq ($(NO_DEPS),true) --include $(LIBGRPC_BENCHMARK_OBJS:.o=.dep) -endif -endif - - LIBGRPC_CLI_LIBS_SRC = \ test/cpp/util/cli_call.cc \ test/cpp/util/cli_credentials.cc \ @@ -12615,16 +12551,16 @@ $(BINDIR)/$(CONFIG)/bm_call_create: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_call_create: $(PROTOBUF_DEP) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_call_create: $(PROTOBUF_DEP) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_call_create + $(Q) $(LDXX) $(LDFLAGS) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_call_create endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_call_create.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_call_create.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_call_create: $(BM_CALL_CREATE_OBJS:.o=.dep) @@ -12658,16 +12594,16 @@ $(BINDIR)/$(CONFIG)/bm_chttp2_hpack: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_chttp2_hpack: $(PROTOBUF_DEP) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_chttp2_hpack: $(PROTOBUF_DEP) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_chttp2_hpack + $(Q) $(LDXX) $(LDFLAGS) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_chttp2_hpack endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_chttp2_hpack.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_chttp2_hpack.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_chttp2_hpack: $(BM_CHTTP2_HPACK_OBJS:.o=.dep) @@ -12701,16 +12637,16 @@ $(BINDIR)/$(CONFIG)/bm_closure: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_closure: $(PROTOBUF_DEP) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_closure: $(PROTOBUF_DEP) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_closure + $(Q) $(LDXX) $(LDFLAGS) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_closure endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_closure.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_closure.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_closure: $(BM_CLOSURE_OBJS:.o=.dep) @@ -12744,16 +12680,16 @@ $(BINDIR)/$(CONFIG)/bm_cq: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_cq: $(PROTOBUF_DEP) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_cq: $(PROTOBUF_DEP) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_cq + $(Q) $(LDXX) $(LDFLAGS) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_cq endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_cq.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_cq.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_cq: $(BM_CQ_OBJS:.o=.dep) @@ -12787,16 +12723,16 @@ $(BINDIR)/$(CONFIG)/bm_error: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_error: $(PROTOBUF_DEP) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_error: $(PROTOBUF_DEP) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_error + $(Q) $(LDXX) $(LDFLAGS) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_error endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_error.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_error.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_error: $(BM_ERROR_OBJS:.o=.dep) @@ -12807,144 +12743,15 @@ endif endif -BM_FULLSTACK_STREAMING_PING_PONG_SRC = \ - test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc \ - -BM_FULLSTACK_STREAMING_PING_PONG_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_STREAMING_PING_PONG_SRC)))) -ifeq ($(NO_SECURE),true) - -# You can't build secure targets if you don't have OpenSSL. - -$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong: openssl_dep_error - -else - - - - -ifeq ($(NO_PROTOBUF),true) - -# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. - -$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong: protobuf_dep_error - -else - -$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong: $(PROTOBUF_DEP) $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong - -endif - -endif - -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a - -deps_bm_fullstack_streaming_ping_pong: $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS:.o=.dep) - -ifneq ($(NO_SECURE),true) -ifneq ($(NO_DEPS),true) --include $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS:.o=.dep) -endif -endif - - -BM_FULLSTACK_STREAMING_PUMP_SRC = \ - test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc \ - -BM_FULLSTACK_STREAMING_PUMP_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_STREAMING_PUMP_SRC)))) -ifeq ($(NO_SECURE),true) - -# You can't build secure targets if you don't have OpenSSL. - -$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump: openssl_dep_error - -else - - - - -ifeq ($(NO_PROTOBUF),true) - -# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. - -$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump: protobuf_dep_error - -else - -$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump: $(PROTOBUF_DEP) $(BM_FULLSTACK_STREAMING_PUMP_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_STREAMING_PUMP_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump - -endif - -endif - -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a - -deps_bm_fullstack_streaming_pump: $(BM_FULLSTACK_STREAMING_PUMP_OBJS:.o=.dep) - -ifneq ($(NO_SECURE),true) -ifneq ($(NO_DEPS),true) --include $(BM_FULLSTACK_STREAMING_PUMP_OBJS:.o=.dep) -endif -endif - - -BM_FULLSTACK_TRICKLE_SRC = \ - test/cpp/microbenchmarks/bm_fullstack_trickle.cc \ - -BM_FULLSTACK_TRICKLE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_TRICKLE_SRC)))) -ifeq ($(NO_SECURE),true) - -# You can't build secure targets if you don't have OpenSSL. - -$(BINDIR)/$(CONFIG)/bm_fullstack_trickle: openssl_dep_error - -else - - - - -ifeq ($(NO_PROTOBUF),true) - -# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. - -$(BINDIR)/$(CONFIG)/bm_fullstack_trickle: protobuf_dep_error - -else - -$(BINDIR)/$(CONFIG)/bm_fullstack_trickle: $(PROTOBUF_DEP) $(BM_FULLSTACK_TRICKLE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_TRICKLE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_trickle - -endif - -endif - -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_trickle.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a - -deps_bm_fullstack_trickle: $(BM_FULLSTACK_TRICKLE_OBJS:.o=.dep) - -ifneq ($(NO_SECURE),true) -ifneq ($(NO_DEPS),true) --include $(BM_FULLSTACK_TRICKLE_OBJS:.o=.dep) -endif -endif - - -BM_FULLSTACK_UNARY_PING_PONG_SRC = \ - test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc \ +BM_FULLSTACK_SRC = \ + test/cpp/microbenchmarks/bm_fullstack.cc \ -BM_FULLSTACK_UNARY_PING_PONG_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_UNARY_PING_PONG_SRC)))) +BM_FULLSTACK_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_SRC)))) ifeq ($(NO_SECURE),true) # You can't build secure targets if you don't have OpenSSL. -$(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong: openssl_dep_error +$(BINDIR)/$(CONFIG)/bm_fullstack: openssl_dep_error else @@ -12955,26 +12762,26 @@ ifeq ($(NO_PROTOBUF),true) # You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. -$(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong: protobuf_dep_error +$(BINDIR)/$(CONFIG)/bm_fullstack: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong: $(PROTOBUF_DEP) $(BM_FULLSTACK_UNARY_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_fullstack: $(PROTOBUF_DEP) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_UNARY_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong + $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -deps_bm_fullstack_unary_ping_pong: $(BM_FULLSTACK_UNARY_PING_PONG_OBJS:.o=.dep) +deps_bm_fullstack: $(BM_FULLSTACK_OBJS:.o=.dep) ifneq ($(NO_SECURE),true) ifneq ($(NO_DEPS),true) --include $(BM_FULLSTACK_UNARY_PING_PONG_OBJS:.o=.dep) +-include $(BM_FULLSTACK_OBJS:.o=.dep) endif endif @@ -13002,16 +12809,16 @@ $(BINDIR)/$(CONFIG)/bm_metadata: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_metadata: $(PROTOBUF_DEP) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_metadata: $(PROTOBUF_DEP) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_metadata + $(Q) $(LDXX) $(LDFLAGS) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_metadata endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_metadata.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_metadata.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_metadata: $(BM_METADATA_OBJS:.o=.dep) @@ -18453,7 +18260,6 @@ test/cpp/interop/interop_client.cc: $(OPENSSL_DEP) test/cpp/interop/interop_server.cc: $(OPENSSL_DEP) test/cpp/interop/interop_server_bootstrap.cc: $(OPENSSL_DEP) test/cpp/interop/server_helper.cc: $(OPENSSL_DEP) -test/cpp/microbenchmarks/helpers.cc: $(OPENSSL_DEP) test/cpp/qps/client_async.cc: $(OPENSSL_DEP) test/cpp/qps/client_sync.cc: $(OPENSSL_DEP) test/cpp/qps/driver.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 106e876a516..4f5b8e07f85 100644 --- a/build.yaml +++ b/build.yaml @@ -586,7 +586,7 @@ filegroups: headers: - test/core/end2end/cq_verifier.h - test/core/end2end/fake_resolver.h - - test/core/end2end/fixtures/http_proxy_fixture.h + - test/core/end2end/fixtures/http_proxy.h - test/core/end2end/fixtures/proxy.h - test/core/iomgr/endpoint_tests.h - test/core/util/debugger_macros.h @@ -602,7 +602,7 @@ filegroups: src: - test/core/end2end/cq_verifier.c - test/core/end2end/fake_resolver.c - - test/core/end2end/fixtures/http_proxy_fixture.c + - test/core/end2end/fixtures/http_proxy.c - test/core/end2end/fixtures/proxy.c - test/core/iomgr/endpoint_tests.c - test/core/util/debugger_macros.c @@ -1214,20 +1214,6 @@ libs: - grpc++_codegen_base_src secure: false vs_project_guid: '{6EE56155-DF7C-4F6E-BFC4-F6F776BEB211}' -- name: grpc_benchmark - build: test - language: c++ - headers: - - test/cpp/microbenchmarks/fullstack_context_mutators.h - - test/cpp/microbenchmarks/fullstack_fixtures.h - - test/cpp/microbenchmarks/helpers.h - src: - - test/cpp/microbenchmarks/helpers.cc - deps: - - benchmark - - grpc++ - - grpc_test_util - - grpc - name: grpc_cli_libs build: private language: c++ @@ -3045,7 +3031,6 @@ targets: src: - test/cpp/microbenchmarks/bm_call_create.cc deps: - - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3065,7 +3050,6 @@ targets: src: - test/cpp/microbenchmarks/bm_chttp2_hpack.cc deps: - - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3085,7 +3069,6 @@ targets: src: - test/cpp/microbenchmarks/bm_closure.cc deps: - - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3105,7 +3088,6 @@ targets: src: - test/cpp/microbenchmarks/bm_cq.cc deps: - - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3125,75 +3107,6 @@ targets: src: - test/cpp/microbenchmarks/bm_error.cc deps: - - grpc_benchmark - - benchmark - - grpc++_test_util - - grpc_test_util - - grpc++ - - grpc - - gpr_test_util - - gpr - args: - - --benchmark_min_time=0 - platforms: - - mac - - linux - - posix -- name: bm_fullstack_streaming_ping_pong - build: test - language: c++ - src: - - test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc - deps: - - grpc_benchmark - - benchmark - - grpc++_test_util - - grpc_test_util - - grpc++ - - grpc - - gpr_test_util - - gpr - args: - - --benchmark_min_time=0 - excluded_poll_engines: - - poll - - poll-cv - platforms: - - mac - - linux - - posix - timeout_seconds: 1200 -- name: bm_fullstack_streaming_pump - build: test - language: c++ - src: - - test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc - deps: - - grpc_benchmark - - benchmark - - grpc++_test_util - - grpc_test_util - - grpc++ - - grpc - - gpr_test_util - - gpr - args: - - --benchmark_min_time=0 - excluded_poll_engines: - - poll - - poll-cv - platforms: - - mac - - linux - - posix - timeout_seconds: 1200 -- name: bm_fullstack_trickle - build: test - language: c++ - src: - - test/cpp/microbenchmarks/bm_fullstack_trickle.cc - deps: - - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3203,21 +3116,16 @@ targets: - gpr args: - --benchmark_min_time=0 - excluded_poll_engines: - - poll - - poll-cv platforms: - mac - linux - posix - timeout_seconds: 1200 -- name: bm_fullstack_unary_ping_pong +- name: bm_fullstack build: test language: c++ src: - - test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc + - test/cpp/microbenchmarks/bm_fullstack.cc deps: - - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3241,7 +3149,6 @@ targets: src: - test/cpp/microbenchmarks/bm_metadata.cc deps: - - grpc_benchmark - benchmark - grpc_test_util - grpc diff --git a/include/grpc/impl/codegen/sync.h b/include/grpc/impl/codegen/sync.h index 6a8e8a644f9..96aec0c2c09 100644 --- a/include/grpc/impl/codegen/sync.h +++ b/include/grpc/impl/codegen/sync.h @@ -52,10 +52,6 @@ provides no memory barriers. */ -#ifdef __cplusplus -extern "C" { -#endif - /* Platform-specific type declarations of gpr_mu and gpr_cv. */ #include #include @@ -68,8 +64,4 @@ extern "C" { #error Unable to determine platform for sync #endif -#ifdef __cplusplus -} -#endif - #endif /* GRPC_IMPL_CODEGEN_SYNC_H */ diff --git a/test/core/end2end/BUILD b/test/core/end2end/BUILD index 0cef7aa01df..a40fb8e083f 100644 --- a/test/core/end2end/BUILD +++ b/test/core/end2end/BUILD @@ -63,8 +63,8 @@ cc_library( cc_library( name = 'http_proxy', - hdrs = ['fixtures/http_proxy_fixture.h'], - srcs = ['fixtures/http_proxy_fixture.c'], + hdrs = ['fixtures/http_proxy.h'], + srcs = ['fixtures/http_proxy.c'], copts = ['-std=c99'], deps = ['//:gpr', '//:grpc', '//test/core/util:grpc_test_util'] ) diff --git a/test/core/end2end/fixtures/h2_http_proxy.c b/test/core/end2end/fixtures/h2_http_proxy.c index 55c65fa70ef..44b223664ab 100644 --- a/test/core/end2end/fixtures/h2_http_proxy.c +++ b/test/core/end2end/fixtures/h2_http_proxy.c @@ -49,7 +49,7 @@ #include "src/core/lib/support/env.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/server.h" -#include "test/core/end2end/fixtures/http_proxy_fixture.h" +#include "test/core/end2end/fixtures/http_proxy.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/test/core/end2end/fixtures/http_proxy_fixture.c b/test/core/end2end/fixtures/http_proxy.c similarity index 99% rename from test/core/end2end/fixtures/http_proxy_fixture.c rename to test/core/end2end/fixtures/http_proxy.c index bcd1c9914b2..9ccb1263ee7 100644 --- a/test/core/end2end/fixtures/http_proxy_fixture.c +++ b/test/core/end2end/fixtures/http_proxy.c @@ -31,7 +31,7 @@ * */ -#include "test/core/end2end/fixtures/http_proxy_fixture.h" +#include "test/core/end2end/fixtures/http_proxy.h" #include "src/core/lib/iomgr/sockaddr.h" diff --git a/test/core/end2end/fixtures/http_proxy_fixture.h b/test/core/end2end/fixtures/http_proxy.h similarity index 100% rename from test/core/end2end/fixtures/http_proxy_fixture.h rename to test/core/end2end/fixtures/http_proxy.h diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index f25bcd2df82..e2e5bbbe00e 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -55,10 +55,12 @@ extern "C" { #include "src/core/lib/transport/transport_impl.h" } -#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -auto &force_library_initialization = Library::get(); +static struct Init { + Init() { grpc_init(); } + ~Init() { grpc_shutdown(); } +} g_init; class BaseChannelFixture { public: @@ -87,7 +89,6 @@ class LameChannel : public BaseChannelFixture { template static void BM_CallCreateDestroy(benchmark::State &state) { - TrackCounters track_counters; Fixture fixture; grpc_completion_queue *cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); @@ -99,7 +100,6 @@ static void BM_CallCreateDestroy(benchmark::State &state) { deadline, NULL)); } grpc_completion_queue_destroy(cq); - track_counters.Finish(state); } BENCHMARK_TEMPLATE(BM_CallCreateDestroy, InsecureChannel); @@ -316,7 +316,6 @@ class SendEmptyMetadata { // perform on said filter. template static void BM_IsolatedFilter(benchmark::State &state) { - TrackCounters track_counters; Fixture fixture; std::ostringstream label; @@ -372,7 +371,6 @@ static void BM_IsolatedFilter(benchmark::State &state) { gpr_free(call_stack); state.SetLabel(label.str()); - track_counters.Finish(state); } typedef Fixture NoFilter; diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc index 563db758f71..5fb3f371306 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc @@ -42,17 +42,18 @@ extern "C" { #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/transport/static_metadata.h" } -#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -auto &force_library_initialization = Library::get(); +static struct Init { + Init() { grpc_init(); } + ~Init() { grpc_shutdown(); } +} g_init; //////////////////////////////////////////////////////////////////////////////// // HPACK encoder // static void BM_HpackEncoderInitDestroy(benchmark::State &state) { - TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_chttp2_hpack_compressor c; while (state.KeepRunning()) { @@ -61,13 +62,11 @@ static void BM_HpackEncoderInitDestroy(benchmark::State &state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_HpackEncoderInitDestroy); template static void BM_HpackEncoderEncodeHeader(benchmark::State &state) { - TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_metadata_batch b; @@ -103,7 +102,6 @@ static void BM_HpackEncoderEncodeHeader(benchmark::State &state) { << " header_bytes/iter:" << (static_cast(stats.header_bytes) / static_cast(state.iterations())); state.SetLabel(label.str()); - track_counters.Finish(state); } namespace hpack_encoder_fixtures { @@ -218,7 +216,6 @@ BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, // static void BM_HpackParserInitDestroy(benchmark::State &state) { - TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_chttp2_hpack_parser p; while (state.KeepRunning()) { @@ -227,7 +224,6 @@ static void BM_HpackParserInitDestroy(benchmark::State &state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_HpackParserInitDestroy); @@ -238,7 +234,6 @@ static void UnrefHeader(grpc_exec_ctx *exec_ctx, void *user_data, template static void BM_HpackParserParseHeader(benchmark::State &state) { - TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; std::vector init_slices = Fixture::GetInitSlices(); std::vector benchmark_slices = Fixture::GetBenchmarkSlices(); @@ -257,7 +252,6 @@ static void BM_HpackParserParseHeader(benchmark::State &state) { } grpc_chttp2_hpack_parser_destroy(&exec_ctx, &p); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } namespace hpack_parser_fixtures { diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc index 28a385b6c11..1f54e8c8b17 100644 --- a/test/cpp/microbenchmarks/bm_closure.cc +++ b/test/cpp/microbenchmarks/bm_closure.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2017, Google Inc. + * Copyright 2015, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -42,47 +42,86 @@ extern "C" { #include "src/core/lib/support/spinlock.h" } -#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -auto& force_library_initialization = Library::get(); +#include + +#ifdef GPR_LOW_LEVEL_COUNTERS +extern "C" gpr_atm gpr_mu_locks; +#endif + +static class InitializeStuff { + public: + InitializeStuff() { grpc_init(); } + ~InitializeStuff() { grpc_shutdown(); } +} initialize_stuff; + +class TrackCounters { + public: + TrackCounters(benchmark::State& state) : state_(state) {} + + ~TrackCounters() { + std::ostringstream out; +#ifdef GPR_LOW_LEVEL_COUNTERS + out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - + mu_locks_at_start_) / + (double)state_.iterations()) + << " atm_cas/iter:" + << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) - + atm_cas_at_start_) / + (double)state_.iterations()) + << " atm_add/iter:" + << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - + atm_add_at_start_) / + (double)state_.iterations()); +#endif + state_.SetLabel(out.str()); + } + + private: + benchmark::State& state_; +#ifdef GPR_LOW_LEVEL_COUNTERS + const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks); + const size_t atm_cas_at_start_ = + gpr_atm_no_barrier_load(&gpr_counter_atm_cas); + const size_t atm_add_at_start_ = + gpr_atm_no_barrier_load(&gpr_counter_atm_add); +#endif +}; static void BM_NoOpExecCtx(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); while (state.KeepRunning()) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_exec_ctx_finish(&exec_ctx); } - track_counters.Finish(state); } BENCHMARK(BM_NoOpExecCtx); static void BM_WellFlushed(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; while (state.KeepRunning()) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_WellFlushed); static void DoNothing(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {} static void BM_ClosureInitAgainstExecCtx(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_closure c; while (state.KeepRunning()) { benchmark::DoNotOptimize( grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx)); } - track_counters.Finish(state); } BENCHMARK(BM_ClosureInitAgainstExecCtx); static void BM_ClosureInitAgainstCombiner(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -92,12 +131,11 @@ static void BM_ClosureInitAgainstCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureInitAgainstCombiner); static void BM_ClosureRunOnExecCtx(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_closure c; grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -106,12 +144,11 @@ static void BM_ClosureRunOnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureRunOnExecCtx); static void BM_ClosureCreateAndRun(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; while (state.KeepRunning()) { grpc_closure_run(&exec_ctx, grpc_closure_create(DoNothing, NULL, @@ -119,12 +156,11 @@ static void BM_ClosureCreateAndRun(benchmark::State& state) { GRPC_ERROR_NONE); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureCreateAndRun); static void BM_ClosureInitAndRun(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_closure c; while (state.KeepRunning()) { @@ -133,12 +169,11 @@ static void BM_ClosureInitAndRun(benchmark::State& state) { GRPC_ERROR_NONE); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureInitAndRun); static void BM_ClosureSchedOnExecCtx(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_closure c; grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -147,12 +182,11 @@ static void BM_ClosureSchedOnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureSchedOnExecCtx); static void BM_ClosureSched2OnExecCtx(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_closure c1; grpc_closure c2; grpc_closure_init(&c1, DoNothing, NULL, grpc_schedule_on_exec_ctx); @@ -164,12 +198,11 @@ static void BM_ClosureSched2OnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureSched2OnExecCtx); static void BM_ClosureSched3OnExecCtx(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_closure c1; grpc_closure c2; grpc_closure c3; @@ -184,12 +217,11 @@ static void BM_ClosureSched3OnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureSched3OnExecCtx); static void BM_AcquireMutex(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); // for comparison with the combiner stuff below gpr_mu mu; gpr_mu_init(&mu); @@ -200,12 +232,11 @@ static void BM_AcquireMutex(benchmark::State& state) { gpr_mu_unlock(&mu); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_AcquireMutex); static void BM_TryAcquireMutex(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); // for comparison with the combiner stuff below gpr_mu mu; gpr_mu_init(&mu); @@ -219,12 +250,11 @@ static void BM_TryAcquireMutex(benchmark::State& state) { } } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_TryAcquireMutex); static void BM_AcquireSpinlock(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); // for comparison with the combiner stuff below gpr_spinlock mu = GPR_SPINLOCK_INITIALIZER; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -234,12 +264,11 @@ static void BM_AcquireSpinlock(benchmark::State& state) { gpr_spinlock_unlock(&mu); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_AcquireSpinlock); static void BM_TryAcquireSpinlock(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); // for comparison with the combiner stuff below gpr_spinlock mu = GPR_SPINLOCK_INITIALIZER; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -252,12 +281,11 @@ static void BM_TryAcquireSpinlock(benchmark::State& state) { } } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_TryAcquireSpinlock); static void BM_ClosureSchedOnCombiner(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c; grpc_closure_init(&c, DoNothing, NULL, @@ -269,12 +297,11 @@ static void BM_ClosureSchedOnCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureSchedOnCombiner); static void BM_ClosureSched2OnCombiner(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c1; grpc_closure c2; @@ -290,12 +317,11 @@ static void BM_ClosureSched2OnCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureSched2OnCombiner); static void BM_ClosureSched3OnCombiner(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c1; grpc_closure c2; @@ -315,12 +341,11 @@ static void BM_ClosureSched3OnCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureSched3OnCombiner); static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_combiner* combiner1 = grpc_combiner_create(NULL); grpc_combiner* combiner2 = grpc_combiner_create(NULL); grpc_closure c1; @@ -338,12 +363,11 @@ static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) { GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished"); GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished"); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureSched2OnTwoCombiners); static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_combiner* combiner1 = grpc_combiner_create(NULL); grpc_combiner* combiner2 = grpc_combiner_create(NULL); grpc_closure c1; @@ -369,7 +393,6 @@ static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) { GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished"); GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished"); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureSched4OnTwoCombiners); @@ -405,17 +428,16 @@ class Rescheduler { }; static void BM_ClosureReschedOnExecCtx(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; Rescheduler r(state, grpc_schedule_on_exec_ctx); r.ScheduleFirst(&exec_ctx); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureReschedOnExecCtx); static void BM_ClosureReschedOnCombiner(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_combiner* combiner = grpc_combiner_create(NULL); Rescheduler r(state, grpc_combiner_scheduler(combiner, false)); @@ -423,12 +445,11 @@ static void BM_ClosureReschedOnCombiner(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureReschedOnCombiner); static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) { - TrackCounters track_counters; + TrackCounters track_counters(state); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_combiner* combiner = grpc_combiner_create(NULL); Rescheduler r(state, grpc_combiner_finally_scheduler(combiner, false)); @@ -437,7 +458,6 @@ static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_ClosureReschedOnCombinerFinally); diff --git a/test/cpp/microbenchmarks/bm_cq.cc b/test/cpp/microbenchmarks/bm_cq.cc index 91e6a851017..c017474bf4a 100644 --- a/test/cpp/microbenchmarks/bm_cq.cc +++ b/test/cpp/microbenchmarks/bm_cq.cc @@ -38,7 +38,6 @@ #include #include -#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" extern "C" { @@ -48,23 +47,27 @@ extern "C" { namespace grpc { namespace testing { -auto& force_library_initialization = Library::get(); +static class InitializeStuff { + public: + InitializeStuff() { init_lib_.init(); } + ~InitializeStuff() { init_lib_.shutdown(); } + + private: + internal::GrpcLibrary init_lib_; + internal::GrpcLibraryInitializer init_; +} initialize_stuff; static void BM_CreateDestroyCpp(benchmark::State& state) { - TrackCounters track_counters; while (state.KeepRunning()) { CompletionQueue cq; } - track_counters.Finish(state); } BENCHMARK(BM_CreateDestroyCpp); static void BM_CreateDestroyCore(benchmark::State& state) { - TrackCounters track_counters; while (state.KeepRunning()) { grpc_completion_queue_destroy(grpc_completion_queue_create(NULL)); } - track_counters.Finish(state); } BENCHMARK(BM_CreateDestroyCore); @@ -77,7 +80,6 @@ class DummyTag final : public CompletionQueueTag { }; static void BM_Pass1Cpp(benchmark::State& state) { - TrackCounters track_counters; CompletionQueue cq; grpc_completion_queue* c_cq = cq.cq(); while (state.KeepRunning()) { @@ -92,12 +94,10 @@ static void BM_Pass1Cpp(benchmark::State& state) { bool ok; cq.Next(&tag, &ok); } - track_counters.Finish(state); } BENCHMARK(BM_Pass1Cpp); static void BM_Pass1Core(benchmark::State& state) { - TrackCounters track_counters; grpc_completion_queue* cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); while (state.KeepRunning()) { @@ -110,12 +110,10 @@ static void BM_Pass1Core(benchmark::State& state) { grpc_completion_queue_next(cq, deadline, NULL); } grpc_completion_queue_destroy(cq); - track_counters.Finish(state); } BENCHMARK(BM_Pass1Core); static void BM_Pluck1Core(benchmark::State& state) { - TrackCounters track_counters; grpc_completion_queue* cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); while (state.KeepRunning()) { @@ -128,19 +126,16 @@ static void BM_Pluck1Core(benchmark::State& state) { grpc_completion_queue_pluck(cq, NULL, deadline, NULL); } grpc_completion_queue_destroy(cq); - track_counters.Finish(state); } BENCHMARK(BM_Pluck1Core); static void BM_EmptyCore(benchmark::State& state) { - TrackCounters track_counters; grpc_completion_queue* cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); while (state.KeepRunning()) { grpc_completion_queue_next(cq, deadline, NULL); } grpc_completion_queue_destroy(cq); - track_counters.Finish(state); } BENCHMARK(BM_EmptyCore); diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc index 95f19e75860..8a4b86f281a 100644 --- a/test/cpp/microbenchmarks/bm_error.cc +++ b/test/cpp/microbenchmarks/bm_error.cc @@ -40,11 +40,8 @@ extern "C" { #include "src/core/lib/transport/error_utils.h" } -#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -auto& force_library_initialization = Library::get(); - class ErrorDeleter { public: void operator()(grpc_error* error) { GRPC_ERROR_UNREF(error); } @@ -52,38 +49,31 @@ class ErrorDeleter { typedef std::unique_ptr ErrorPtr; static void BM_ErrorCreate(benchmark::State& state) { - TrackCounters track_counters; while (state.KeepRunning()) { GRPC_ERROR_UNREF(GRPC_ERROR_CREATE("Error")); } - track_counters.Finish(state); } BENCHMARK(BM_ErrorCreate); static void BM_ErrorCreateAndSetStatus(benchmark::State& state) { - TrackCounters track_counters; while (state.KeepRunning()) { GRPC_ERROR_UNREF(grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_ABORTED)); } - track_counters.Finish(state); } BENCHMARK(BM_ErrorCreateAndSetStatus); static void BM_ErrorRefUnref(benchmark::State& state) { - TrackCounters track_counters; grpc_error* error = GRPC_ERROR_CREATE("Error"); while (state.KeepRunning()) { GRPC_ERROR_UNREF(GRPC_ERROR_REF(error)); } GRPC_ERROR_UNREF(error); - track_counters.Finish(state); } BENCHMARK(BM_ErrorRefUnref); static void BM_ErrorUnrefNone(benchmark::State& state) { - TrackCounters track_counters; while (state.KeepRunning()) { GRPC_ERROR_UNREF(GRPC_ERROR_NONE); } @@ -91,36 +81,30 @@ static void BM_ErrorUnrefNone(benchmark::State& state) { BENCHMARK(BM_ErrorUnrefNone); static void BM_ErrorGetIntFromNoError(benchmark::State& state) { - TrackCounters track_counters; while (state.KeepRunning()) { intptr_t value; grpc_error_get_int(GRPC_ERROR_NONE, GRPC_ERROR_INT_GRPC_STATUS, &value); } - track_counters.Finish(state); } BENCHMARK(BM_ErrorGetIntFromNoError); static void BM_ErrorGetMissingInt(benchmark::State& state) { - TrackCounters track_counters; ErrorPtr error( grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_INDEX, 1)); while (state.KeepRunning()) { intptr_t value; grpc_error_get_int(error.get(), GRPC_ERROR_INT_OFFSET, &value); } - track_counters.Finish(state); } BENCHMARK(BM_ErrorGetMissingInt); static void BM_ErrorGetPresentInt(benchmark::State& state) { - TrackCounters track_counters; ErrorPtr error( grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_OFFSET, 1)); while (state.KeepRunning()) { intptr_t value; grpc_error_get_int(error.get(), GRPC_ERROR_INT_OFFSET, &value); } - track_counters.Finish(state); } BENCHMARK(BM_ErrorGetPresentInt); @@ -193,27 +177,22 @@ class ErrorWithNestedGrpcStatus { template static void BM_ErrorStringOnNewError(benchmark::State& state) { - TrackCounters track_counters; while (state.KeepRunning()) { Fixture fixture; grpc_error_string(fixture.error()); } - track_counters.Finish(state); } template static void BM_ErrorStringRepeatedly(benchmark::State& state) { - TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_error_string(fixture.error()); } - track_counters.Finish(state); } template static void BM_ErrorGetStatus(benchmark::State& state) { - TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_status_code status; @@ -221,41 +200,34 @@ static void BM_ErrorGetStatus(benchmark::State& state) { grpc_error_get_status(fixture.error(), fixture.deadline(), &status, &msg, NULL); } - track_counters.Finish(state); } template static void BM_ErrorGetStatusCode(benchmark::State& state) { - TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_status_code status; grpc_error_get_status(fixture.error(), fixture.deadline(), &status, NULL, NULL); } - track_counters.Finish(state); } template static void BM_ErrorHttpError(benchmark::State& state) { - TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_http2_error_code error; grpc_error_get_status(fixture.error(), fixture.deadline(), NULL, NULL, &error); } - track_counters.Finish(state); } template static void BM_HasClearGrpcStatus(benchmark::State& state) { - TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_error_has_clear_grpc_status(fixture.error()); } - track_counters.Finish(state); } #define BENCHMARK_SUITE(fixture) \ diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc new file mode 100644 index 00000000000..48e131f1be0 --- /dev/null +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -0,0 +1,1079 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Benchmark gRPC end2end in various configurations */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" +#include "src/core/ext/transport/chttp2/transport/internal.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/endpoint.h" +#include "src/core/lib/iomgr/endpoint_pair.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/iomgr/tcp_posix.h" +#include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/completion_queue.h" +#include "src/core/lib/surface/server.h" +#include "test/core/util/memory_counters.h" +#include "test/core/util/passthru_endpoint.h" +#include "test/core/util/port.h" +#include "test/core/util/trickle_endpoint.h" +} +#include "src/core/lib/profiling/timers.h" +#include "src/cpp/client/create_channel_internal.h" +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +namespace grpc { +namespace testing { + +static class InitializeStuff { + public: + InitializeStuff() { + grpc_memory_counters_init(); + init_lib_.init(); + rq_ = grpc_resource_quota_create("bm"); + } + + ~InitializeStuff() { init_lib_.shutdown(); } + + grpc_resource_quota* rq() { return rq_; } + + private: + internal::GrpcLibrary init_lib_; + grpc_resource_quota* rq_; +} initialize_stuff; + +/******************************************************************************* + * FIXTURES + */ + +static void ApplyCommonServerBuilderConfig(ServerBuilder* b) { + b->SetMaxReceiveMessageSize(INT_MAX); + b->SetMaxSendMessageSize(INT_MAX); +} + +static void ApplyCommonChannelArguments(ChannelArguments* c) { + c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX); + c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX); +} + +#ifdef GPR_LOW_LEVEL_COUNTERS +extern "C" gpr_atm gpr_mu_locks; +extern "C" gpr_atm gpr_counter_atm_cas; +extern "C" gpr_atm gpr_counter_atm_add; +#endif + +class BaseFixture { + public: + void Finish(benchmark::State& s) { + std::ostringstream out; + this->AddToLabel(out, s); +#ifdef GPR_LOW_LEVEL_COUNTERS + out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - + mu_locks_at_start_) / + (double)s.iterations()) + << " atm_cas/iter:" + << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) - + atm_cas_at_start_) / + (double)s.iterations()) + << " atm_add/iter:" + << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - + atm_add_at_start_) / + (double)s.iterations()); +#endif + grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); + out << " allocs/iter:" + << ((double)(counters_at_end.total_allocs_absolute - + counters_at_start_.total_allocs_absolute) / + (double)s.iterations()); + auto label = out.str(); + if (label.length() && label[0] == ' ') { + label = label.substr(1); + } + s.SetLabel(label); + } + + virtual void AddToLabel(std::ostream& out, benchmark::State& s) = 0; + + private: +#ifdef GPR_LOW_LEVEL_COUNTERS + const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks); + const size_t atm_cas_at_start_ = + gpr_atm_no_barrier_load(&gpr_counter_atm_cas); + const size_t atm_add_at_start_ = + gpr_atm_no_barrier_load(&gpr_counter_atm_add); +#endif + grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot(); +}; + +class FullstackFixture : public BaseFixture { + public: + FullstackFixture(Service* service, const grpc::string& address) { + ServerBuilder b; + b.AddListeningPort(address, InsecureServerCredentials()); + cq_ = b.AddCompletionQueue(true); + b.RegisterService(service); + ApplyCommonServerBuilderConfig(&b); + server_ = b.BuildAndStart(); + ChannelArguments args; + ApplyCommonChannelArguments(&args); + channel_ = CreateCustomChannel(address, InsecureChannelCredentials(), args); + } + + virtual ~FullstackFixture() { + server_->Shutdown(); + cq_->Shutdown(); + void* tag; + bool ok; + while (cq_->Next(&tag, &ok)) { + } + } + + ServerCompletionQueue* cq() { return cq_.get(); } + std::shared_ptr channel() { return channel_; } + + private: + std::unique_ptr server_; + std::unique_ptr cq_; + std::shared_ptr channel_; +}; + +class TCP : public FullstackFixture { + public: + TCP(Service* service) : FullstackFixture(service, MakeAddress()) {} + + void AddToLabel(std::ostream& out, benchmark::State& state) {} + + private: + static grpc::string MakeAddress() { + int port = grpc_pick_unused_port_or_die(); + std::stringstream addr; + addr << "localhost:" << port; + return addr.str(); + } +}; + +class UDS : public FullstackFixture { + public: + UDS(Service* service) : FullstackFixture(service, MakeAddress()) {} + + void AddToLabel(std::ostream& out, benchmark::State& state) override {} + + private: + static grpc::string MakeAddress() { + int port = grpc_pick_unused_port_or_die(); // just for a unique id - not a + // real port + std::stringstream addr; + addr << "unix:/tmp/bm_fullstack." << port; + return addr.str(); + } +}; + +class EndpointPairFixture : public BaseFixture { + public: + EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) + : endpoint_pair_(endpoints) { + ServerBuilder b; + cq_ = b.AddCompletionQueue(true); + b.RegisterService(service); + ApplyCommonServerBuilderConfig(&b); + server_ = b.BuildAndStart(); + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + + /* add server endpoint to server_ */ + { + const grpc_channel_args* server_args = + grpc_server_get_channel_args(server_->c_server()); + server_transport_ = grpc_create_chttp2_transport( + &exec_ctx, server_args, endpoints.server, 0 /* is_client */); + + grpc_pollset** pollsets; + size_t num_pollsets = 0; + grpc_server_get_pollsets(server_->c_server(), &pollsets, &num_pollsets); + + for (size_t i = 0; i < num_pollsets; i++) { + grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]); + } + + grpc_server_setup_transport(&exec_ctx, server_->c_server(), + server_transport_, NULL, server_args); + grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL); + } + + /* create channel */ + { + ChannelArguments args; + args.SetString(GRPC_ARG_DEFAULT_AUTHORITY, "test.authority"); + ApplyCommonChannelArguments(&args); + + grpc_channel_args c_args = args.c_channel_args(); + client_transport_ = + grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1); + GPR_ASSERT(client_transport_); + grpc_channel* channel = + grpc_channel_create(&exec_ctx, "target", &c_args, + GRPC_CLIENT_DIRECT_CHANNEL, client_transport_); + grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL); + + channel_ = CreateChannelInternal("", channel); + } + + grpc_exec_ctx_finish(&exec_ctx); + } + + virtual ~EndpointPairFixture() { + server_->Shutdown(); + cq_->Shutdown(); + void* tag; + bool ok; + while (cq_->Next(&tag, &ok)) { + } + } + + ServerCompletionQueue* cq() { return cq_.get(); } + std::shared_ptr channel() { return channel_; } + + protected: + grpc_endpoint_pair endpoint_pair_; + grpc_transport* client_transport_; + grpc_transport* server_transport_; + + private: + std::unique_ptr server_; + std::unique_ptr cq_; + std::shared_ptr channel_; +}; + +class SockPair : public EndpointPairFixture { + public: + SockPair(Service* service) + : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair( + "test", initialize_stuff.rq(), 8192)) { + } + + void AddToLabel(std::ostream& out, benchmark::State& state) {} +}; + +class InProcessCHTTP2 : public EndpointPairFixture { + public: + InProcessCHTTP2(Service* service) + : EndpointPairFixture(service, MakeEndpoints()) {} + + void AddToLabel(std::ostream& out, benchmark::State& state) { + out << " writes/iter:" + << ((double)stats_.num_writes / (double)state.iterations()); + } + + private: + grpc_passthru_endpoint_stats stats_; + + grpc_endpoint_pair MakeEndpoints() { + grpc_endpoint_pair p; + grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(), + &stats_); + return p; + } +}; + +class TrickledCHTTP2 : public EndpointPairFixture { + public: + TrickledCHTTP2(Service* service, size_t megabits_per_second) + : EndpointPairFixture(service, MakeEndpoints(megabits_per_second)) {} + + void AddToLabel(std::ostream& out, benchmark::State& state) { + out << " writes/iter:" + << ((double)stats_.num_writes / (double)state.iterations()) + << " cli_transport_stalls/iter:" + << ((double) + client_stats_.streams_stalled_due_to_transport_flow_control / + (double)state.iterations()) + << " cli_stream_stalls/iter:" + << ((double)client_stats_.streams_stalled_due_to_stream_flow_control / + (double)state.iterations()) + << " svr_transport_stalls/iter:" + << ((double) + server_stats_.streams_stalled_due_to_transport_flow_control / + (double)state.iterations()) + << " svr_stream_stalls/iter:" + << ((double)server_stats_.streams_stalled_due_to_stream_flow_control / + (double)state.iterations()); + } + + void Step() { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + size_t client_backlog = + grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.client); + size_t server_backlog = + grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server); + grpc_exec_ctx_finish(&exec_ctx); + + UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_, + client_backlog); + UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_, + server_backlog); + } + + private: + grpc_passthru_endpoint_stats stats_; + struct Stats { + int streams_stalled_due_to_stream_flow_control = 0; + int streams_stalled_due_to_transport_flow_control = 0; + }; + Stats client_stats_; + Stats server_stats_; + + grpc_endpoint_pair MakeEndpoints(size_t kilobits) { + grpc_endpoint_pair p; + grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(), + &stats_); + double bytes_per_second = 125.0 * kilobits; + p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second); + p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second); + return p; + } + + void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) { + if (backlog == 0) { + if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) { + s->streams_stalled_due_to_stream_flow_control++; + } + if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) { + s->streams_stalled_due_to_transport_flow_control++; + } + } + } +}; + +/******************************************************************************* + * CONTEXT MUTATORS + */ + +static const int kPregenerateKeyCount = 100000; + +template +auto MakeVector(size_t length, F f) -> std::vector { + std::vector out; + out.reserve(length); + for (size_t i = 0; i < length; i++) { + out.push_back(f()); + } + return out; +} + +class NoOpMutator { + public: + template + NoOpMutator(ContextType* context) {} +}; + +template +class RandomBinaryMetadata { + public: + static const grpc::string& Key() { return kKey; } + + static const grpc::string& Value() { + return kValues[rand() % kValues.size()]; + } + + private: + static const grpc::string kKey; + static const std::vector kValues; + + static grpc::string GenerateOneString() { + grpc::string s; + s.reserve(length + 1); + for (int i = 0; i < length; i++) { + s += (char)rand(); + } + return s; + } +}; + +template +const grpc::string RandomBinaryMetadata::kKey = "foo-bin"; + +template +const std::vector RandomBinaryMetadata::kValues = + MakeVector(kPregenerateKeyCount, GenerateOneString); + +template +class RandomAsciiMetadata { + public: + static const grpc::string& Key() { return kKey; } + + static const grpc::string& Value() { + return kValues[rand() % kValues.size()]; + } + + private: + static const grpc::string kKey; + static const std::vector kValues; + + static grpc::string GenerateOneString() { + grpc::string s; + s.reserve(length + 1); + for (int i = 0; i < length; i++) { + s += (char)(rand() % 26 + 'a'); + } + return s; + } +}; + +template +const grpc::string RandomAsciiMetadata::kKey = "foo"; + +template +const std::vector RandomAsciiMetadata::kValues = + MakeVector(kPregenerateKeyCount, GenerateOneString); + +template +class Client_AddMetadata : public NoOpMutator { + public: + Client_AddMetadata(ClientContext* context) : NoOpMutator(context) { + for (int i = 0; i < kNumKeys; i++) { + context->AddMetadata(Generator::Key(), Generator::Value()); + } + } +}; + +template +class Server_AddInitialMetadata : public NoOpMutator { + public: + Server_AddInitialMetadata(ServerContext* context) : NoOpMutator(context) { + for (int i = 0; i < kNumKeys; i++) { + context->AddInitialMetadata(Generator::Key(), Generator::Value()); + } + } +}; + +/******************************************************************************* + * BENCHMARKING KERNELS + */ + +static void* tag(intptr_t x) { return reinterpret_cast(x); } + +template +static void BM_UnaryPingPong(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + EchoRequest send_request; + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_request.set_message(std::string(state.range(0), 'a')); + } + if (state.range(1) > 0) { + send_response.set_message(std::string(state.range(1), 'a')); + } + Status recv_status; + struct ServerEnv { + ServerContext ctx; + EchoRequest recv_request; + grpc::ServerAsyncResponseWriter response_writer; + ServerEnv() : response_writer(&ctx) {} + }; + uint8_t server_env_buffer[2 * sizeof(ServerEnv)]; + ServerEnv* server_env[2] = { + reinterpret_cast(server_env_buffer), + reinterpret_cast(server_env_buffer + sizeof(ServerEnv))}; + new (server_env[0]) ServerEnv; + new (server_env[1]) ServerEnv; + service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request, + &server_env[0]->response_writer, fixture->cq(), + fixture->cq(), tag(0)); + service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request, + &server_env[1]->response_writer, fixture->cq(), + fixture->cq(), tag(1)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + recv_response.Clear(); + ClientContext cli_ctx; + ClientContextMutator cli_ctx_mut(&cli_ctx); + std::unique_ptr> response_reader( + stub->AsyncEcho(&cli_ctx, send_request, fixture->cq())); + void* t; + bool ok; + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + GPR_ASSERT(t == tag(0) || t == tag(1)); + intptr_t slot = reinterpret_cast(t); + ServerEnv* senv = server_env[slot]; + ServerContextMutator svr_ctx_mut(&senv->ctx); + senv->response_writer.Finish(send_response, Status::OK, tag(3)); + response_reader->Finish(&recv_response, &recv_status, tag(4)); + for (int i = (1 << 3) | (1 << 4); i != 0;) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int tagnum = (int)reinterpret_cast(t); + GPR_ASSERT(i & (1 << tagnum)); + i -= 1 << tagnum; + } + GPR_ASSERT(recv_status.ok()); + + senv->~ServerEnv(); + senv = new (senv) ServerEnv(); + service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer, + fixture->cq(), fixture->cq(), tag(slot)); + } + fixture->Finish(state); + fixture.reset(); + server_env[0]->~ServerEnv(); + server_env[1]->~ServerEnv(); + state.SetBytesProcessed(state.range(0) * state.iterations() + + state.range(1) * state.iterations()); +} + +// Repeatedly makes Streaming Bidi calls (exchanging a configurable number of +// messages in each call) in a loop on a single channel +// +// First parmeter (i.e state.range(0)): Message size (in bytes) to use +// Second parameter (i.e state.range(1)): Number of ping pong messages. +// Note: One ping-pong means two messages (one from client to server and +// the other from server to client): +template +static void BM_StreamingPingPong(benchmark::State& state) { + const int msg_size = state.range(0); + const int max_ping_pongs = state.range(1); + + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoResponse send_response; + EchoResponse recv_response; + EchoRequest send_request; + EchoRequest recv_request; + + if (msg_size > 0) { + send_request.set_message(std::string(msg_size, 'a')); + send_response.set_message(std::string(msg_size, 'b')); + } + + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + + while (state.KeepRunning()) { + ServerContext svr_ctx; + ServerContextMutator svr_ctx_mut(&svr_ctx); + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + + ClientContext cli_ctx; + ClientContextMutator cli_ctx_mut(&cli_ctx); + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + + // Establish async stream between client side and server side + void* t; + bool ok; + int need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + + // Send 'max_ping_pongs' number of ping pong messages + int ping_pong_cnt = 0; + while (ping_pong_cnt < max_ping_pongs) { + request_rw->Write(send_request, tag(0)); // Start client send + response_rw.Read(&recv_request, tag(1)); // Start server recv + request_rw->Read(&recv_response, tag(2)); // Start client recv + + need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + + // If server recv is complete, start the server send operation + if (i == 1) { + response_rw.Write(send_response, tag(3)); + } + + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + + ping_pong_cnt++; + } + + request_rw->WritesDone(tag(0)); + response_rw.Finish(Status::OK, tag(1)); + + Status recv_status; + request_rw->Finish(&recv_status, tag(2)); + + need_tags = (1 << 0) | (1 << 1) | (1 << 2); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + + GPR_ASSERT(recv_status.ok()); + } + } + + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(msg_size * state.iterations() * max_ping_pongs * 2); +} + +// Repeatedly sends ping pong messages in a single streaming Bidi call in a loop +// First parmeter (i.e state.range(0)): Message size (in bytes) to use +template +static void BM_StreamingPingPongMsgs(benchmark::State& state) { + const int msg_size = state.range(0); + + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoResponse send_response; + EchoResponse recv_response; + EchoRequest send_request; + EchoRequest recv_request; + + if (msg_size > 0) { + send_request.set_message(std::string(msg_size, 'a')); + send_response.set_message(std::string(msg_size, 'b')); + } + + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + + ServerContext svr_ctx; + ServerContextMutator svr_ctx_mut(&svr_ctx); + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + + ClientContext cli_ctx; + ClientContextMutator cli_ctx_mut(&cli_ctx); + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + + // Establish async stream between client side and server side + void* t; + bool ok; + int need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + request_rw->Write(send_request, tag(0)); // Start client send + response_rw.Read(&recv_request, tag(1)); // Start server recv + request_rw->Read(&recv_response, tag(2)); // Start client recv + + need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + + // If server recv is complete, start the server send operation + if (i == 1) { + response_rw.Write(send_response, tag(3)); + } + + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + + request_rw->WritesDone(tag(0)); + response_rw.Finish(Status::OK, tag(1)); + Status recv_status; + request_rw->Finish(&recv_status, tag(2)); + + need_tags = (1 << 0) | (1 << 1) | (1 << 2); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + + GPR_ASSERT(recv_status.ok()); + } + + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(msg_size * state.iterations() * 2); +} + +template +static void BM_PumpStreamClientToServer(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoRequest send_request; + EchoRequest recv_request; + if (state.range(0) > 0) { + send_request.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + response_rw.Read(&recv_request, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + request_rw->Write(send_request, tag(1)); + while (true) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + if (t == tag(0)) { + response_rw.Read(&recv_request, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + request_rw->WritesDone(tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +template +static void BM_PumpStreamServerToClient(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_response.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + request_rw->Read(&recv_response, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + response_rw.Write(send_response, tag(1)); + while (true) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + if (t == tag(0)) { + request_rw->Read(&recv_response, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + response_rw.Finish(Status::OK, tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) { + while (true) { + switch (fixture->cq()->AsyncNext( + t, ok, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(100, GPR_TIMESPAN)))) { + case CompletionQueue::TIMEOUT: + fixture->Step(); + break; + case CompletionQueue::SHUTDOWN: + GPR_ASSERT(false); + break; + case CompletionQueue::GOT_EVENT: + return; + } + } +} + +static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture( + new TrickledCHTTP2(&service, state.range(1))); + { + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_response.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + TrickleCQNext(fixture.get(), &t, &ok); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + request_rw->Read(&recv_response, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + response_rw.Write(send_response, tag(1)); + while (true) { + TrickleCQNext(fixture.get(), &t, &ok); + if (t == tag(0)) { + request_rw->Read(&recv_response, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + response_rw.Finish(Status::OK, tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + TrickleCQNext(fixture.get(), &t, &ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +/******************************************************************************* + * CONFIGURATIONS + */ + +static void SweepSizesArgs(benchmark::internal::Benchmark* b) { + b->Args({0, 0}); + for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { + b->Args({i, 0}); + b->Args({0, i}); + b->Args({i, i}); + } +} + +BENCHMARK_TEMPLATE(BM_UnaryPingPong, TCP, NoOpMutator, NoOpMutator) + ->Apply(SweepSizesArgs); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, UDS, NoOpMutator, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, SockPair, NoOpMutator, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, NoOpMutator) + ->Apply(SweepSizesArgs); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, + NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 2>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 2>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 2>, + NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 100>) + ->Args({0, 0}); + +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) + ->Range(0, 128 * 1024 * 1024); + +static void TrickleArgs(benchmark::internal::Benchmark* b) { + for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { + for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) { + double expected_time = + static_cast(14 + i) / (125.0 * static_cast(j)); + if (expected_time > 0.01) continue; + b->Args({i, j}); + } + } +} + +BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs); + +// Generate Args for StreamingPingPong benchmarks. Currently generates args for +// only "small streams" (i.e streams with 0, 1 or 2 messages) +static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) { + int msg_size = 0; + + b->Args({0, 0}); // spl case: 0 ping-pong msgs (msg_size doesn't matter here) + + for (msg_size = 0; msg_size <= 128 * 1024 * 1024; + msg_size == 0 ? msg_size++ : msg_size *= 8) { + b->Args({msg_size, 1}); + b->Args({msg_size, 2}); + } +} + +BENCHMARK_TEMPLATE(BM_StreamingPingPong, InProcessCHTTP2, NoOpMutator, + NoOpMutator) + ->Apply(StreamingPingPongArgs); +BENCHMARK_TEMPLATE(BM_StreamingPingPong, TCP, NoOpMutator, NoOpMutator) + ->Apply(StreamingPingPongArgs); + +BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, InProcessCHTTP2, NoOpMutator, + NoOpMutator) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, TCP, NoOpMutator, NoOpMutator) + ->Range(0, 128 * 1024 * 1024); + +} // namespace testing +} // namespace grpc + +BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc deleted file mode 100644 index dc0e7d769ab..00000000000 --- a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc +++ /dev/null @@ -1,197 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* Benchmark gRPC end2end in various configurations */ - -#include - -#include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" -#include "src/proto/grpc/testing/echo.grpc.pb.h" -#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" -#include "test/cpp/microbenchmarks/fullstack_fixtures.h" -#include "third_party/benchmark/include/benchmark/benchmark.h" - -namespace grpc { -namespace testing { - -// force library initialization -auto& force_library_initialization = Library::get(); - -/******************************************************************************* - * BENCHMARKING KERNELS - */ - -static void* tag(intptr_t x) { return reinterpret_cast(x); } - -template -static void BM_PumpStreamClientToServer(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoRequest send_request; - EchoRequest recv_request; - if (state.range(0) > 0) { - send_request.set_message(std::string(state.range(0), 'a')); - } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - response_rw.Read(&recv_request, tag(0)); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - request_rw->Write(send_request, tag(1)); - while (true) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - if (t == tag(0)) { - response_rw.Read(&recv_request, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); - } - } - } - request_rw->WritesDone(tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); -} - -template -static void BM_PumpStreamServerToClient(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoResponse send_response; - EchoResponse recv_response; - if (state.range(0) > 0) { - send_response.set_message(std::string(state.range(0), 'a')); - } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - request_rw->Read(&recv_response, tag(0)); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - response_rw.Write(send_response, tag(1)); - while (true) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - if (t == tag(0)) { - request_rw->Read(&recv_response, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); - } - } - } - response_rw.Finish(Status::OK, tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); -} - -/******************************************************************************* - * CONFIGURATIONS - */ - -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) - ->Range(0, 128 * 1024 * 1024); - -} // namespace testing -} // namespace grpc - -BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc deleted file mode 100644 index dc0e7d769ab..00000000000 --- a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc +++ /dev/null @@ -1,197 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* Benchmark gRPC end2end in various configurations */ - -#include - -#include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" -#include "src/proto/grpc/testing/echo.grpc.pb.h" -#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" -#include "test/cpp/microbenchmarks/fullstack_fixtures.h" -#include "third_party/benchmark/include/benchmark/benchmark.h" - -namespace grpc { -namespace testing { - -// force library initialization -auto& force_library_initialization = Library::get(); - -/******************************************************************************* - * BENCHMARKING KERNELS - */ - -static void* tag(intptr_t x) { return reinterpret_cast(x); } - -template -static void BM_PumpStreamClientToServer(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoRequest send_request; - EchoRequest recv_request; - if (state.range(0) > 0) { - send_request.set_message(std::string(state.range(0), 'a')); - } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - response_rw.Read(&recv_request, tag(0)); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - request_rw->Write(send_request, tag(1)); - while (true) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - if (t == tag(0)) { - response_rw.Read(&recv_request, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); - } - } - } - request_rw->WritesDone(tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); -} - -template -static void BM_PumpStreamServerToClient(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoResponse send_response; - EchoResponse recv_response; - if (state.range(0) > 0) { - send_response.set_message(std::string(state.range(0), 'a')); - } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - request_rw->Read(&recv_response, tag(0)); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - response_rw.Write(send_response, tag(1)); - while (true) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - if (t == tag(0)) { - request_rw->Read(&recv_response, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); - } - } - } - response_rw.Finish(Status::OK, tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); -} - -/******************************************************************************* - * CONFIGURATIONS - */ - -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) - ->Range(0, 128 * 1024 * 1024); - -} // namespace testing -} // namespace grpc - -BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc deleted file mode 100644 index 5011f06368e..00000000000 --- a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc +++ /dev/null @@ -1,219 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* Benchmark gRPC end2end in various configurations */ - -#include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" -#include "src/proto/grpc/testing/echo.grpc.pb.h" -#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" -#include "test/cpp/microbenchmarks/fullstack_fixtures.h" -#include "third_party/benchmark/include/benchmark/benchmark.h" -extern "C" { -#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" -#include "src/core/ext/transport/chttp2/transport/internal.h" -#include "test/core/util/trickle_endpoint.h" -} - -namespace grpc { -namespace testing { - -static void* tag(intptr_t x) { return reinterpret_cast(x); } - -class TrickledCHTTP2 : public EndpointPairFixture { - public: - TrickledCHTTP2(Service* service, size_t megabits_per_second) - : EndpointPairFixture(service, MakeEndpoints(megabits_per_second)) {} - - void AddToLabel(std::ostream& out, benchmark::State& state) { - out << " writes/iter:" - << ((double)stats_.num_writes / (double)state.iterations()) - << " cli_transport_stalls/iter:" - << ((double) - client_stats_.streams_stalled_due_to_transport_flow_control / - (double)state.iterations()) - << " cli_stream_stalls/iter:" - << ((double)client_stats_.streams_stalled_due_to_stream_flow_control / - (double)state.iterations()) - << " svr_transport_stalls/iter:" - << ((double) - server_stats_.streams_stalled_due_to_transport_flow_control / - (double)state.iterations()) - << " svr_stream_stalls/iter:" - << ((double)server_stats_.streams_stalled_due_to_stream_flow_control / - (double)state.iterations()); - } - - void Step() { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - size_t client_backlog = - grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.client); - size_t server_backlog = - grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server); - grpc_exec_ctx_finish(&exec_ctx); - - UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_, - client_backlog); - UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_, - server_backlog); - } - - private: - grpc_passthru_endpoint_stats stats_; - struct Stats { - int streams_stalled_due_to_stream_flow_control = 0; - int streams_stalled_due_to_transport_flow_control = 0; - }; - Stats client_stats_; - Stats server_stats_; - - grpc_endpoint_pair MakeEndpoints(size_t kilobits) { - grpc_endpoint_pair p; - grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(), - &stats_); - double bytes_per_second = 125.0 * kilobits; - p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second); - p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second); - return p; - } - - void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) { - if (backlog == 0) { - if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) { - s->streams_stalled_due_to_stream_flow_control++; - } - if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) { - s->streams_stalled_due_to_transport_flow_control++; - } - } - } -}; - -// force library initialization -auto& force_library_initialization = Library::get(); - -static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) { - while (true) { - switch (fixture->cq()->AsyncNext( - t, ok, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_micros(100, GPR_TIMESPAN)))) { - case CompletionQueue::TIMEOUT: - fixture->Step(); - break; - case CompletionQueue::SHUTDOWN: - GPR_ASSERT(false); - break; - case CompletionQueue::GOT_EVENT: - return; - } - } -} - -static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture( - new TrickledCHTTP2(&service, state.range(1))); - { - EchoResponse send_response; - EchoResponse recv_response; - if (state.range(0) > 0) { - send_response.set_message(std::string(state.range(0), 'a')); - } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - TrickleCQNext(fixture.get(), &t, &ok); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - request_rw->Read(&recv_response, tag(0)); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - response_rw.Write(send_response, tag(1)); - while (true) { - TrickleCQNext(fixture.get(), &t, &ok); - if (t == tag(0)) { - request_rw->Read(&recv_response, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); - } - } - } - response_rw.Finish(Status::OK, tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - TrickleCQNext(fixture.get(), &t, &ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); -} - -/******************************************************************************* - * CONFIGURATIONS - */ - -static void TrickleArgs(benchmark::internal::Benchmark* b) { - for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { - for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) { - double expected_time = - static_cast(14 + i) / (125.0 * static_cast(j)); - if (expected_time > 0.01) continue; - b->Args({i, j}); - } - } -} - -BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs); -} -} - -BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc deleted file mode 100644 index e51d272b104..00000000000 --- a/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc +++ /dev/null @@ -1,205 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* Benchmark gRPC end2end in various configurations */ - -#include - -#include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" -#include "src/proto/grpc/testing/echo.grpc.pb.h" -#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" -#include "test/cpp/microbenchmarks/fullstack_fixtures.h" -#include "third_party/benchmark/include/benchmark/benchmark.h" - -namespace grpc { -namespace testing { - -// force library initialization -auto& force_library_initialization = Library::get(); - -/******************************************************************************* - * BENCHMARKING KERNELS - */ - -static void* tag(intptr_t x) { return reinterpret_cast(x); } - -template -static void BM_UnaryPingPong(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - EchoRequest send_request; - EchoResponse send_response; - EchoResponse recv_response; - if (state.range(0) > 0) { - send_request.set_message(std::string(state.range(0), 'a')); - } - if (state.range(1) > 0) { - send_response.set_message(std::string(state.range(1), 'a')); - } - Status recv_status; - struct ServerEnv { - ServerContext ctx; - EchoRequest recv_request; - grpc::ServerAsyncResponseWriter response_writer; - ServerEnv() : response_writer(&ctx) {} - }; - uint8_t server_env_buffer[2 * sizeof(ServerEnv)]; - ServerEnv* server_env[2] = { - reinterpret_cast(server_env_buffer), - reinterpret_cast(server_env_buffer + sizeof(ServerEnv))}; - new (server_env[0]) ServerEnv; - new (server_env[1]) ServerEnv; - service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request, - &server_env[0]->response_writer, fixture->cq(), - fixture->cq(), tag(0)); - service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request, - &server_env[1]->response_writer, fixture->cq(), - fixture->cq(), tag(1)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - recv_response.Clear(); - ClientContext cli_ctx; - ClientContextMutator cli_ctx_mut(&cli_ctx); - std::unique_ptr> response_reader( - stub->AsyncEcho(&cli_ctx, send_request, fixture->cq())); - void* t; - bool ok; - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - GPR_ASSERT(t == tag(0) || t == tag(1)); - intptr_t slot = reinterpret_cast(t); - ServerEnv* senv = server_env[slot]; - ServerContextMutator svr_ctx_mut(&senv->ctx); - senv->response_writer.Finish(send_response, Status::OK, tag(3)); - response_reader->Finish(&recv_response, &recv_status, tag(4)); - for (int i = (1 << 3) | (1 << 4); i != 0;) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int tagnum = (int)reinterpret_cast(t); - GPR_ASSERT(i & (1 << tagnum)); - i -= 1 << tagnum; - } - GPR_ASSERT(recv_status.ok()); - - senv->~ServerEnv(); - senv = new (senv) ServerEnv(); - service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer, - fixture->cq(), fixture->cq(), tag(slot)); - } - fixture->Finish(state); - fixture.reset(); - server_env[0]->~ServerEnv(); - server_env[1]->~ServerEnv(); - state.SetBytesProcessed(state.range(0) * state.iterations() + - state.range(1) * state.iterations()); -} - -/******************************************************************************* - * CONFIGURATIONS - */ - -static void SweepSizesArgs(benchmark::internal::Benchmark* b) { - b->Args({0, 0}); - for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { - b->Args({i, 0}); - b->Args({0, i}); - b->Args({i, i}); - } -} - -BENCHMARK_TEMPLATE(BM_UnaryPingPong, TCP, NoOpMutator, NoOpMutator) - ->Apply(SweepSizesArgs); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, UDS, NoOpMutator, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, SockPair, NoOpMutator, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, NoOpMutator) - ->Apply(SweepSizesArgs); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, - NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 2>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 2>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 2>, - NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 100>) - ->Args({0, 0}); - -} // namespace testing -} // namespace grpc - -BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_metadata.cc b/test/cpp/microbenchmarks/bm_metadata.cc index ee3dec2bce0..f4686908342 100644 --- a/test/cpp/microbenchmarks/bm_metadata.cc +++ b/test/cpp/microbenchmarks/bm_metadata.cc @@ -42,26 +42,25 @@ extern "C" { #include "src/core/lib/transport/transport.h" } -#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -auto& force_library_initialization = Library::get(); +static class InitializeStuff { + public: + InitializeStuff() { grpc_init(); } + ~InitializeStuff() { grpc_shutdown(); } +} initialize_stuff; static void BM_SliceFromStatic(benchmark::State& state) { - TrackCounters track_counters; while (state.KeepRunning()) { benchmark::DoNotOptimize(grpc_slice_from_static_string("abc")); } - track_counters.Finish(state); } BENCHMARK(BM_SliceFromStatic); static void BM_SliceFromCopied(benchmark::State& state) { - TrackCounters track_counters; while (state.KeepRunning()) { grpc_slice_unref(grpc_slice_from_copied_string("abc")); } - track_counters.Finish(state); } BENCHMARK(BM_SliceFromCopied); @@ -79,47 +78,38 @@ static void BM_SliceFromStreamOwnedBuffer(benchmark::State& state) { BENCHMARK(BM_SliceFromStreamOwnedBuffer); static void BM_SliceIntern(benchmark::State& state) { - TrackCounters track_counters; gpr_slice slice = grpc_slice_from_static_string("abc"); while (state.KeepRunning()) { grpc_slice_unref(grpc_slice_intern(slice)); } - track_counters.Finish(state); } BENCHMARK(BM_SliceIntern); static void BM_SliceReIntern(benchmark::State& state) { - TrackCounters track_counters; gpr_slice slice = grpc_slice_intern(grpc_slice_from_static_string("abc")); while (state.KeepRunning()) { grpc_slice_unref(grpc_slice_intern(slice)); } grpc_slice_unref(slice); - track_counters.Finish(state); } BENCHMARK(BM_SliceReIntern); static void BM_SliceInternStaticMetadata(benchmark::State& state) { - TrackCounters track_counters; while (state.KeepRunning()) { grpc_slice_intern(GRPC_MDSTR_GZIP); } - track_counters.Finish(state); } BENCHMARK(BM_SliceInternStaticMetadata); static void BM_SliceInternEqualToStaticMetadata(benchmark::State& state) { - TrackCounters track_counters; gpr_slice slice = grpc_slice_from_static_string("gzip"); while (state.KeepRunning()) { grpc_slice_intern(slice); } - track_counters.Finish(state); } BENCHMARK(BM_SliceInternEqualToStaticMetadata); static void BM_MetadataFromNonInternedSlices(benchmark::State& state) { - TrackCounters track_counters; gpr_slice k = grpc_slice_from_static_string("key"); gpr_slice v = grpc_slice_from_static_string("value"); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -127,12 +117,10 @@ static void BM_MetadataFromNonInternedSlices(benchmark::State& state) { GRPC_MDELEM_UNREF(&exec_ctx, grpc_mdelem_create(&exec_ctx, k, v, NULL)); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_MetadataFromNonInternedSlices); static void BM_MetadataFromInternedSlices(benchmark::State& state) { - TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value")); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -142,13 +130,11 @@ static void BM_MetadataFromInternedSlices(benchmark::State& state) { grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); grpc_slice_unref(v); - track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedSlices); static void BM_MetadataFromInternedSlicesAlreadyInIndex( benchmark::State& state) { - TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value")); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -160,12 +146,10 @@ static void BM_MetadataFromInternedSlicesAlreadyInIndex( grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); grpc_slice_unref(v); - track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedSlicesAlreadyInIndex); static void BM_MetadataFromInternedKey(benchmark::State& state) { - TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_from_static_string("value"); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -174,13 +158,11 @@ static void BM_MetadataFromInternedKey(benchmark::State& state) { } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); - track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedKey); static void BM_MetadataFromNonInternedSlicesWithBackingStore( benchmark::State& state) { - TrackCounters track_counters; gpr_slice k = grpc_slice_from_static_string("key"); gpr_slice v = grpc_slice_from_static_string("value"); char backing_store[sizeof(grpc_mdelem_data)]; @@ -192,13 +174,11 @@ static void BM_MetadataFromNonInternedSlicesWithBackingStore( reinterpret_cast(backing_store))); } grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_MetadataFromNonInternedSlicesWithBackingStore); static void BM_MetadataFromInternedSlicesWithBackingStore( benchmark::State& state) { - TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value")); char backing_store[sizeof(grpc_mdelem_data)]; @@ -212,13 +192,11 @@ static void BM_MetadataFromInternedSlicesWithBackingStore( grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); grpc_slice_unref(v); - track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedSlicesWithBackingStore); static void BM_MetadataFromInternedKeyWithBackingStore( benchmark::State& state) { - TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_from_static_string("value"); char backing_store[sizeof(grpc_mdelem_data)]; @@ -231,12 +209,10 @@ static void BM_MetadataFromInternedKeyWithBackingStore( } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); - track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedKeyWithBackingStore); static void BM_MetadataFromStaticMetadataStrings(benchmark::State& state) { - TrackCounters track_counters; gpr_slice k = GRPC_MDSTR_STATUS; gpr_slice v = GRPC_MDSTR_200; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -245,13 +221,11 @@ static void BM_MetadataFromStaticMetadataStrings(benchmark::State& state) { } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); - track_counters.Finish(state); } BENCHMARK(BM_MetadataFromStaticMetadataStrings); static void BM_MetadataFromStaticMetadataStringsNotIndexed( benchmark::State& state) { - TrackCounters track_counters; gpr_slice k = GRPC_MDSTR_STATUS; gpr_slice v = GRPC_MDSTR_GZIP; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -260,12 +234,10 @@ static void BM_MetadataFromStaticMetadataStringsNotIndexed( } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); - track_counters.Finish(state); } BENCHMARK(BM_MetadataFromStaticMetadataStringsNotIndexed); static void BM_MetadataRefUnrefExternal(benchmark::State& state) { - TrackCounters track_counters; char backing_store[sizeof(grpc_mdelem_data)]; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_mdelem el = @@ -277,12 +249,10 @@ static void BM_MetadataRefUnrefExternal(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefExternal); static void BM_MetadataRefUnrefInterned(benchmark::State& state) { - TrackCounters track_counters; char backing_store[sizeof(grpc_mdelem_data)]; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); @@ -296,12 +266,10 @@ static void BM_MetadataRefUnrefInterned(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefInterned); static void BM_MetadataRefUnrefAllocated(benchmark::State& state) { - TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_mdelem el = grpc_mdelem_create(&exec_ctx, grpc_slice_from_static_string("a"), @@ -311,12 +279,10 @@ static void BM_MetadataRefUnrefAllocated(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefAllocated); static void BM_MetadataRefUnrefStatic(benchmark::State& state) { - TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_mdelem el = grpc_mdelem_create(&exec_ctx, GRPC_MDSTR_STATUS, GRPC_MDSTR_200, NULL); @@ -325,7 +291,6 @@ static void BM_MetadataRefUnrefStatic(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); - track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefStatic); diff --git a/test/cpp/microbenchmarks/fullstack_context_mutators.h b/test/cpp/microbenchmarks/fullstack_context_mutators.h deleted file mode 100644 index 676f9aa1cc6..00000000000 --- a/test/cpp/microbenchmarks/fullstack_context_mutators.h +++ /dev/null @@ -1,158 +0,0 @@ -/* - * - * Copyright 2017, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef TEST_CPP_MICROBENCHMARKS_FULLSTACK_CONTEXT_MUTATORS_H -#define TEST_CPP_MICROBENCHMARKS_FULLSTACK_CONTEXT_MUTATORS_H - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "test/cpp/microbenchmarks/helpers.h" - -namespace grpc { -namespace testing { - -/******************************************************************************* - * CONTEXT MUTATORS - */ - -static const int kPregenerateKeyCount = 100000; - -template -auto MakeVector(size_t length, F f) -> std::vector { - std::vector out; - out.reserve(length); - for (size_t i = 0; i < length; i++) { - out.push_back(f()); - } - return out; -} - -class NoOpMutator { - public: - template - NoOpMutator(ContextType* context) {} -}; - -template -class RandomBinaryMetadata { - public: - static const grpc::string& Key() { return kKey; } - - static const grpc::string& Value() { - return kValues[rand() % kValues.size()]; - } - - private: - static const grpc::string kKey; - static const std::vector kValues; - - static grpc::string GenerateOneString() { - grpc::string s; - s.reserve(length + 1); - for (int i = 0; i < length; i++) { - s += (char)rand(); - } - return s; - } -}; - -template -class RandomAsciiMetadata { - public: - static const grpc::string& Key() { return kKey; } - - static const grpc::string& Value() { - return kValues[rand() % kValues.size()]; - } - - private: - static const grpc::string kKey; - static const std::vector kValues; - - static grpc::string GenerateOneString() { - grpc::string s; - s.reserve(length + 1); - for (int i = 0; i < length; i++) { - s += (char)(rand() % 26 + 'a'); - } - return s; - } -}; - -template -class Client_AddMetadata : public NoOpMutator { - public: - Client_AddMetadata(ClientContext* context) : NoOpMutator(context) { - for (int i = 0; i < kNumKeys; i++) { - context->AddMetadata(Generator::Key(), Generator::Value()); - } - } -}; - -template -class Server_AddInitialMetadata : public NoOpMutator { - public: - Server_AddInitialMetadata(ServerContext* context) : NoOpMutator(context) { - for (int i = 0; i < kNumKeys; i++) { - context->AddInitialMetadata(Generator::Key(), Generator::Value()); - } - } -}; - -// static initialization - -template -const grpc::string RandomBinaryMetadata::kKey = "foo-bin"; - -template -const std::vector RandomBinaryMetadata::kValues = - MakeVector(kPregenerateKeyCount, GenerateOneString); - -template -const grpc::string RandomAsciiMetadata::kKey = "foo"; - -template -const std::vector RandomAsciiMetadata::kValues = - MakeVector(kPregenerateKeyCount, GenerateOneString); - -} // namespace testing -} // namespace grpc - -#endif diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h deleted file mode 100644 index dc297010599..00000000000 --- a/test/cpp/microbenchmarks/fullstack_fixtures.h +++ /dev/null @@ -1,244 +0,0 @@ -/* - * - * Copyright 2017, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H -#define TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H - -#include -#include -#include -#include -#include -#include -#include - -extern "C" { -#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/iomgr/endpoint.h" -#include "src/core/lib/iomgr/endpoint_pair.h" -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/iomgr/tcp_posix.h" -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/surface/completion_queue.h" -#include "src/core/lib/surface/server.h" -#include "test/core/util/passthru_endpoint.h" -#include "test/core/util/port.h" -} - -#include "test/cpp/microbenchmarks/helpers.h" - -namespace grpc { -namespace testing { - -static void ApplyCommonServerBuilderConfig(ServerBuilder* b) { - b->SetMaxReceiveMessageSize(INT_MAX); - b->SetMaxSendMessageSize(INT_MAX); -} - -static void ApplyCommonChannelArguments(ChannelArguments* c) { - c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX); - c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX); -} - -class BaseFixture : public TrackCounters {}; - -class FullstackFixture : public BaseFixture { - public: - FullstackFixture(Service* service, const grpc::string& address) { - ServerBuilder b; - b.AddListeningPort(address, InsecureServerCredentials()); - cq_ = b.AddCompletionQueue(true); - b.RegisterService(service); - ApplyCommonServerBuilderConfig(&b); - server_ = b.BuildAndStart(); - ChannelArguments args; - ApplyCommonChannelArguments(&args); - channel_ = CreateCustomChannel(address, InsecureChannelCredentials(), args); - } - - virtual ~FullstackFixture() { - server_->Shutdown(); - cq_->Shutdown(); - void* tag; - bool ok; - while (cq_->Next(&tag, &ok)) { - } - } - - ServerCompletionQueue* cq() { return cq_.get(); } - std::shared_ptr channel() { return channel_; } - - private: - std::unique_ptr server_; - std::unique_ptr cq_; - std::shared_ptr channel_; -}; - -class TCP : public FullstackFixture { - public: - TCP(Service* service) : FullstackFixture(service, MakeAddress()) {} - - private: - static grpc::string MakeAddress() { - int port = grpc_pick_unused_port_or_die(); - std::stringstream addr; - addr << "localhost:" << port; - return addr.str(); - } -}; - -class UDS : public FullstackFixture { - public: - UDS(Service* service) : FullstackFixture(service, MakeAddress()) {} - - private: - static grpc::string MakeAddress() { - int port = grpc_pick_unused_port_or_die(); // just for a unique id - not a - // real port - std::stringstream addr; - addr << "unix:/tmp/bm_fullstack." << port; - return addr.str(); - } -}; - -class EndpointPairFixture : public BaseFixture { - public: - EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) - : endpoint_pair_(endpoints) { - ServerBuilder b; - cq_ = b.AddCompletionQueue(true); - b.RegisterService(service); - ApplyCommonServerBuilderConfig(&b); - server_ = b.BuildAndStart(); - - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - - /* add server endpoint to server_ - * */ - { - const grpc_channel_args* server_args = - grpc_server_get_channel_args(server_->c_server()); - server_transport_ = grpc_create_chttp2_transport( - &exec_ctx, server_args, endpoints.server, 0 /* is_client */); - - grpc_pollset** pollsets; - size_t num_pollsets = 0; - grpc_server_get_pollsets(server_->c_server(), &pollsets, &num_pollsets); - - for (size_t i = 0; i < num_pollsets; i++) { - grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]); - } - - grpc_server_setup_transport(&exec_ctx, server_->c_server(), - server_transport_, NULL, server_args); - grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL); - } - - /* create channel */ - { - ChannelArguments args; - args.SetString(GRPC_ARG_DEFAULT_AUTHORITY, "test.authority"); - ApplyCommonChannelArguments(&args); - - grpc_channel_args c_args = args.c_channel_args(); - client_transport_ = - grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1); - GPR_ASSERT(client_transport_); - grpc_channel* channel = - grpc_channel_create(&exec_ctx, "target", &c_args, - GRPC_CLIENT_DIRECT_CHANNEL, client_transport_); - grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL); - - channel_ = CreateChannelInternal("", channel); - } - - grpc_exec_ctx_finish(&exec_ctx); - } - - virtual ~EndpointPairFixture() { - server_->Shutdown(); - cq_->Shutdown(); - void* tag; - bool ok; - while (cq_->Next(&tag, &ok)) { - } - } - - ServerCompletionQueue* cq() { return cq_.get(); } - std::shared_ptr channel() { return channel_; } - - protected: - grpc_endpoint_pair endpoint_pair_; - grpc_transport* client_transport_; - grpc_transport* server_transport_; - - private: - std::unique_ptr server_; - std::unique_ptr cq_; - std::shared_ptr channel_; -}; - -class SockPair : public EndpointPairFixture { - public: - SockPair(Service* service) - : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair( - "test", Library::get().rq(), 8192)) {} -}; - -class InProcessCHTTP2 : public EndpointPairFixture { - public: - InProcessCHTTP2(Service* service) - : EndpointPairFixture(service, MakeEndpoints()) {} - - void AddToLabel(std::ostream& out, benchmark::State& state) { - EndpointPairFixture::AddToLabel(out, state); - out << " writes/iter:" - << ((double)stats_.num_writes / (double)state.iterations()); - } - - private: - grpc_passthru_endpoint_stats stats_; - - grpc_endpoint_pair MakeEndpoints() { - grpc_endpoint_pair p; - grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(), - &stats_); - return p; - } -}; - -} // namespace testing -} // namespace grpc - -#endif diff --git a/test/cpp/microbenchmarks/helpers.cc b/test/cpp/microbenchmarks/helpers.cc deleted file mode 100644 index 947e81ffd83..00000000000 --- a/test/cpp/microbenchmarks/helpers.cc +++ /dev/null @@ -1,65 +0,0 @@ -/* - * - * Copyright 2017, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "test/cpp/microbenchmarks/helpers.h" - -void TrackCounters::Finish(benchmark::State &state) { - std::ostringstream out; - AddToLabel(out, state); - auto label = out.str(); - if (label.length() && label[0] == ' ') { - label = label.substr(1); - } - state.SetLabel(label); -} - -void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) { -#ifdef GPR_LOW_LEVEL_COUNTERS - out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - - mu_locks_at_start_) / - (double)state.iterations()) - << " atm_cas/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) - - atm_cas_at_start_) / - (double)state.iterations()) - << " atm_add/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - - atm_add_at_start_) / - (double)state.iterations()); -#endif - grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); - out << " allocs/iter:" - << ((double)(counters_at_end.total_allocs_absolute - - counters_at_start_.total_allocs_absolute) / - (double)state.iterations()); -} diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h deleted file mode 100644 index 42a8fbaf0bf..00000000000 --- a/test/cpp/microbenchmarks/helpers.h +++ /dev/null @@ -1,91 +0,0 @@ -/* - * - * Copyright 2017, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef TEST_CPP_MICROBENCHMARKS_COUNTERS_H -#define TEST_CPP_MICROBENCHMARKS_COUNTERS_H - -#include - -extern "C" { -#include -#include "test/core/util/memory_counters.h" -} - -#include -#include "third_party/benchmark/include/benchmark/benchmark.h" - -class Library { - public: - static Library& get() { - static Library lib; - return lib; - } - - grpc_resource_quota* rq() { return rq_; } - - private: - Library() { - grpc_memory_counters_init(); - init_lib_.init(); - rq_ = grpc_resource_quota_create("bm"); - } - - ~Library() { init_lib_.shutdown(); } - - grpc::internal::GrpcLibrary init_lib_; - grpc_resource_quota* rq_; -}; - -#ifdef GPR_LOW_LEVEL_COUNTERS -extern "C" gpr_atm gpr_mu_locks; -extern "C" gpr_atm gpr_counter_atm_cas; -extern "C" gpr_atm gpr_counter_atm_add; -#endif - -class TrackCounters { - public: - virtual void Finish(benchmark::State& state); - virtual void AddToLabel(std::ostream& out, benchmark::State& state); - - private: -#ifdef GPR_LOW_LEVEL_COUNTERS - const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks); - const size_t atm_cas_at_start_ = - gpr_atm_no_barrier_load(&gpr_counter_atm_cas); - const size_t atm_add_at_start_ = - gpr_atm_no_barrier_load(&gpr_counter_atm_add); -#endif - grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot(); -}; - -#endif diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 8a31a6e0552..0f63612ed27 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2420,7 +2420,6 @@ "grpc", "grpc++", "grpc++_test_util", - "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2441,7 +2440,6 @@ "grpc", "grpc++", "grpc++_test_util", - "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2462,7 +2460,6 @@ "grpc", "grpc++", "grpc++_test_util", - "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2483,7 +2480,6 @@ "grpc", "grpc++", "grpc++_test_util", - "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2504,7 +2500,6 @@ "grpc", "grpc++", "grpc++_test_util", - "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2525,15 +2520,14 @@ "grpc", "grpc++", "grpc++_test_util", - "grpc_benchmark", "grpc_test_util" ], "headers": [], "is_filegroup": false, "language": "c++", - "name": "bm_fullstack_streaming_ping_pong", + "name": "bm_fullstack", "src": [ - "test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc" + "test/cpp/microbenchmarks/bm_fullstack.cc" ], "third_party": false, "type": "target" @@ -2544,70 +2538,6 @@ "gpr", "gpr_test_util", "grpc", - "grpc++", - "grpc++_test_util", - "grpc_benchmark", - "grpc_test_util" - ], - "headers": [], - "is_filegroup": false, - "language": "c++", - "name": "bm_fullstack_streaming_pump", - "src": [ - "test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ - "benchmark", - "gpr", - "gpr_test_util", - "grpc", - "grpc++", - "grpc++_test_util", - "grpc_benchmark", - "grpc_test_util" - ], - "headers": [], - "is_filegroup": false, - "language": "c++", - "name": "bm_fullstack_trickle", - "src": [ - "test/cpp/microbenchmarks/bm_fullstack_trickle.cc" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ - "benchmark", - "gpr", - "gpr_test_util", - "grpc", - "grpc++", - "grpc++_test_util", - "grpc_benchmark", - "grpc_test_util" - ], - "headers": [], - "is_filegroup": false, - "language": "c++", - "name": "bm_fullstack_unary_ping_pong", - "src": [ - "test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ - "benchmark", - "gpr", - "gpr_test_util", - "grpc", - "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -5811,30 +5741,6 @@ "third_party": false, "type": "lib" }, - { - "deps": [ - "benchmark", - "grpc", - "grpc++", - "grpc_test_util" - ], - "headers": [ - "test/cpp/microbenchmarks/fullstack_context_mutators.h", - "test/cpp/microbenchmarks/fullstack_fixtures.h", - "test/cpp/microbenchmarks/helpers.h" - ], - "is_filegroup": false, - "language": "c++", - "name": "grpc_benchmark", - "src": [ - "test/cpp/microbenchmarks/fullstack_context_mutators.h", - "test/cpp/microbenchmarks/fullstack_fixtures.h", - "test/cpp/microbenchmarks/helpers.cc", - "test/cpp/microbenchmarks/helpers.h" - ], - "third_party": false, - "type": "lib" - }, { "deps": [ "grpc++", @@ -8040,7 +7946,7 @@ "headers": [ "test/core/end2end/cq_verifier.h", "test/core/end2end/fake_resolver.h", - "test/core/end2end/fixtures/http_proxy_fixture.h", + "test/core/end2end/fixtures/http_proxy.h", "test/core/end2end/fixtures/proxy.h", "test/core/iomgr/endpoint_tests.h", "test/core/util/debugger_macros.h", @@ -8062,8 +7968,8 @@ "test/core/end2end/cq_verifier.h", "test/core/end2end/fake_resolver.c", "test/core/end2end/fake_resolver.h", - "test/core/end2end/fixtures/http_proxy_fixture.c", - "test/core/end2end/fixtures/http_proxy_fixture.h", + "test/core/end2end/fixtures/http_proxy.c", + "test/core/end2end/fixtures/http_proxy.h", "test/core/end2end/fixtures/proxy.c", "test/core/end2end/fixtures/proxy.h", "test/core/iomgr/endpoint_tests.c", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 402dabc554d..46c83384019 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2712,88 +2712,7 @@ "flaky": false, "gtest": false, "language": "c++", - "name": "bm_fullstack_streaming_ping_pong", - "platforms": [ - "linux", - "mac", - "posix" - ], - "timeout_seconds": 1200 - }, - { - "args": [ - "--benchmark_min_time=0" - ], - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "excluded_poll_engines": [ - "poll", - "poll-cv" - ], - "flaky": false, - "gtest": false, - "language": "c++", - "name": "bm_fullstack_streaming_pump", - "platforms": [ - "linux", - "mac", - "posix" - ], - "timeout_seconds": 1200 - }, - { - "args": [ - "--benchmark_min_time=0" - ], - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "excluded_poll_engines": [ - "poll", - "poll-cv" - ], - "flaky": false, - "gtest": false, - "language": "c++", - "name": "bm_fullstack_trickle", - "platforms": [ - "linux", - "mac", - "posix" - ], - "timeout_seconds": 1200 - }, - { - "args": [ - "--benchmark_min_time=0" - ], - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "excluded_poll_engines": [ - "poll", - "poll-cv" - ], - "flaky": false, - "gtest": false, - "language": "c++", - "name": "bm_fullstack_unary_ping_pong", + "name": "bm_fullstack", "platforms": [ "linux", "mac", diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 4fb1d5fc7c2..3a9461ecd3b 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -199,17 +199,13 @@ argp.add_argument('-c', '--collect', default=sorted(collectors.keys()), help='Which collectors should be run against each benchmark') argp.add_argument('-b', '--benchmarks', - default=['bm_fullstack_unary_ping_pong', - 'bm_fullstack_streaming_ping_pong', - 'bm_fullstack_streaming_pump', + default=['bm_fullstack', 'bm_closure', 'bm_cq', 'bm_call_create', 'bm_error', 'bm_chttp2_hpack', - 'bm_metadata', - 'bm_fullstack_trickle', - ], + 'bm_metadata'], nargs='+', type=str, help='Which microbenchmarks should be run') diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index e7c9fb71f33..d08ceb68287 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -182,7 +182,7 @@ - + @@ -317,7 +317,7 @@ - + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 3d36948aaef..3beaa80994f 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -22,7 +22,7 @@ test\core\end2end - + test\core\end2end\fixtures @@ -518,7 +518,7 @@ test\core\end2end - + test\core\end2end\fixtures diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj index af13acef455..5a58cae83ff 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj @@ -149,7 +149,7 @@ - + @@ -168,7 +168,7 @@ - + diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters index 4da043ea908..88c875cb011 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters @@ -7,7 +7,7 @@ test\core\end2end - + test\core\end2end\fixtures @@ -54,7 +54,7 @@ test\core\end2end - + test\core\end2end\fixtures diff --git a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj deleted file mode 100644 index 58586f0cb8d..00000000000 --- a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj +++ /dev/null @@ -1,206 +0,0 @@ - - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {31FCED31-7D88-BE3D-2D61-0840F08E0850} - true - $(SolutionDir)IntDir\$(MSBuildProjectName)\ - - - - v100 - - - v110 - - - v120 - - - v140 - - - Application - true - Unicode - - - Application - false - true - Unicode - - - - - - - - - - - - - - - - grpc_benchmark - static - Debug - static - Debug - - - grpc_benchmark - static - Release - static - Release - - - - NotUsing - Level3 - Disabled - WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) - true - MultiThreadedDebug - true - None - false - - - Windows - true - false - - - - - - NotUsing - Level3 - Disabled - WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) - true - MultiThreadedDebug - true - None - false - - - Windows - true - false - - - - - - NotUsing - Level3 - MaxSpeed - WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) - true - true - true - MultiThreaded - true - None - false - - - Windows - true - false - true - true - - - - - - NotUsing - Level3 - MaxSpeed - WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) - true - true - true - MultiThreaded - true - None - false - - - Windows - true - false - true - true - - - - - - - - - - - - - - - {07978586-E47C-8709-A63E-895FBF3C3C7D} - - - {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} - - - {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} - - - {29D16885-7228-4C31-81ED-5F9187C7F2A9} - - - - - - - - - - - - - - - This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - - - - - - - diff --git a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters deleted file mode 100644 index 8e865bcc1bb..00000000000 --- a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters +++ /dev/null @@ -1,32 +0,0 @@ - - - - - test\cpp\microbenchmarks - - - - - test\cpp\microbenchmarks - - - test\cpp\microbenchmarks - - - test\cpp\microbenchmarks - - - - - - {46d1162d-13b8-d144-8b76-77a6d982a9f1} - - - {1d2b47d7-8fc3-a5b6-cc85-47e31600e9d7} - - - {2a1ac913-6c7b-fbd2-6e8f-1215e92b4af8} - - - - From b597dcf53a5057425ba24a7c8dbed2f65f8c31f4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Mar 2017 07:02:11 -0800 Subject: [PATCH 077/188] Race fixes --- src/core/lib/surface/call.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 2f3ddd6bcb9..eafeef0a207 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -161,9 +161,9 @@ struct grpc_call { bool received_initial_metadata; bool receiving_message; bool requested_final_op; - bool received_final_op; - gpr_atm num_ops_sent; - + gpr_atm any_ops_sent_atm; + gpr_atm received_final_op_atm; + /* have we received initial metadata */ bool has_initial_md_been_received; @@ -458,7 +458,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, for (i = 0; i < STATUS_SOURCE_COUNT; i++) { GRPC_ERROR_UNREF( - unpack_received_status(gpr_atm_no_barrier_load(&c->status[i])).error); + unpack_received_status(gpr_atm_acq_load(&c->status[i])).error); } grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), &c->final_info, c); @@ -490,7 +490,7 @@ void grpc_call_destroy(grpc_call *c) { GPR_ASSERT(!c->destroy_called); c->destroy_called = 1; - cancel = gpr_atm_no_barrier_load(&c->num_ops_sent) && !c->received_final_op; + cancel = gpr_atm_acq_load(&c->any_ops_sent_atm) && !gpr_atm_acq_load(&c->received_final_op_atm); if (cancel) { cancel_with_error(&exec_ctx, c, STATUS_FROM_API_OVERRIDE, GRPC_ERROR_CANCELLED); @@ -1048,7 +1048,7 @@ static void finish_batch_completion(grpc_exec_ctx *exec_ctx, void *user_data, } static grpc_error *consolidate_batch_errors(batch_control *bctl) { - size_t n = (size_t)gpr_atm_no_barrier_load(&bctl->num_errors); + size_t n = (size_t)gpr_atm_acq_load(&bctl->num_errors); if (n == 0) { return GRPC_ERROR_NONE; } else if (n == 1) { @@ -1093,7 +1093,7 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; recv_trailing_filter(exec_ctx, call, md); - call->received_final_op = true; + gpr_atm_rel_store(&call->received_final_op_atm, 1); /* propagate cancellation to any interested children */ if (gpr_atm_acq_load(&call->has_children)) { gpr_mu_lock(&call->child_list_mu); @@ -1286,7 +1286,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, static void add_batch_error(grpc_exec_ctx *exec_ctx, batch_control *bctl, grpc_error *error, bool has_cancelled) { if (error == GRPC_ERROR_NONE) return; - int idx = (int)gpr_atm_no_barrier_fetch_add(&bctl->num_errors, 1); + int idx = (int)gpr_atm_full_fetch_add(&bctl->num_errors, 1); if (idx == 0 && !has_cancelled) { cancel_with_error(exec_ctx, bctl->call, STATUS_FROM_CORE, GRPC_ERROR_REF(error)); @@ -1664,7 +1664,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, grpc_closure_init(&bctl->finish_batch, finish_batch, bctl, grpc_schedule_on_exec_ctx); stream_op->on_complete = &bctl->finish_batch; - gpr_atm_no_barrier_fetch_add(&call->num_ops_sent, 1); + gpr_atm_rel_store(&call->any_ops_sent_atm, 1); execute_op(exec_ctx, call, stream_op); From 71c286466acffef07efbd83cf5424b0cfcf2aad8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Mar 2017 08:03:27 -0800 Subject: [PATCH 078/188] Dont use memory counters unless counters mode is on --- test/cpp/microbenchmarks/helpers.cc | 8 ++++---- test/cpp/microbenchmarks/helpers.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/cpp/microbenchmarks/helpers.cc b/test/cpp/microbenchmarks/helpers.cc index 947e81ffd83..d277c5984cd 100644 --- a/test/cpp/microbenchmarks/helpers.cc +++ b/test/cpp/microbenchmarks/helpers.cc @@ -45,6 +45,7 @@ void TrackCounters::Finish(benchmark::State &state) { void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) { #ifdef GPR_LOW_LEVEL_COUNTERS + grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - mu_locks_at_start_) / (double)state.iterations()) @@ -55,11 +56,10 @@ void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) { << " atm_add/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - atm_add_at_start_) / - (double)state.iterations()); -#endif - grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); - out << " allocs/iter:" + (double)state.iterations()) + << " allocs/iter:" << ((double)(counters_at_end.total_allocs_absolute - counters_at_start_.total_allocs_absolute) / (double)state.iterations()); +#endif } diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h index 42a8fbaf0bf..2829a46e5a5 100644 --- a/test/cpp/microbenchmarks/helpers.h +++ b/test/cpp/microbenchmarks/helpers.h @@ -84,8 +84,8 @@ class TrackCounters { gpr_atm_no_barrier_load(&gpr_counter_atm_cas); const size_t atm_add_at_start_ = gpr_atm_no_barrier_load(&gpr_counter_atm_add); -#endif grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot(); +#endif }; #endif From d720d5dd0aeaa96e2ee3fceef9dda17d33622077 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Mar 2017 08:10:17 -0800 Subject: [PATCH 079/188] Track zalloc times in microbenchmark This forms our speed of light for call creation: at some point call creation is dominated by zeroing memory and all we can do is reduce the size of the call... let's track where that point is. --- test/cpp/microbenchmarks/bm_call_create.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index f25bcd2df82..5810ebddebc 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -60,6 +60,16 @@ extern "C" { auto &force_library_initialization = Library::get(); +void BM_Zalloc(benchmark::State& state) { + // speed of light for call creation is zalloc, so benchmark a few interesting + // sizes + size_t sz = state.range(0); + while (state.KeepRunning()) { + gpr_free(gpr_zalloc(sz)); + } +} +BENCHMARK(BM_Zalloc)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024)->Arg(1536)->Arg(2048)->Arg(3072)->Arg(4096)->Arg(5120)->Arg(6144)->Arg(7168); + class BaseChannelFixture { public: BaseChannelFixture(grpc_channel *channel) : channel_(channel) {} From e667aaf9f97d37d331f4bb6c0a761061409c607a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Mar 2017 08:12:21 -0800 Subject: [PATCH 080/188] clang-format --- test/cpp/microbenchmarks/bm_call_create.cc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index 5810ebddebc..d763e93e78d 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -60,7 +60,7 @@ extern "C" { auto &force_library_initialization = Library::get(); -void BM_Zalloc(benchmark::State& state) { +void BM_Zalloc(benchmark::State &state) { // speed of light for call creation is zalloc, so benchmark a few interesting // sizes size_t sz = state.range(0); @@ -68,7 +68,19 @@ void BM_Zalloc(benchmark::State& state) { gpr_free(gpr_zalloc(sz)); } } -BENCHMARK(BM_Zalloc)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024)->Arg(1536)->Arg(2048)->Arg(3072)->Arg(4096)->Arg(5120)->Arg(6144)->Arg(7168); +BENCHMARK(BM_Zalloc) + ->Arg(64) + ->Arg(128) + ->Arg(256) + ->Arg(512) + ->Arg(1024) + ->Arg(1536) + ->Arg(2048) + ->Arg(3072) + ->Arg(4096) + ->Arg(5120) + ->Arg(6144) + ->Arg(7168); class BaseChannelFixture { public: From ad14bcec61817e8c328dbca0d2ace03c144b5705 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Mar 2017 08:32:01 -0800 Subject: [PATCH 081/188] Fix mac build --- CMakeLists.txt | 2 ++ Makefile | 6 +++--- build.yaml | 2 ++ tools/run_tests/generated/sources_and_headers.json | 2 ++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 24b84fd15a8..d3a76a29d76 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7967,7 +7967,9 @@ target_link_libraries(bm_metadata ${_gRPC_ALLTARGETS_LIBRARIES} grpc_benchmark benchmark + grpc++_test_util grpc_test_util + grpc++ grpc gpr_test_util gpr diff --git a/Makefile b/Makefile index a49fa928aac..3cffa73224b 100644 --- a/Makefile +++ b/Makefile @@ -13002,16 +13002,16 @@ $(BINDIR)/$(CONFIG)/bm_metadata: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_metadata: $(PROTOBUF_DEP) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_metadata: $(PROTOBUF_DEP) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_metadata + $(Q) $(LDXX) $(LDFLAGS) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_metadata endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_metadata.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_metadata.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_metadata: $(BM_METADATA_OBJS:.o=.dep) diff --git a/build.yaml b/build.yaml index 106e876a516..dc69998cccb 100644 --- a/build.yaml +++ b/build.yaml @@ -3243,7 +3243,9 @@ targets: deps: - grpc_benchmark - benchmark + - grpc++_test_util - grpc_test_util + - grpc++ - grpc - gpr_test_util - gpr diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 8a31a6e0552..22cf618c229 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2607,6 +2607,8 @@ "gpr", "gpr_test_util", "grpc", + "grpc++", + "grpc++_test_util", "grpc_benchmark", "grpc_test_util" ], From c945c12b47d2a30f12f1108fccbe6272a08da5e6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Mar 2017 08:35:14 -0800 Subject: [PATCH 082/188] Revert "Revert "Split bm_fullstack into pieces, reuse infrastructure across other microbenchmarks"" This reverts commit 0cb7fef40669789c8720427c70b892a41e27bbdf. --- CMakeLists.txt | 191 ++- Makefile | 264 +++- build.yaml | 101 +- include/grpc/impl/codegen/sync.h | 8 + test/core/end2end/BUILD | 4 +- test/core/end2end/fixtures/h2_http_proxy.c | 2 +- .../{http_proxy.c => http_proxy_fixture.c} | 2 +- .../{http_proxy.h => http_proxy_fixture.h} | 0 test/cpp/microbenchmarks/bm_call_create.cc | 10 +- test/cpp/microbenchmarks/bm_chttp2_hpack.cc | 14 +- test/cpp/microbenchmarks/bm_closure.cc | 114 +- test/cpp/microbenchmarks/bm_cq.cc | 23 +- test/cpp/microbenchmarks/bm_error.cc | 28 + test/cpp/microbenchmarks/bm_fullstack.cc | 1079 ----------------- .../bm_fullstack_streaming_ping_pong.cc | 197 +++ .../bm_fullstack_streaming_pump.cc | 197 +++ .../microbenchmarks/bm_fullstack_trickle.cc | 219 ++++ .../bm_fullstack_unary_ping_pong.cc | 205 ++++ test/cpp/microbenchmarks/bm_metadata.cc | 45 +- .../fullstack_context_mutators.h | 158 +++ test/cpp/microbenchmarks/fullstack_fixtures.h | 244 ++++ test/cpp/microbenchmarks/helpers.cc | 65 + test/cpp/microbenchmarks/helpers.h | 91 ++ .../generated/sources_and_headers.json | 104 +- tools/run_tests/generated/tests.json | 83 +- tools/run_tests/run_microbenchmark.py | 8 +- .../grpc_test_util/grpc_test_util.vcxproj | 4 +- .../grpc_test_util.vcxproj.filters | 4 +- .../grpc_test_util_unsecure.vcxproj | 4 +- .../grpc_test_util_unsecure.vcxproj.filters | 4 +- .../grpc_benchmark/grpc_benchmark.vcxproj | 206 ++++ .../grpc_benchmark.vcxproj.filters | 32 + 32 files changed, 2476 insertions(+), 1234 deletions(-) rename test/core/end2end/fixtures/{http_proxy.c => http_proxy_fixture.c} (99%) rename test/core/end2end/fixtures/{http_proxy.h => http_proxy_fixture.h} (100%) delete mode 100644 test/cpp/microbenchmarks/bm_fullstack.cc create mode 100644 test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc create mode 100644 test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc create mode 100644 test/cpp/microbenchmarks/bm_fullstack_trickle.cc create mode 100644 test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc create mode 100644 test/cpp/microbenchmarks/fullstack_context_mutators.h create mode 100644 test/cpp/microbenchmarks/fullstack_fixtures.h create mode 100644 test/cpp/microbenchmarks/helpers.cc create mode 100644 test/cpp/microbenchmarks/helpers.h create mode 100644 vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj create mode 100644 vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index 607de349cb1..24b84fd15a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -588,7 +588,16 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_error) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) -add_dependencies(buildtests_cxx bm_fullstack) +add_dependencies(buildtests_cxx bm_fullstack_streaming_ping_pong) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_fullstack_streaming_pump) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_fullstack_trickle) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_fullstack_unary_ping_pong) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_metadata) @@ -1440,7 +1449,7 @@ add_library(grpc_test_util test/core/security/oauth2_utils.c test/core/end2end/cq_verifier.c test/core/end2end/fake_resolver.c - test/core/end2end/fixtures/http_proxy.c + test/core/end2end/fixtures/http_proxy_fixture.c test/core/end2end/fixtures/proxy.c test/core/iomgr/endpoint_tests.c test/core/util/debugger_macros.c @@ -1647,7 +1656,7 @@ if (gRPC_BUILD_TESTS) add_library(grpc_test_util_unsecure test/core/end2end/cq_verifier.c test/core/end2end/fake_resolver.c - test/core/end2end/fixtures/http_proxy.c + test/core/end2end/fixtures/http_proxy_fixture.c test/core/end2end/fixtures/proxy.c test/core/iomgr/endpoint_tests.c test/core/util/debugger_macros.c @@ -3098,6 +3107,50 @@ endif() if (gRPC_BUILD_TESTS) +add_library(grpc_benchmark + test/cpp/microbenchmarks/helpers.cc +) + +if(WIN32 AND MSVC) + set_target_properties(grpc_benchmark PROPERTIES COMPILE_PDB_NAME "grpc_benchmark" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc_benchmark.pdb + DESTINATION ${CMAKE_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + + +target_include_directories(grpc_benchmark + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${ZLIB_INCLUDE_DIR} + PRIVATE ${BENCHMARK}/include + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(grpc_benchmark + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + benchmark + grpc++ + grpc_test_util + grpc + ${_gRPC_GFLAGS_LIBRARIES} +) + + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_library(grpc_cli_libs test/cpp/util/cli_call.cc test/cpp/util/cli_credentials.cc @@ -7561,6 +7614,7 @@ target_include_directories(bm_call_create target_link_libraries(bm_call_create ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7599,6 +7653,7 @@ target_include_directories(bm_chttp2_hpack target_link_libraries(bm_chttp2_hpack ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7637,6 +7692,7 @@ target_include_directories(bm_closure target_link_libraries(bm_closure ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7675,6 +7731,7 @@ target_include_directories(bm_cq target_link_libraries(bm_cq ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7713,6 +7770,124 @@ target_include_directories(bm_error target_link_libraries(bm_error ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark + benchmark + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(bm_fullstack_streaming_ping_pong + test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_fullstack_streaming_ping_pong + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_fullstack_streaming_ping_pong + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark + benchmark + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(bm_fullstack_streaming_pump + test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_fullstack_streaming_pump + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_fullstack_streaming_pump + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark + benchmark + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(bm_fullstack_trickle + test/cpp/microbenchmarks/bm_fullstack_trickle.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_fullstack_trickle + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_fullstack_trickle + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7728,13 +7903,13 @@ endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) -add_executable(bm_fullstack - test/cpp/microbenchmarks/bm_fullstack.cc +add_executable(bm_fullstack_unary_ping_pong + test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc third_party/googletest/src/gtest-all.cc ) -target_include_directories(bm_fullstack +target_include_directories(bm_fullstack_unary_ping_pong PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${BORINGSSL_ROOT_DIR}/include @@ -7748,9 +7923,10 @@ target_include_directories(bm_fullstack PRIVATE ${_gRPC_PROTO_GENS_DIR} ) -target_link_libraries(bm_fullstack +target_link_libraries(bm_fullstack_unary_ping_pong ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc++_test_util grpc_test_util @@ -7789,6 +7965,7 @@ target_include_directories(bm_metadata target_link_libraries(bm_metadata ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark benchmark grpc_test_util grpc diff --git a/Makefile b/Makefile index e8643855eb3..a49fa928aac 100644 --- a/Makefile +++ b/Makefile @@ -1051,7 +1051,10 @@ bm_chttp2_hpack: $(BINDIR)/$(CONFIG)/bm_chttp2_hpack bm_closure: $(BINDIR)/$(CONFIG)/bm_closure bm_cq: $(BINDIR)/$(CONFIG)/bm_cq bm_error: $(BINDIR)/$(CONFIG)/bm_error -bm_fullstack: $(BINDIR)/$(CONFIG)/bm_fullstack +bm_fullstack_streaming_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong +bm_fullstack_streaming_pump: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump +bm_fullstack_trickle: $(BINDIR)/$(CONFIG)/bm_fullstack_trickle +bm_fullstack_unary_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong bm_metadata: $(BINDIR)/$(CONFIG)/bm_metadata channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test channel_filter_test: $(BINDIR)/$(CONFIG)/channel_filter_test @@ -1467,7 +1470,10 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_closure \ $(BINDIR)/$(CONFIG)/bm_cq \ $(BINDIR)/$(CONFIG)/bm_error \ - $(BINDIR)/$(CONFIG)/bm_fullstack \ + $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong \ + $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump \ + $(BINDIR)/$(CONFIG)/bm_fullstack_trickle \ + $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ @@ -1578,7 +1584,10 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_closure \ $(BINDIR)/$(CONFIG)/bm_cq \ $(BINDIR)/$(CONFIG)/bm_error \ - $(BINDIR)/$(CONFIG)/bm_fullstack \ + $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong \ + $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump \ + $(BINDIR)/$(CONFIG)/bm_fullstack_trickle \ + $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ @@ -1917,8 +1926,14 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/bm_cq || ( echo test bm_cq failed ; exit 1 ) $(E) "[RUN] Testing bm_error" $(Q) $(BINDIR)/$(CONFIG)/bm_error || ( echo test bm_error failed ; exit 1 ) - $(E) "[RUN] Testing bm_fullstack" - $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack || ( echo test bm_fullstack failed ; exit 1 ) + $(E) "[RUN] Testing bm_fullstack_streaming_ping_pong" + $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong || ( echo test bm_fullstack_streaming_ping_pong failed ; exit 1 ) + $(E) "[RUN] Testing bm_fullstack_streaming_pump" + $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump || ( echo test bm_fullstack_streaming_pump failed ; exit 1 ) + $(E) "[RUN] Testing bm_fullstack_trickle" + $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_trickle || ( echo test bm_fullstack_trickle failed ; exit 1 ) + $(E) "[RUN] Testing bm_fullstack_unary_ping_pong" + $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong || ( echo test bm_fullstack_unary_ping_pong failed ; exit 1 ) $(E) "[RUN] Testing bm_metadata" $(Q) $(BINDIR)/$(CONFIG)/bm_metadata || ( echo test bm_metadata failed ; exit 1 ) $(E) "[RUN] Testing channel_arguments_test" @@ -3318,7 +3333,7 @@ LIBGRPC_TEST_UTIL_SRC = \ test/core/security/oauth2_utils.c \ test/core/end2end/cq_verifier.c \ test/core/end2end/fake_resolver.c \ - test/core/end2end/fixtures/http_proxy.c \ + test/core/end2end/fixtures/http_proxy_fixture.c \ test/core/end2end/fixtures/proxy.c \ test/core/iomgr/endpoint_tests.c \ test/core/util/debugger_macros.c \ @@ -3518,7 +3533,7 @@ endif LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ test/core/end2end/cq_verifier.c \ test/core/end2end/fake_resolver.c \ - test/core/end2end/fixtures/http_proxy.c \ + test/core/end2end/fixtures/http_proxy_fixture.c \ test/core/end2end/fixtures/proxy.c \ test/core/iomgr/endpoint_tests.c \ test/core/util/debugger_macros.c \ @@ -4968,6 +4983,55 @@ ifneq ($(NO_DEPS),true) endif +LIBGRPC_BENCHMARK_SRC = \ + test/cpp/microbenchmarks/helpers.cc \ + +PUBLIC_HEADERS_CXX += \ + +LIBGRPC_BENCHMARK_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC_BENCHMARK_SRC)))) + + +ifeq ($(NO_SECURE),true) + +# You can't build secure libraries if you don't have OpenSSL. + +$(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: openssl_dep_error + + +else + +ifeq ($(NO_PROTOBUF),true) + +# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay. + +$(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: protobuf_dep_error + + +else + +$(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC_BENCHMARK_OBJS) + $(E) "[AR] Creating $@" + $(Q) mkdir -p `dirname $@` + $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a + $(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBGRPC_BENCHMARK_OBJS) +ifeq ($(SYSTEM),Darwin) + $(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a +endif + + + + +endif + +endif + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(LIBGRPC_BENCHMARK_OBJS:.o=.dep) +endif +endif + + LIBGRPC_CLI_LIBS_SRC = \ test/cpp/util/cli_call.cc \ test/cpp/util/cli_credentials.cc \ @@ -12551,16 +12615,16 @@ $(BINDIR)/$(CONFIG)/bm_call_create: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_call_create: $(PROTOBUF_DEP) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_call_create: $(PROTOBUF_DEP) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_call_create + $(Q) $(LDXX) $(LDFLAGS) $(BM_CALL_CREATE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_call_create endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_call_create.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_call_create.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_call_create: $(BM_CALL_CREATE_OBJS:.o=.dep) @@ -12594,16 +12658,16 @@ $(BINDIR)/$(CONFIG)/bm_chttp2_hpack: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_chttp2_hpack: $(PROTOBUF_DEP) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_chttp2_hpack: $(PROTOBUF_DEP) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_chttp2_hpack + $(Q) $(LDXX) $(LDFLAGS) $(BM_CHTTP2_HPACK_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_chttp2_hpack endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_chttp2_hpack.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_chttp2_hpack.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_chttp2_hpack: $(BM_CHTTP2_HPACK_OBJS:.o=.dep) @@ -12637,16 +12701,16 @@ $(BINDIR)/$(CONFIG)/bm_closure: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_closure: $(PROTOBUF_DEP) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_closure: $(PROTOBUF_DEP) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_closure + $(Q) $(LDXX) $(LDFLAGS) $(BM_CLOSURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_closure endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_closure.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_closure.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_closure: $(BM_CLOSURE_OBJS:.o=.dep) @@ -12680,16 +12744,16 @@ $(BINDIR)/$(CONFIG)/bm_cq: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_cq: $(PROTOBUF_DEP) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_cq: $(PROTOBUF_DEP) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_cq + $(Q) $(LDXX) $(LDFLAGS) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_cq endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_cq.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_cq.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_cq: $(BM_CQ_OBJS:.o=.dep) @@ -12723,16 +12787,16 @@ $(BINDIR)/$(CONFIG)/bm_error: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_error: $(PROTOBUF_DEP) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_error: $(PROTOBUF_DEP) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_error + $(Q) $(LDXX) $(LDFLAGS) $(BM_ERROR_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_error endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_error.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_error.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_error: $(BM_ERROR_OBJS:.o=.dep) @@ -12743,15 +12807,144 @@ endif endif -BM_FULLSTACK_SRC = \ - test/cpp/microbenchmarks/bm_fullstack.cc \ +BM_FULLSTACK_STREAMING_PING_PONG_SRC = \ + test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc \ + +BM_FULLSTACK_STREAMING_PING_PONG_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_STREAMING_PING_PONG_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong: $(PROTOBUF_DEP) $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_fullstack_streaming_ping_pong: $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS:.o=.dep) +endif +endif + + +BM_FULLSTACK_STREAMING_PUMP_SRC = \ + test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc \ + +BM_FULLSTACK_STREAMING_PUMP_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_STREAMING_PUMP_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump: $(PROTOBUF_DEP) $(BM_FULLSTACK_STREAMING_PUMP_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_STREAMING_PUMP_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_fullstack_streaming_pump: $(BM_FULLSTACK_STREAMING_PUMP_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_FULLSTACK_STREAMING_PUMP_OBJS:.o=.dep) +endif +endif + + +BM_FULLSTACK_TRICKLE_SRC = \ + test/cpp/microbenchmarks/bm_fullstack_trickle.cc \ + +BM_FULLSTACK_TRICKLE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_TRICKLE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_fullstack_trickle: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_fullstack_trickle: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_fullstack_trickle: $(PROTOBUF_DEP) $(BM_FULLSTACK_TRICKLE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_TRICKLE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_trickle + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_trickle.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_fullstack_trickle: $(BM_FULLSTACK_TRICKLE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_FULLSTACK_TRICKLE_OBJS:.o=.dep) +endif +endif + + +BM_FULLSTACK_UNARY_PING_PONG_SRC = \ + test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc \ -BM_FULLSTACK_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_SRC)))) +BM_FULLSTACK_UNARY_PING_PONG_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_FULLSTACK_UNARY_PING_PONG_SRC)))) ifeq ($(NO_SECURE),true) # You can't build secure targets if you don't have OpenSSL. -$(BINDIR)/$(CONFIG)/bm_fullstack: openssl_dep_error +$(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong: openssl_dep_error else @@ -12762,26 +12955,26 @@ ifeq ($(NO_PROTOBUF),true) # You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. -$(BINDIR)/$(CONFIG)/bm_fullstack: protobuf_dep_error +$(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_fullstack: $(PROTOBUF_DEP) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong: $(PROTOBUF_DEP) $(BM_FULLSTACK_UNARY_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack + $(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_UNARY_PING_PONG_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -deps_bm_fullstack: $(BM_FULLSTACK_OBJS:.o=.dep) +deps_bm_fullstack_unary_ping_pong: $(BM_FULLSTACK_UNARY_PING_PONG_OBJS:.o=.dep) ifneq ($(NO_SECURE),true) ifneq ($(NO_DEPS),true) --include $(BM_FULLSTACK_OBJS:.o=.dep) +-include $(BM_FULLSTACK_UNARY_PING_PONG_OBJS:.o=.dep) endif endif @@ -12809,16 +13002,16 @@ $(BINDIR)/$(CONFIG)/bm_metadata: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_metadata: $(PROTOBUF_DEP) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_metadata: $(PROTOBUF_DEP) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_metadata + $(Q) $(LDXX) $(LDFLAGS) $(BM_METADATA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_metadata endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_metadata.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_metadata.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_metadata: $(BM_METADATA_OBJS:.o=.dep) @@ -18260,6 +18453,7 @@ test/cpp/interop/interop_client.cc: $(OPENSSL_DEP) test/cpp/interop/interop_server.cc: $(OPENSSL_DEP) test/cpp/interop/interop_server_bootstrap.cc: $(OPENSSL_DEP) test/cpp/interop/server_helper.cc: $(OPENSSL_DEP) +test/cpp/microbenchmarks/helpers.cc: $(OPENSSL_DEP) test/cpp/qps/client_async.cc: $(OPENSSL_DEP) test/cpp/qps/client_sync.cc: $(OPENSSL_DEP) test/cpp/qps/driver.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 4f5b8e07f85..106e876a516 100644 --- a/build.yaml +++ b/build.yaml @@ -586,7 +586,7 @@ filegroups: headers: - test/core/end2end/cq_verifier.h - test/core/end2end/fake_resolver.h - - test/core/end2end/fixtures/http_proxy.h + - test/core/end2end/fixtures/http_proxy_fixture.h - test/core/end2end/fixtures/proxy.h - test/core/iomgr/endpoint_tests.h - test/core/util/debugger_macros.h @@ -602,7 +602,7 @@ filegroups: src: - test/core/end2end/cq_verifier.c - test/core/end2end/fake_resolver.c - - test/core/end2end/fixtures/http_proxy.c + - test/core/end2end/fixtures/http_proxy_fixture.c - test/core/end2end/fixtures/proxy.c - test/core/iomgr/endpoint_tests.c - test/core/util/debugger_macros.c @@ -1214,6 +1214,20 @@ libs: - grpc++_codegen_base_src secure: false vs_project_guid: '{6EE56155-DF7C-4F6E-BFC4-F6F776BEB211}' +- name: grpc_benchmark + build: test + language: c++ + headers: + - test/cpp/microbenchmarks/fullstack_context_mutators.h + - test/cpp/microbenchmarks/fullstack_fixtures.h + - test/cpp/microbenchmarks/helpers.h + src: + - test/cpp/microbenchmarks/helpers.cc + deps: + - benchmark + - grpc++ + - grpc_test_util + - grpc - name: grpc_cli_libs build: private language: c++ @@ -3031,6 +3045,7 @@ targets: src: - test/cpp/microbenchmarks/bm_call_create.cc deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3050,6 +3065,7 @@ targets: src: - test/cpp/microbenchmarks/bm_chttp2_hpack.cc deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3069,6 +3085,7 @@ targets: src: - test/cpp/microbenchmarks/bm_closure.cc deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3088,6 +3105,7 @@ targets: src: - test/cpp/microbenchmarks/bm_cq.cc deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3107,6 +3125,75 @@ targets: src: - test/cpp/microbenchmarks/bm_error.cc deps: + - grpc_benchmark + - benchmark + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + platforms: + - mac + - linux + - posix +- name: bm_fullstack_streaming_ping_pong + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc + deps: + - grpc_benchmark + - benchmark + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + excluded_poll_engines: + - poll + - poll-cv + platforms: + - mac + - linux + - posix + timeout_seconds: 1200 +- name: bm_fullstack_streaming_pump + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc + deps: + - grpc_benchmark + - benchmark + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + excluded_poll_engines: + - poll + - poll-cv + platforms: + - mac + - linux + - posix + timeout_seconds: 1200 +- name: bm_fullstack_trickle + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_fullstack_trickle.cc + deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3116,16 +3203,21 @@ targets: - gpr args: - --benchmark_min_time=0 + excluded_poll_engines: + - poll + - poll-cv platforms: - mac - linux - posix -- name: bm_fullstack + timeout_seconds: 1200 +- name: bm_fullstack_unary_ping_pong build: test language: c++ src: - - test/cpp/microbenchmarks/bm_fullstack.cc + - test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc deps: + - grpc_benchmark - benchmark - grpc++_test_util - grpc_test_util @@ -3149,6 +3241,7 @@ targets: src: - test/cpp/microbenchmarks/bm_metadata.cc deps: + - grpc_benchmark - benchmark - grpc_test_util - grpc diff --git a/include/grpc/impl/codegen/sync.h b/include/grpc/impl/codegen/sync.h index 96aec0c2c09..6a8e8a644f9 100644 --- a/include/grpc/impl/codegen/sync.h +++ b/include/grpc/impl/codegen/sync.h @@ -52,6 +52,10 @@ provides no memory barriers. */ +#ifdef __cplusplus +extern "C" { +#endif + /* Platform-specific type declarations of gpr_mu and gpr_cv. */ #include #include @@ -64,4 +68,8 @@ #error Unable to determine platform for sync #endif +#ifdef __cplusplus +} +#endif + #endif /* GRPC_IMPL_CODEGEN_SYNC_H */ diff --git a/test/core/end2end/BUILD b/test/core/end2end/BUILD index a40fb8e083f..0cef7aa01df 100644 --- a/test/core/end2end/BUILD +++ b/test/core/end2end/BUILD @@ -63,8 +63,8 @@ cc_library( cc_library( name = 'http_proxy', - hdrs = ['fixtures/http_proxy.h'], - srcs = ['fixtures/http_proxy.c'], + hdrs = ['fixtures/http_proxy_fixture.h'], + srcs = ['fixtures/http_proxy_fixture.c'], copts = ['-std=c99'], deps = ['//:gpr', '//:grpc', '//test/core/util:grpc_test_util'] ) diff --git a/test/core/end2end/fixtures/h2_http_proxy.c b/test/core/end2end/fixtures/h2_http_proxy.c index 44b223664ab..55c65fa70ef 100644 --- a/test/core/end2end/fixtures/h2_http_proxy.c +++ b/test/core/end2end/fixtures/h2_http_proxy.c @@ -49,7 +49,7 @@ #include "src/core/lib/support/env.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/server.h" -#include "test/core/end2end/fixtures/http_proxy.h" +#include "test/core/end2end/fixtures/http_proxy_fixture.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/test/core/end2end/fixtures/http_proxy.c b/test/core/end2end/fixtures/http_proxy_fixture.c similarity index 99% rename from test/core/end2end/fixtures/http_proxy.c rename to test/core/end2end/fixtures/http_proxy_fixture.c index 9ccb1263ee7..bcd1c9914b2 100644 --- a/test/core/end2end/fixtures/http_proxy.c +++ b/test/core/end2end/fixtures/http_proxy_fixture.c @@ -31,7 +31,7 @@ * */ -#include "test/core/end2end/fixtures/http_proxy.h" +#include "test/core/end2end/fixtures/http_proxy_fixture.h" #include "src/core/lib/iomgr/sockaddr.h" diff --git a/test/core/end2end/fixtures/http_proxy.h b/test/core/end2end/fixtures/http_proxy_fixture.h similarity index 100% rename from test/core/end2end/fixtures/http_proxy.h rename to test/core/end2end/fixtures/http_proxy_fixture.h diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index e2e5bbbe00e..f25bcd2df82 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -55,12 +55,10 @@ extern "C" { #include "src/core/lib/transport/transport_impl.h" } +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -static struct Init { - Init() { grpc_init(); } - ~Init() { grpc_shutdown(); } -} g_init; +auto &force_library_initialization = Library::get(); class BaseChannelFixture { public: @@ -89,6 +87,7 @@ class LameChannel : public BaseChannelFixture { template static void BM_CallCreateDestroy(benchmark::State &state) { + TrackCounters track_counters; Fixture fixture; grpc_completion_queue *cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); @@ -100,6 +99,7 @@ static void BM_CallCreateDestroy(benchmark::State &state) { deadline, NULL)); } grpc_completion_queue_destroy(cq); + track_counters.Finish(state); } BENCHMARK_TEMPLATE(BM_CallCreateDestroy, InsecureChannel); @@ -316,6 +316,7 @@ class SendEmptyMetadata { // perform on said filter. template static void BM_IsolatedFilter(benchmark::State &state) { + TrackCounters track_counters; Fixture fixture; std::ostringstream label; @@ -371,6 +372,7 @@ static void BM_IsolatedFilter(benchmark::State &state) { gpr_free(call_stack); state.SetLabel(label.str()); + track_counters.Finish(state); } typedef Fixture NoFilter; diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc index 5fb3f371306..563db758f71 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc @@ -42,18 +42,17 @@ extern "C" { #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/transport/static_metadata.h" } +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -static struct Init { - Init() { grpc_init(); } - ~Init() { grpc_shutdown(); } -} g_init; +auto &force_library_initialization = Library::get(); //////////////////////////////////////////////////////////////////////////////// // HPACK encoder // static void BM_HpackEncoderInitDestroy(benchmark::State &state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_chttp2_hpack_compressor c; while (state.KeepRunning()) { @@ -62,11 +61,13 @@ static void BM_HpackEncoderInitDestroy(benchmark::State &state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_HpackEncoderInitDestroy); template static void BM_HpackEncoderEncodeHeader(benchmark::State &state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_metadata_batch b; @@ -102,6 +103,7 @@ static void BM_HpackEncoderEncodeHeader(benchmark::State &state) { << " header_bytes/iter:" << (static_cast(stats.header_bytes) / static_cast(state.iterations())); state.SetLabel(label.str()); + track_counters.Finish(state); } namespace hpack_encoder_fixtures { @@ -216,6 +218,7 @@ BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, // static void BM_HpackParserInitDestroy(benchmark::State &state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_chttp2_hpack_parser p; while (state.KeepRunning()) { @@ -224,6 +227,7 @@ static void BM_HpackParserInitDestroy(benchmark::State &state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_HpackParserInitDestroy); @@ -234,6 +238,7 @@ static void UnrefHeader(grpc_exec_ctx *exec_ctx, void *user_data, template static void BM_HpackParserParseHeader(benchmark::State &state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; std::vector init_slices = Fixture::GetInitSlices(); std::vector benchmark_slices = Fixture::GetBenchmarkSlices(); @@ -252,6 +257,7 @@ static void BM_HpackParserParseHeader(benchmark::State &state) { } grpc_chttp2_hpack_parser_destroy(&exec_ctx, &p); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } namespace hpack_parser_fixtures { diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc index 1f54e8c8b17..28a385b6c11 100644 --- a/test/cpp/microbenchmarks/bm_closure.cc +++ b/test/cpp/microbenchmarks/bm_closure.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -42,86 +42,47 @@ extern "C" { #include "src/core/lib/support/spinlock.h" } +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -#include - -#ifdef GPR_LOW_LEVEL_COUNTERS -extern "C" gpr_atm gpr_mu_locks; -#endif - -static class InitializeStuff { - public: - InitializeStuff() { grpc_init(); } - ~InitializeStuff() { grpc_shutdown(); } -} initialize_stuff; - -class TrackCounters { - public: - TrackCounters(benchmark::State& state) : state_(state) {} - - ~TrackCounters() { - std::ostringstream out; -#ifdef GPR_LOW_LEVEL_COUNTERS - out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - - mu_locks_at_start_) / - (double)state_.iterations()) - << " atm_cas/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) - - atm_cas_at_start_) / - (double)state_.iterations()) - << " atm_add/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - - atm_add_at_start_) / - (double)state_.iterations()); -#endif - state_.SetLabel(out.str()); - } - - private: - benchmark::State& state_; -#ifdef GPR_LOW_LEVEL_COUNTERS - const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks); - const size_t atm_cas_at_start_ = - gpr_atm_no_barrier_load(&gpr_counter_atm_cas); - const size_t atm_add_at_start_ = - gpr_atm_no_barrier_load(&gpr_counter_atm_add); -#endif -}; +auto& force_library_initialization = Library::get(); static void BM_NoOpExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; while (state.KeepRunning()) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_exec_ctx_finish(&exec_ctx); } + track_counters.Finish(state); } BENCHMARK(BM_NoOpExecCtx); static void BM_WellFlushed(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; while (state.KeepRunning()) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_WellFlushed); static void DoNothing(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {} static void BM_ClosureInitAgainstExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_closure c; while (state.KeepRunning()) { benchmark::DoNotOptimize( grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx)); } + track_counters.Finish(state); } BENCHMARK(BM_ClosureInitAgainstExecCtx); static void BM_ClosureInitAgainstCombiner(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -131,11 +92,12 @@ static void BM_ClosureInitAgainstCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureInitAgainstCombiner); static void BM_ClosureRunOnExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_closure c; grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -144,11 +106,12 @@ static void BM_ClosureRunOnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureRunOnExecCtx); static void BM_ClosureCreateAndRun(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; while (state.KeepRunning()) { grpc_closure_run(&exec_ctx, grpc_closure_create(DoNothing, NULL, @@ -156,11 +119,12 @@ static void BM_ClosureCreateAndRun(benchmark::State& state) { GRPC_ERROR_NONE); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureCreateAndRun); static void BM_ClosureInitAndRun(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_closure c; while (state.KeepRunning()) { @@ -169,11 +133,12 @@ static void BM_ClosureInitAndRun(benchmark::State& state) { GRPC_ERROR_NONE); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureInitAndRun); static void BM_ClosureSchedOnExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_closure c; grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -182,11 +147,12 @@ static void BM_ClosureSchedOnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSchedOnExecCtx); static void BM_ClosureSched2OnExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_closure c1; grpc_closure c2; grpc_closure_init(&c1, DoNothing, NULL, grpc_schedule_on_exec_ctx); @@ -198,11 +164,12 @@ static void BM_ClosureSched2OnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched2OnExecCtx); static void BM_ClosureSched3OnExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_closure c1; grpc_closure c2; grpc_closure c3; @@ -217,11 +184,12 @@ static void BM_ClosureSched3OnExecCtx(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched3OnExecCtx); static void BM_AcquireMutex(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; // for comparison with the combiner stuff below gpr_mu mu; gpr_mu_init(&mu); @@ -232,11 +200,12 @@ static void BM_AcquireMutex(benchmark::State& state) { gpr_mu_unlock(&mu); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_AcquireMutex); static void BM_TryAcquireMutex(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; // for comparison with the combiner stuff below gpr_mu mu; gpr_mu_init(&mu); @@ -250,11 +219,12 @@ static void BM_TryAcquireMutex(benchmark::State& state) { } } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_TryAcquireMutex); static void BM_AcquireSpinlock(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; // for comparison with the combiner stuff below gpr_spinlock mu = GPR_SPINLOCK_INITIALIZER; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -264,11 +234,12 @@ static void BM_AcquireSpinlock(benchmark::State& state) { gpr_spinlock_unlock(&mu); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_AcquireSpinlock); static void BM_TryAcquireSpinlock(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; // for comparison with the combiner stuff below gpr_spinlock mu = GPR_SPINLOCK_INITIALIZER; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -281,11 +252,12 @@ static void BM_TryAcquireSpinlock(benchmark::State& state) { } } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_TryAcquireSpinlock); static void BM_ClosureSchedOnCombiner(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c; grpc_closure_init(&c, DoNothing, NULL, @@ -297,11 +269,12 @@ static void BM_ClosureSchedOnCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSchedOnCombiner); static void BM_ClosureSched2OnCombiner(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c1; grpc_closure c2; @@ -317,11 +290,12 @@ static void BM_ClosureSched2OnCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched2OnCombiner); static void BM_ClosureSched3OnCombiner(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner = grpc_combiner_create(NULL); grpc_closure c1; grpc_closure c2; @@ -341,11 +315,12 @@ static void BM_ClosureSched3OnCombiner(benchmark::State& state) { } GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched3OnCombiner); static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner1 = grpc_combiner_create(NULL); grpc_combiner* combiner2 = grpc_combiner_create(NULL); grpc_closure c1; @@ -363,11 +338,12 @@ static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) { GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished"); GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched2OnTwoCombiners); static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_combiner* combiner1 = grpc_combiner_create(NULL); grpc_combiner* combiner2 = grpc_combiner_create(NULL); grpc_closure c1; @@ -393,6 +369,7 @@ static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) { GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished"); GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureSched4OnTwoCombiners); @@ -428,16 +405,17 @@ class Rescheduler { }; static void BM_ClosureReschedOnExecCtx(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; Rescheduler r(state, grpc_schedule_on_exec_ctx); r.ScheduleFirst(&exec_ctx); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureReschedOnExecCtx); static void BM_ClosureReschedOnCombiner(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_combiner* combiner = grpc_combiner_create(NULL); Rescheduler r(state, grpc_combiner_scheduler(combiner, false)); @@ -445,11 +423,12 @@ static void BM_ClosureReschedOnCombiner(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureReschedOnCombiner); static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) { - TrackCounters track_counters(state); + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_combiner* combiner = grpc_combiner_create(NULL); Rescheduler r(state, grpc_combiner_finally_scheduler(combiner, false)); @@ -458,6 +437,7 @@ static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) { grpc_exec_ctx_flush(&exec_ctx); GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished"); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_ClosureReschedOnCombinerFinally); diff --git a/test/cpp/microbenchmarks/bm_cq.cc b/test/cpp/microbenchmarks/bm_cq.cc index c017474bf4a..91e6a851017 100644 --- a/test/cpp/microbenchmarks/bm_cq.cc +++ b/test/cpp/microbenchmarks/bm_cq.cc @@ -38,6 +38,7 @@ #include #include +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" extern "C" { @@ -47,27 +48,23 @@ extern "C" { namespace grpc { namespace testing { -static class InitializeStuff { - public: - InitializeStuff() { init_lib_.init(); } - ~InitializeStuff() { init_lib_.shutdown(); } - - private: - internal::GrpcLibrary init_lib_; - internal::GrpcLibraryInitializer init_; -} initialize_stuff; +auto& force_library_initialization = Library::get(); static void BM_CreateDestroyCpp(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { CompletionQueue cq; } + track_counters.Finish(state); } BENCHMARK(BM_CreateDestroyCpp); static void BM_CreateDestroyCore(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { grpc_completion_queue_destroy(grpc_completion_queue_create(NULL)); } + track_counters.Finish(state); } BENCHMARK(BM_CreateDestroyCore); @@ -80,6 +77,7 @@ class DummyTag final : public CompletionQueueTag { }; static void BM_Pass1Cpp(benchmark::State& state) { + TrackCounters track_counters; CompletionQueue cq; grpc_completion_queue* c_cq = cq.cq(); while (state.KeepRunning()) { @@ -94,10 +92,12 @@ static void BM_Pass1Cpp(benchmark::State& state) { bool ok; cq.Next(&tag, &ok); } + track_counters.Finish(state); } BENCHMARK(BM_Pass1Cpp); static void BM_Pass1Core(benchmark::State& state) { + TrackCounters track_counters; grpc_completion_queue* cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); while (state.KeepRunning()) { @@ -110,10 +110,12 @@ static void BM_Pass1Core(benchmark::State& state) { grpc_completion_queue_next(cq, deadline, NULL); } grpc_completion_queue_destroy(cq); + track_counters.Finish(state); } BENCHMARK(BM_Pass1Core); static void BM_Pluck1Core(benchmark::State& state) { + TrackCounters track_counters; grpc_completion_queue* cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); while (state.KeepRunning()) { @@ -126,16 +128,19 @@ static void BM_Pluck1Core(benchmark::State& state) { grpc_completion_queue_pluck(cq, NULL, deadline, NULL); } grpc_completion_queue_destroy(cq); + track_counters.Finish(state); } BENCHMARK(BM_Pluck1Core); static void BM_EmptyCore(benchmark::State& state) { + TrackCounters track_counters; grpc_completion_queue* cq = grpc_completion_queue_create(NULL); gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); while (state.KeepRunning()) { grpc_completion_queue_next(cq, deadline, NULL); } grpc_completion_queue_destroy(cq); + track_counters.Finish(state); } BENCHMARK(BM_EmptyCore); diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc index 8a4b86f281a..95f19e75860 100644 --- a/test/cpp/microbenchmarks/bm_error.cc +++ b/test/cpp/microbenchmarks/bm_error.cc @@ -40,8 +40,11 @@ extern "C" { #include "src/core/lib/transport/error_utils.h" } +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" +auto& force_library_initialization = Library::get(); + class ErrorDeleter { public: void operator()(grpc_error* error) { GRPC_ERROR_UNREF(error); } @@ -49,31 +52,38 @@ class ErrorDeleter { typedef std::unique_ptr ErrorPtr; static void BM_ErrorCreate(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { GRPC_ERROR_UNREF(GRPC_ERROR_CREATE("Error")); } + track_counters.Finish(state); } BENCHMARK(BM_ErrorCreate); static void BM_ErrorCreateAndSetStatus(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { GRPC_ERROR_UNREF(grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_ABORTED)); } + track_counters.Finish(state); } BENCHMARK(BM_ErrorCreateAndSetStatus); static void BM_ErrorRefUnref(benchmark::State& state) { + TrackCounters track_counters; grpc_error* error = GRPC_ERROR_CREATE("Error"); while (state.KeepRunning()) { GRPC_ERROR_UNREF(GRPC_ERROR_REF(error)); } GRPC_ERROR_UNREF(error); + track_counters.Finish(state); } BENCHMARK(BM_ErrorRefUnref); static void BM_ErrorUnrefNone(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { GRPC_ERROR_UNREF(GRPC_ERROR_NONE); } @@ -81,30 +91,36 @@ static void BM_ErrorUnrefNone(benchmark::State& state) { BENCHMARK(BM_ErrorUnrefNone); static void BM_ErrorGetIntFromNoError(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { intptr_t value; grpc_error_get_int(GRPC_ERROR_NONE, GRPC_ERROR_INT_GRPC_STATUS, &value); } + track_counters.Finish(state); } BENCHMARK(BM_ErrorGetIntFromNoError); static void BM_ErrorGetMissingInt(benchmark::State& state) { + TrackCounters track_counters; ErrorPtr error( grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_INDEX, 1)); while (state.KeepRunning()) { intptr_t value; grpc_error_get_int(error.get(), GRPC_ERROR_INT_OFFSET, &value); } + track_counters.Finish(state); } BENCHMARK(BM_ErrorGetMissingInt); static void BM_ErrorGetPresentInt(benchmark::State& state) { + TrackCounters track_counters; ErrorPtr error( grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_OFFSET, 1)); while (state.KeepRunning()) { intptr_t value; grpc_error_get_int(error.get(), GRPC_ERROR_INT_OFFSET, &value); } + track_counters.Finish(state); } BENCHMARK(BM_ErrorGetPresentInt); @@ -177,22 +193,27 @@ class ErrorWithNestedGrpcStatus { template static void BM_ErrorStringOnNewError(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { Fixture fixture; grpc_error_string(fixture.error()); } + track_counters.Finish(state); } template static void BM_ErrorStringRepeatedly(benchmark::State& state) { + TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_error_string(fixture.error()); } + track_counters.Finish(state); } template static void BM_ErrorGetStatus(benchmark::State& state) { + TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_status_code status; @@ -200,34 +221,41 @@ static void BM_ErrorGetStatus(benchmark::State& state) { grpc_error_get_status(fixture.error(), fixture.deadline(), &status, &msg, NULL); } + track_counters.Finish(state); } template static void BM_ErrorGetStatusCode(benchmark::State& state) { + TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_status_code status; grpc_error_get_status(fixture.error(), fixture.deadline(), &status, NULL, NULL); } + track_counters.Finish(state); } template static void BM_ErrorHttpError(benchmark::State& state) { + TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_http2_error_code error; grpc_error_get_status(fixture.error(), fixture.deadline(), NULL, NULL, &error); } + track_counters.Finish(state); } template static void BM_HasClearGrpcStatus(benchmark::State& state) { + TrackCounters track_counters; Fixture fixture; while (state.KeepRunning()) { grpc_error_has_clear_grpc_status(fixture.error()); } + track_counters.Finish(state); } #define BENCHMARK_SUITE(fixture) \ diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc deleted file mode 100644 index 48e131f1be0..00000000000 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ /dev/null @@ -1,1079 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* Benchmark gRPC end2end in various configurations */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" { -#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" -#include "src/core/ext/transport/chttp2/transport/internal.h" -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/iomgr/endpoint.h" -#include "src/core/lib/iomgr/endpoint_pair.h" -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/iomgr/tcp_posix.h" -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/surface/completion_queue.h" -#include "src/core/lib/surface/server.h" -#include "test/core/util/memory_counters.h" -#include "test/core/util/passthru_endpoint.h" -#include "test/core/util/port.h" -#include "test/core/util/trickle_endpoint.h" -} -#include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" -#include "src/proto/grpc/testing/echo.grpc.pb.h" -#include "third_party/benchmark/include/benchmark/benchmark.h" - -namespace grpc { -namespace testing { - -static class InitializeStuff { - public: - InitializeStuff() { - grpc_memory_counters_init(); - init_lib_.init(); - rq_ = grpc_resource_quota_create("bm"); - } - - ~InitializeStuff() { init_lib_.shutdown(); } - - grpc_resource_quota* rq() { return rq_; } - - private: - internal::GrpcLibrary init_lib_; - grpc_resource_quota* rq_; -} initialize_stuff; - -/******************************************************************************* - * FIXTURES - */ - -static void ApplyCommonServerBuilderConfig(ServerBuilder* b) { - b->SetMaxReceiveMessageSize(INT_MAX); - b->SetMaxSendMessageSize(INT_MAX); -} - -static void ApplyCommonChannelArguments(ChannelArguments* c) { - c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX); - c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX); -} - -#ifdef GPR_LOW_LEVEL_COUNTERS -extern "C" gpr_atm gpr_mu_locks; -extern "C" gpr_atm gpr_counter_atm_cas; -extern "C" gpr_atm gpr_counter_atm_add; -#endif - -class BaseFixture { - public: - void Finish(benchmark::State& s) { - std::ostringstream out; - this->AddToLabel(out, s); -#ifdef GPR_LOW_LEVEL_COUNTERS - out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - - mu_locks_at_start_) / - (double)s.iterations()) - << " atm_cas/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) - - atm_cas_at_start_) / - (double)s.iterations()) - << " atm_add/iter:" - << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - - atm_add_at_start_) / - (double)s.iterations()); -#endif - grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); - out << " allocs/iter:" - << ((double)(counters_at_end.total_allocs_absolute - - counters_at_start_.total_allocs_absolute) / - (double)s.iterations()); - auto label = out.str(); - if (label.length() && label[0] == ' ') { - label = label.substr(1); - } - s.SetLabel(label); - } - - virtual void AddToLabel(std::ostream& out, benchmark::State& s) = 0; - - private: -#ifdef GPR_LOW_LEVEL_COUNTERS - const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks); - const size_t atm_cas_at_start_ = - gpr_atm_no_barrier_load(&gpr_counter_atm_cas); - const size_t atm_add_at_start_ = - gpr_atm_no_barrier_load(&gpr_counter_atm_add); -#endif - grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot(); -}; - -class FullstackFixture : public BaseFixture { - public: - FullstackFixture(Service* service, const grpc::string& address) { - ServerBuilder b; - b.AddListeningPort(address, InsecureServerCredentials()); - cq_ = b.AddCompletionQueue(true); - b.RegisterService(service); - ApplyCommonServerBuilderConfig(&b); - server_ = b.BuildAndStart(); - ChannelArguments args; - ApplyCommonChannelArguments(&args); - channel_ = CreateCustomChannel(address, InsecureChannelCredentials(), args); - } - - virtual ~FullstackFixture() { - server_->Shutdown(); - cq_->Shutdown(); - void* tag; - bool ok; - while (cq_->Next(&tag, &ok)) { - } - } - - ServerCompletionQueue* cq() { return cq_.get(); } - std::shared_ptr channel() { return channel_; } - - private: - std::unique_ptr server_; - std::unique_ptr cq_; - std::shared_ptr channel_; -}; - -class TCP : public FullstackFixture { - public: - TCP(Service* service) : FullstackFixture(service, MakeAddress()) {} - - void AddToLabel(std::ostream& out, benchmark::State& state) {} - - private: - static grpc::string MakeAddress() { - int port = grpc_pick_unused_port_or_die(); - std::stringstream addr; - addr << "localhost:" << port; - return addr.str(); - } -}; - -class UDS : public FullstackFixture { - public: - UDS(Service* service) : FullstackFixture(service, MakeAddress()) {} - - void AddToLabel(std::ostream& out, benchmark::State& state) override {} - - private: - static grpc::string MakeAddress() { - int port = grpc_pick_unused_port_or_die(); // just for a unique id - not a - // real port - std::stringstream addr; - addr << "unix:/tmp/bm_fullstack." << port; - return addr.str(); - } -}; - -class EndpointPairFixture : public BaseFixture { - public: - EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) - : endpoint_pair_(endpoints) { - ServerBuilder b; - cq_ = b.AddCompletionQueue(true); - b.RegisterService(service); - ApplyCommonServerBuilderConfig(&b); - server_ = b.BuildAndStart(); - - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - - /* add server endpoint to server_ */ - { - const grpc_channel_args* server_args = - grpc_server_get_channel_args(server_->c_server()); - server_transport_ = grpc_create_chttp2_transport( - &exec_ctx, server_args, endpoints.server, 0 /* is_client */); - - grpc_pollset** pollsets; - size_t num_pollsets = 0; - grpc_server_get_pollsets(server_->c_server(), &pollsets, &num_pollsets); - - for (size_t i = 0; i < num_pollsets; i++) { - grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]); - } - - grpc_server_setup_transport(&exec_ctx, server_->c_server(), - server_transport_, NULL, server_args); - grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL); - } - - /* create channel */ - { - ChannelArguments args; - args.SetString(GRPC_ARG_DEFAULT_AUTHORITY, "test.authority"); - ApplyCommonChannelArguments(&args); - - grpc_channel_args c_args = args.c_channel_args(); - client_transport_ = - grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1); - GPR_ASSERT(client_transport_); - grpc_channel* channel = - grpc_channel_create(&exec_ctx, "target", &c_args, - GRPC_CLIENT_DIRECT_CHANNEL, client_transport_); - grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL); - - channel_ = CreateChannelInternal("", channel); - } - - grpc_exec_ctx_finish(&exec_ctx); - } - - virtual ~EndpointPairFixture() { - server_->Shutdown(); - cq_->Shutdown(); - void* tag; - bool ok; - while (cq_->Next(&tag, &ok)) { - } - } - - ServerCompletionQueue* cq() { return cq_.get(); } - std::shared_ptr channel() { return channel_; } - - protected: - grpc_endpoint_pair endpoint_pair_; - grpc_transport* client_transport_; - grpc_transport* server_transport_; - - private: - std::unique_ptr server_; - std::unique_ptr cq_; - std::shared_ptr channel_; -}; - -class SockPair : public EndpointPairFixture { - public: - SockPair(Service* service) - : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair( - "test", initialize_stuff.rq(), 8192)) { - } - - void AddToLabel(std::ostream& out, benchmark::State& state) {} -}; - -class InProcessCHTTP2 : public EndpointPairFixture { - public: - InProcessCHTTP2(Service* service) - : EndpointPairFixture(service, MakeEndpoints()) {} - - void AddToLabel(std::ostream& out, benchmark::State& state) { - out << " writes/iter:" - << ((double)stats_.num_writes / (double)state.iterations()); - } - - private: - grpc_passthru_endpoint_stats stats_; - - grpc_endpoint_pair MakeEndpoints() { - grpc_endpoint_pair p; - grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(), - &stats_); - return p; - } -}; - -class TrickledCHTTP2 : public EndpointPairFixture { - public: - TrickledCHTTP2(Service* service, size_t megabits_per_second) - : EndpointPairFixture(service, MakeEndpoints(megabits_per_second)) {} - - void AddToLabel(std::ostream& out, benchmark::State& state) { - out << " writes/iter:" - << ((double)stats_.num_writes / (double)state.iterations()) - << " cli_transport_stalls/iter:" - << ((double) - client_stats_.streams_stalled_due_to_transport_flow_control / - (double)state.iterations()) - << " cli_stream_stalls/iter:" - << ((double)client_stats_.streams_stalled_due_to_stream_flow_control / - (double)state.iterations()) - << " svr_transport_stalls/iter:" - << ((double) - server_stats_.streams_stalled_due_to_transport_flow_control / - (double)state.iterations()) - << " svr_stream_stalls/iter:" - << ((double)server_stats_.streams_stalled_due_to_stream_flow_control / - (double)state.iterations()); - } - - void Step() { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - size_t client_backlog = - grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.client); - size_t server_backlog = - grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server); - grpc_exec_ctx_finish(&exec_ctx); - - UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_, - client_backlog); - UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_, - server_backlog); - } - - private: - grpc_passthru_endpoint_stats stats_; - struct Stats { - int streams_stalled_due_to_stream_flow_control = 0; - int streams_stalled_due_to_transport_flow_control = 0; - }; - Stats client_stats_; - Stats server_stats_; - - grpc_endpoint_pair MakeEndpoints(size_t kilobits) { - grpc_endpoint_pair p; - grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(), - &stats_); - double bytes_per_second = 125.0 * kilobits; - p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second); - p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second); - return p; - } - - void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) { - if (backlog == 0) { - if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) { - s->streams_stalled_due_to_stream_flow_control++; - } - if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) { - s->streams_stalled_due_to_transport_flow_control++; - } - } - } -}; - -/******************************************************************************* - * CONTEXT MUTATORS - */ - -static const int kPregenerateKeyCount = 100000; - -template -auto MakeVector(size_t length, F f) -> std::vector { - std::vector out; - out.reserve(length); - for (size_t i = 0; i < length; i++) { - out.push_back(f()); - } - return out; -} - -class NoOpMutator { - public: - template - NoOpMutator(ContextType* context) {} -}; - -template -class RandomBinaryMetadata { - public: - static const grpc::string& Key() { return kKey; } - - static const grpc::string& Value() { - return kValues[rand() % kValues.size()]; - } - - private: - static const grpc::string kKey; - static const std::vector kValues; - - static grpc::string GenerateOneString() { - grpc::string s; - s.reserve(length + 1); - for (int i = 0; i < length; i++) { - s += (char)rand(); - } - return s; - } -}; - -template -const grpc::string RandomBinaryMetadata::kKey = "foo-bin"; - -template -const std::vector RandomBinaryMetadata::kValues = - MakeVector(kPregenerateKeyCount, GenerateOneString); - -template -class RandomAsciiMetadata { - public: - static const grpc::string& Key() { return kKey; } - - static const grpc::string& Value() { - return kValues[rand() % kValues.size()]; - } - - private: - static const grpc::string kKey; - static const std::vector kValues; - - static grpc::string GenerateOneString() { - grpc::string s; - s.reserve(length + 1); - for (int i = 0; i < length; i++) { - s += (char)(rand() % 26 + 'a'); - } - return s; - } -}; - -template -const grpc::string RandomAsciiMetadata::kKey = "foo"; - -template -const std::vector RandomAsciiMetadata::kValues = - MakeVector(kPregenerateKeyCount, GenerateOneString); - -template -class Client_AddMetadata : public NoOpMutator { - public: - Client_AddMetadata(ClientContext* context) : NoOpMutator(context) { - for (int i = 0; i < kNumKeys; i++) { - context->AddMetadata(Generator::Key(), Generator::Value()); - } - } -}; - -template -class Server_AddInitialMetadata : public NoOpMutator { - public: - Server_AddInitialMetadata(ServerContext* context) : NoOpMutator(context) { - for (int i = 0; i < kNumKeys; i++) { - context->AddInitialMetadata(Generator::Key(), Generator::Value()); - } - } -}; - -/******************************************************************************* - * BENCHMARKING KERNELS - */ - -static void* tag(intptr_t x) { return reinterpret_cast(x); } - -template -static void BM_UnaryPingPong(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - EchoRequest send_request; - EchoResponse send_response; - EchoResponse recv_response; - if (state.range(0) > 0) { - send_request.set_message(std::string(state.range(0), 'a')); - } - if (state.range(1) > 0) { - send_response.set_message(std::string(state.range(1), 'a')); - } - Status recv_status; - struct ServerEnv { - ServerContext ctx; - EchoRequest recv_request; - grpc::ServerAsyncResponseWriter response_writer; - ServerEnv() : response_writer(&ctx) {} - }; - uint8_t server_env_buffer[2 * sizeof(ServerEnv)]; - ServerEnv* server_env[2] = { - reinterpret_cast(server_env_buffer), - reinterpret_cast(server_env_buffer + sizeof(ServerEnv))}; - new (server_env[0]) ServerEnv; - new (server_env[1]) ServerEnv; - service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request, - &server_env[0]->response_writer, fixture->cq(), - fixture->cq(), tag(0)); - service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request, - &server_env[1]->response_writer, fixture->cq(), - fixture->cq(), tag(1)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - recv_response.Clear(); - ClientContext cli_ctx; - ClientContextMutator cli_ctx_mut(&cli_ctx); - std::unique_ptr> response_reader( - stub->AsyncEcho(&cli_ctx, send_request, fixture->cq())); - void* t; - bool ok; - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - GPR_ASSERT(t == tag(0) || t == tag(1)); - intptr_t slot = reinterpret_cast(t); - ServerEnv* senv = server_env[slot]; - ServerContextMutator svr_ctx_mut(&senv->ctx); - senv->response_writer.Finish(send_response, Status::OK, tag(3)); - response_reader->Finish(&recv_response, &recv_status, tag(4)); - for (int i = (1 << 3) | (1 << 4); i != 0;) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int tagnum = (int)reinterpret_cast(t); - GPR_ASSERT(i & (1 << tagnum)); - i -= 1 << tagnum; - } - GPR_ASSERT(recv_status.ok()); - - senv->~ServerEnv(); - senv = new (senv) ServerEnv(); - service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer, - fixture->cq(), fixture->cq(), tag(slot)); - } - fixture->Finish(state); - fixture.reset(); - server_env[0]->~ServerEnv(); - server_env[1]->~ServerEnv(); - state.SetBytesProcessed(state.range(0) * state.iterations() + - state.range(1) * state.iterations()); -} - -// Repeatedly makes Streaming Bidi calls (exchanging a configurable number of -// messages in each call) in a loop on a single channel -// -// First parmeter (i.e state.range(0)): Message size (in bytes) to use -// Second parameter (i.e state.range(1)): Number of ping pong messages. -// Note: One ping-pong means two messages (one from client to server and -// the other from server to client): -template -static void BM_StreamingPingPong(benchmark::State& state) { - const int msg_size = state.range(0); - const int max_ping_pongs = state.range(1); - - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoResponse send_response; - EchoResponse recv_response; - EchoRequest send_request; - EchoRequest recv_request; - - if (msg_size > 0) { - send_request.set_message(std::string(msg_size, 'a')); - send_response.set_message(std::string(msg_size, 'b')); - } - - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - - while (state.KeepRunning()) { - ServerContext svr_ctx; - ServerContextMutator svr_ctx_mut(&svr_ctx); - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - - ClientContext cli_ctx; - ClientContextMutator cli_ctx_mut(&cli_ctx); - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - - // Establish async stream between client side and server side - void* t; - bool ok; - int need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - - // Send 'max_ping_pongs' number of ping pong messages - int ping_pong_cnt = 0; - while (ping_pong_cnt < max_ping_pongs) { - request_rw->Write(send_request, tag(0)); // Start client send - response_rw.Read(&recv_request, tag(1)); // Start server recv - request_rw->Read(&recv_response, tag(2)); // Start client recv - - need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - - // If server recv is complete, start the server send operation - if (i == 1) { - response_rw.Write(send_response, tag(3)); - } - - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - - ping_pong_cnt++; - } - - request_rw->WritesDone(tag(0)); - response_rw.Finish(Status::OK, tag(1)); - - Status recv_status; - request_rw->Finish(&recv_status, tag(2)); - - need_tags = (1 << 0) | (1 << 1) | (1 << 2); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - - GPR_ASSERT(recv_status.ok()); - } - } - - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(msg_size * state.iterations() * max_ping_pongs * 2); -} - -// Repeatedly sends ping pong messages in a single streaming Bidi call in a loop -// First parmeter (i.e state.range(0)): Message size (in bytes) to use -template -static void BM_StreamingPingPongMsgs(benchmark::State& state) { - const int msg_size = state.range(0); - - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoResponse send_response; - EchoResponse recv_response; - EchoRequest send_request; - EchoRequest recv_request; - - if (msg_size > 0) { - send_request.set_message(std::string(msg_size, 'a')); - send_response.set_message(std::string(msg_size, 'b')); - } - - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - - ServerContext svr_ctx; - ServerContextMutator svr_ctx_mut(&svr_ctx); - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - - ClientContext cli_ctx; - ClientContextMutator cli_ctx_mut(&cli_ctx); - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - - // Establish async stream between client side and server side - void* t; - bool ok; - int need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - request_rw->Write(send_request, tag(0)); // Start client send - response_rw.Read(&recv_request, tag(1)); // Start server recv - request_rw->Read(&recv_response, tag(2)); // Start client recv - - need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - - // If server recv is complete, start the server send operation - if (i == 1) { - response_rw.Write(send_response, tag(3)); - } - - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - - request_rw->WritesDone(tag(0)); - response_rw.Finish(Status::OK, tag(1)); - Status recv_status; - request_rw->Finish(&recv_status, tag(2)); - - need_tags = (1 << 0) | (1 << 1) | (1 << 2); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - - GPR_ASSERT(recv_status.ok()); - } - - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(msg_size * state.iterations() * 2); -} - -template -static void BM_PumpStreamClientToServer(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoRequest send_request; - EchoRequest recv_request; - if (state.range(0) > 0) { - send_request.set_message(std::string(state.range(0), 'a')); - } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - response_rw.Read(&recv_request, tag(0)); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - request_rw->Write(send_request, tag(1)); - while (true) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - if (t == tag(0)) { - response_rw.Read(&recv_request, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); - } - } - } - request_rw->WritesDone(tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); -} - -template -static void BM_PumpStreamServerToClient(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture(new Fixture(&service)); - { - EchoResponse send_response; - EchoResponse recv_response; - if (state.range(0) > 0) { - send_response.set_message(std::string(state.range(0), 'a')); - } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - request_rw->Read(&recv_response, tag(0)); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - response_rw.Write(send_response, tag(1)); - while (true) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - if (t == tag(0)) { - request_rw->Read(&recv_response, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); - } - } - } - response_rw.Finish(Status::OK, tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); -} - -static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) { - while (true) { - switch (fixture->cq()->AsyncNext( - t, ok, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_micros(100, GPR_TIMESPAN)))) { - case CompletionQueue::TIMEOUT: - fixture->Step(); - break; - case CompletionQueue::SHUTDOWN: - GPR_ASSERT(false); - break; - case CompletionQueue::GOT_EVENT: - return; - } - } -} - -static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { - EchoTestService::AsyncService service; - std::unique_ptr fixture( - new TrickledCHTTP2(&service, state.range(1))); - { - EchoResponse send_response; - EchoResponse recv_response; - if (state.range(0) > 0) { - send_response.set_message(std::string(state.range(0), 'a')); - } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - TrickleCQNext(fixture.get(), &t, &ok); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - request_rw->Read(&recv_response, tag(0)); - while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - response_rw.Write(send_response, tag(1)); - while (true) { - TrickleCQNext(fixture.get(), &t, &ok); - if (t == tag(0)) { - request_rw->Read(&recv_response, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); - } - } - } - response_rw.Finish(Status::OK, tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - TrickleCQNext(fixture.get(), &t, &ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - } - fixture->Finish(state); - fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); -} - -/******************************************************************************* - * CONFIGURATIONS - */ - -static void SweepSizesArgs(benchmark::internal::Benchmark* b) { - b->Args({0, 0}); - for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { - b->Args({i, 0}); - b->Args({0, i}); - b->Args({i, i}); - } -} - -BENCHMARK_TEMPLATE(BM_UnaryPingPong, TCP, NoOpMutator, NoOpMutator) - ->Apply(SweepSizesArgs); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, UDS, NoOpMutator, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, SockPair, NoOpMutator, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, NoOpMutator) - ->Apply(SweepSizesArgs); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, - NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 2>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 2>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 2>, - NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, - Client_AddMetadata, 1>, NoOpMutator) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 1>) - ->Args({0, 0}); -BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, - Server_AddInitialMetadata, 100>) - ->Args({0, 0}); - -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) - ->Range(0, 128 * 1024 * 1024); - -static void TrickleArgs(benchmark::internal::Benchmark* b) { - for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { - for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) { - double expected_time = - static_cast(14 + i) / (125.0 * static_cast(j)); - if (expected_time > 0.01) continue; - b->Args({i, j}); - } - } -} - -BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs); - -// Generate Args for StreamingPingPong benchmarks. Currently generates args for -// only "small streams" (i.e streams with 0, 1 or 2 messages) -static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) { - int msg_size = 0; - - b->Args({0, 0}); // spl case: 0 ping-pong msgs (msg_size doesn't matter here) - - for (msg_size = 0; msg_size <= 128 * 1024 * 1024; - msg_size == 0 ? msg_size++ : msg_size *= 8) { - b->Args({msg_size, 1}); - b->Args({msg_size, 2}); - } -} - -BENCHMARK_TEMPLATE(BM_StreamingPingPong, InProcessCHTTP2, NoOpMutator, - NoOpMutator) - ->Apply(StreamingPingPongArgs); -BENCHMARK_TEMPLATE(BM_StreamingPingPong, TCP, NoOpMutator, NoOpMutator) - ->Apply(StreamingPingPongArgs); - -BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, InProcessCHTTP2, NoOpMutator, - NoOpMutator) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, TCP, NoOpMutator, NoOpMutator) - ->Range(0, 128 * 1024 * 1024); - -} // namespace testing -} // namespace grpc - -BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc new file mode 100644 index 00000000000..dc0e7d769ab --- /dev/null +++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc @@ -0,0 +1,197 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Benchmark gRPC end2end in various configurations */ + +#include + +#include "src/core/lib/profiling/timers.h" +#include "src/cpp/client/create_channel_internal.h" +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +namespace grpc { +namespace testing { + +// force library initialization +auto& force_library_initialization = Library::get(); + +/******************************************************************************* + * BENCHMARKING KERNELS + */ + +static void* tag(intptr_t x) { return reinterpret_cast(x); } + +template +static void BM_PumpStreamClientToServer(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoRequest send_request; + EchoRequest recv_request; + if (state.range(0) > 0) { + send_request.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + response_rw.Read(&recv_request, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + request_rw->Write(send_request, tag(1)); + while (true) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + if (t == tag(0)) { + response_rw.Read(&recv_request, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + request_rw->WritesDone(tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +template +static void BM_PumpStreamServerToClient(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_response.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + request_rw->Read(&recv_response, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + response_rw.Write(send_response, tag(1)); + while (true) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + if (t == tag(0)) { + request_rw->Read(&recv_response, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + response_rw.Finish(Status::OK, tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +/******************************************************************************* + * CONFIGURATIONS + */ + +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) + ->Range(0, 128 * 1024 * 1024); + +} // namespace testing +} // namespace grpc + +BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc new file mode 100644 index 00000000000..dc0e7d769ab --- /dev/null +++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc @@ -0,0 +1,197 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Benchmark gRPC end2end in various configurations */ + +#include + +#include "src/core/lib/profiling/timers.h" +#include "src/cpp/client/create_channel_internal.h" +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +namespace grpc { +namespace testing { + +// force library initialization +auto& force_library_initialization = Library::get(); + +/******************************************************************************* + * BENCHMARKING KERNELS + */ + +static void* tag(intptr_t x) { return reinterpret_cast(x); } + +template +static void BM_PumpStreamClientToServer(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoRequest send_request; + EchoRequest recv_request; + if (state.range(0) > 0) { + send_request.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + response_rw.Read(&recv_request, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + request_rw->Write(send_request, tag(1)); + while (true) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + if (t == tag(0)) { + response_rw.Read(&recv_request, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + request_rw->WritesDone(tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +template +static void BM_PumpStreamServerToClient(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + { + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_response.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + request_rw->Read(&recv_response, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + response_rw.Write(send_response, tag(1)); + while (true) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + if (t == tag(0)) { + request_rw->Read(&recv_response, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + response_rw.Finish(Status::OK, tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +/******************************************************************************* + * CONFIGURATIONS + */ + +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) + ->Range(0, 128 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) + ->Range(0, 128 * 1024 * 1024); + +} // namespace testing +} // namespace grpc + +BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc new file mode 100644 index 00000000000..5011f06368e --- /dev/null +++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc @@ -0,0 +1,219 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Benchmark gRPC end2end in various configurations */ + +#include "src/core/lib/profiling/timers.h" +#include "src/cpp/client/create_channel_internal.h" +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" +extern "C" { +#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" +#include "src/core/ext/transport/chttp2/transport/internal.h" +#include "test/core/util/trickle_endpoint.h" +} + +namespace grpc { +namespace testing { + +static void* tag(intptr_t x) { return reinterpret_cast(x); } + +class TrickledCHTTP2 : public EndpointPairFixture { + public: + TrickledCHTTP2(Service* service, size_t megabits_per_second) + : EndpointPairFixture(service, MakeEndpoints(megabits_per_second)) {} + + void AddToLabel(std::ostream& out, benchmark::State& state) { + out << " writes/iter:" + << ((double)stats_.num_writes / (double)state.iterations()) + << " cli_transport_stalls/iter:" + << ((double) + client_stats_.streams_stalled_due_to_transport_flow_control / + (double)state.iterations()) + << " cli_stream_stalls/iter:" + << ((double)client_stats_.streams_stalled_due_to_stream_flow_control / + (double)state.iterations()) + << " svr_transport_stalls/iter:" + << ((double) + server_stats_.streams_stalled_due_to_transport_flow_control / + (double)state.iterations()) + << " svr_stream_stalls/iter:" + << ((double)server_stats_.streams_stalled_due_to_stream_flow_control / + (double)state.iterations()); + } + + void Step() { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + size_t client_backlog = + grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.client); + size_t server_backlog = + grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server); + grpc_exec_ctx_finish(&exec_ctx); + + UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_, + client_backlog); + UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_, + server_backlog); + } + + private: + grpc_passthru_endpoint_stats stats_; + struct Stats { + int streams_stalled_due_to_stream_flow_control = 0; + int streams_stalled_due_to_transport_flow_control = 0; + }; + Stats client_stats_; + Stats server_stats_; + + grpc_endpoint_pair MakeEndpoints(size_t kilobits) { + grpc_endpoint_pair p; + grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(), + &stats_); + double bytes_per_second = 125.0 * kilobits; + p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second); + p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second); + return p; + } + + void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) { + if (backlog == 0) { + if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) { + s->streams_stalled_due_to_stream_flow_control++; + } + if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) { + s->streams_stalled_due_to_transport_flow_control++; + } + } + } +}; + +// force library initialization +auto& force_library_initialization = Library::get(); + +static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) { + while (true) { + switch (fixture->cq()->AsyncNext( + t, ok, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_micros(100, GPR_TIMESPAN)))) { + case CompletionQueue::TIMEOUT: + fixture->Step(); + break; + case CompletionQueue::SHUTDOWN: + GPR_ASSERT(false); + break; + case CompletionQueue::GOT_EVENT: + return; + } + } +} + +static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture( + new TrickledCHTTP2(&service, state.range(1))); + { + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_response.set_message(std::string(state.range(0), 'a')); + } + Status recv_status; + ServerContext svr_ctx; + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + int need_tags = (1 << 0) | (1 << 1); + void* t; + bool ok; + while (need_tags) { + TrickleCQNext(fixture.get(), &t, &ok); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + request_rw->Read(&recv_response, tag(0)); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + response_rw.Write(send_response, tag(1)); + while (true) { + TrickleCQNext(fixture.get(), &t, &ok); + if (t == tag(0)) { + request_rw->Read(&recv_response, tag(0)); + } else if (t == tag(1)) { + break; + } else { + GPR_ASSERT(false); + } + } + } + response_rw.Finish(Status::OK, tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + TrickleCQNext(fixture.get(), &t, &ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + } + fixture->Finish(state); + fixture.reset(); + state.SetBytesProcessed(state.range(0) * state.iterations()); +} + +/******************************************************************************* + * CONFIGURATIONS + */ + +static void TrickleArgs(benchmark::internal::Benchmark* b) { + for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { + for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) { + double expected_time = + static_cast(14 + i) / (125.0 * static_cast(j)); + if (expected_time > 0.01) continue; + b->Args({i, j}); + } + } +} + +BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs); +} +} + +BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc new file mode 100644 index 00000000000..e51d272b104 --- /dev/null +++ b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc @@ -0,0 +1,205 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Benchmark gRPC end2end in various configurations */ + +#include + +#include "src/core/lib/profiling/timers.h" +#include "src/cpp/client/create_channel_internal.h" +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/cpp/microbenchmarks/fullstack_context_mutators.h" +#include "test/cpp/microbenchmarks/fullstack_fixtures.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +namespace grpc { +namespace testing { + +// force library initialization +auto& force_library_initialization = Library::get(); + +/******************************************************************************* + * BENCHMARKING KERNELS + */ + +static void* tag(intptr_t x) { return reinterpret_cast(x); } + +template +static void BM_UnaryPingPong(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new Fixture(&service)); + EchoRequest send_request; + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_request.set_message(std::string(state.range(0), 'a')); + } + if (state.range(1) > 0) { + send_response.set_message(std::string(state.range(1), 'a')); + } + Status recv_status; + struct ServerEnv { + ServerContext ctx; + EchoRequest recv_request; + grpc::ServerAsyncResponseWriter response_writer; + ServerEnv() : response_writer(&ctx) {} + }; + uint8_t server_env_buffer[2 * sizeof(ServerEnv)]; + ServerEnv* server_env[2] = { + reinterpret_cast(server_env_buffer), + reinterpret_cast(server_env_buffer + sizeof(ServerEnv))}; + new (server_env[0]) ServerEnv; + new (server_env[1]) ServerEnv; + service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request, + &server_env[0]->response_writer, fixture->cq(), + fixture->cq(), tag(0)); + service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request, + &server_env[1]->response_writer, fixture->cq(), + fixture->cq(), tag(1)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + recv_response.Clear(); + ClientContext cli_ctx; + ClientContextMutator cli_ctx_mut(&cli_ctx); + std::unique_ptr> response_reader( + stub->AsyncEcho(&cli_ctx, send_request, fixture->cq())); + void* t; + bool ok; + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + GPR_ASSERT(t == tag(0) || t == tag(1)); + intptr_t slot = reinterpret_cast(t); + ServerEnv* senv = server_env[slot]; + ServerContextMutator svr_ctx_mut(&senv->ctx); + senv->response_writer.Finish(send_response, Status::OK, tag(3)); + response_reader->Finish(&recv_response, &recv_status, tag(4)); + for (int i = (1 << 3) | (1 << 4); i != 0;) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int tagnum = (int)reinterpret_cast(t); + GPR_ASSERT(i & (1 << tagnum)); + i -= 1 << tagnum; + } + GPR_ASSERT(recv_status.ok()); + + senv->~ServerEnv(); + senv = new (senv) ServerEnv(); + service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer, + fixture->cq(), fixture->cq(), tag(slot)); + } + fixture->Finish(state); + fixture.reset(); + server_env[0]->~ServerEnv(); + server_env[1]->~ServerEnv(); + state.SetBytesProcessed(state.range(0) * state.iterations() + + state.range(1) * state.iterations()); +} + +/******************************************************************************* + * CONFIGURATIONS + */ + +static void SweepSizesArgs(benchmark::internal::Benchmark* b) { + b->Args({0, 0}); + for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { + b->Args({i, 0}); + b->Args({0, i}); + b->Args({i, i}); + } +} + +BENCHMARK_TEMPLATE(BM_UnaryPingPong, TCP, NoOpMutator, NoOpMutator) + ->Apply(SweepSizesArgs); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, UDS, NoOpMutator, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, SockPair, NoOpMutator, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, NoOpMutator) + ->Apply(SweepSizesArgs); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, + NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 2>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 2>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 2>, + NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, + Client_AddMetadata, 1>, NoOpMutator) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 1>) + ->Args({0, 0}); +BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, + Server_AddInitialMetadata, 100>) + ->Args({0, 0}); + +} // namespace testing +} // namespace grpc + +BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_metadata.cc b/test/cpp/microbenchmarks/bm_metadata.cc index f4686908342..ee3dec2bce0 100644 --- a/test/cpp/microbenchmarks/bm_metadata.cc +++ b/test/cpp/microbenchmarks/bm_metadata.cc @@ -42,25 +42,26 @@ extern "C" { #include "src/core/lib/transport/transport.h" } +#include "test/cpp/microbenchmarks/helpers.h" #include "third_party/benchmark/include/benchmark/benchmark.h" -static class InitializeStuff { - public: - InitializeStuff() { grpc_init(); } - ~InitializeStuff() { grpc_shutdown(); } -} initialize_stuff; +auto& force_library_initialization = Library::get(); static void BM_SliceFromStatic(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { benchmark::DoNotOptimize(grpc_slice_from_static_string("abc")); } + track_counters.Finish(state); } BENCHMARK(BM_SliceFromStatic); static void BM_SliceFromCopied(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { grpc_slice_unref(grpc_slice_from_copied_string("abc")); } + track_counters.Finish(state); } BENCHMARK(BM_SliceFromCopied); @@ -78,38 +79,47 @@ static void BM_SliceFromStreamOwnedBuffer(benchmark::State& state) { BENCHMARK(BM_SliceFromStreamOwnedBuffer); static void BM_SliceIntern(benchmark::State& state) { + TrackCounters track_counters; gpr_slice slice = grpc_slice_from_static_string("abc"); while (state.KeepRunning()) { grpc_slice_unref(grpc_slice_intern(slice)); } + track_counters.Finish(state); } BENCHMARK(BM_SliceIntern); static void BM_SliceReIntern(benchmark::State& state) { + TrackCounters track_counters; gpr_slice slice = grpc_slice_intern(grpc_slice_from_static_string("abc")); while (state.KeepRunning()) { grpc_slice_unref(grpc_slice_intern(slice)); } grpc_slice_unref(slice); + track_counters.Finish(state); } BENCHMARK(BM_SliceReIntern); static void BM_SliceInternStaticMetadata(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { grpc_slice_intern(GRPC_MDSTR_GZIP); } + track_counters.Finish(state); } BENCHMARK(BM_SliceInternStaticMetadata); static void BM_SliceInternEqualToStaticMetadata(benchmark::State& state) { + TrackCounters track_counters; gpr_slice slice = grpc_slice_from_static_string("gzip"); while (state.KeepRunning()) { grpc_slice_intern(slice); } + track_counters.Finish(state); } BENCHMARK(BM_SliceInternEqualToStaticMetadata); static void BM_MetadataFromNonInternedSlices(benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_from_static_string("key"); gpr_slice v = grpc_slice_from_static_string("value"); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -117,10 +127,12 @@ static void BM_MetadataFromNonInternedSlices(benchmark::State& state) { GRPC_MDELEM_UNREF(&exec_ctx, grpc_mdelem_create(&exec_ctx, k, v, NULL)); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromNonInternedSlices); static void BM_MetadataFromInternedSlices(benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value")); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -130,11 +142,13 @@ static void BM_MetadataFromInternedSlices(benchmark::State& state) { grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); grpc_slice_unref(v); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedSlices); static void BM_MetadataFromInternedSlicesAlreadyInIndex( benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value")); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -146,10 +160,12 @@ static void BM_MetadataFromInternedSlicesAlreadyInIndex( grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); grpc_slice_unref(v); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedSlicesAlreadyInIndex); static void BM_MetadataFromInternedKey(benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_from_static_string("value"); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -158,11 +174,13 @@ static void BM_MetadataFromInternedKey(benchmark::State& state) { } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedKey); static void BM_MetadataFromNonInternedSlicesWithBackingStore( benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_from_static_string("key"); gpr_slice v = grpc_slice_from_static_string("value"); char backing_store[sizeof(grpc_mdelem_data)]; @@ -174,11 +192,13 @@ static void BM_MetadataFromNonInternedSlicesWithBackingStore( reinterpret_cast(backing_store))); } grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromNonInternedSlicesWithBackingStore); static void BM_MetadataFromInternedSlicesWithBackingStore( benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value")); char backing_store[sizeof(grpc_mdelem_data)]; @@ -192,11 +212,13 @@ static void BM_MetadataFromInternedSlicesWithBackingStore( grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); grpc_slice_unref(v); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedSlicesWithBackingStore); static void BM_MetadataFromInternedKeyWithBackingStore( benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); gpr_slice v = grpc_slice_from_static_string("value"); char backing_store[sizeof(grpc_mdelem_data)]; @@ -209,10 +231,12 @@ static void BM_MetadataFromInternedKeyWithBackingStore( } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromInternedKeyWithBackingStore); static void BM_MetadataFromStaticMetadataStrings(benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = GRPC_MDSTR_STATUS; gpr_slice v = GRPC_MDSTR_200; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -221,11 +245,13 @@ static void BM_MetadataFromStaticMetadataStrings(benchmark::State& state) { } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromStaticMetadataStrings); static void BM_MetadataFromStaticMetadataStringsNotIndexed( benchmark::State& state) { + TrackCounters track_counters; gpr_slice k = GRPC_MDSTR_STATUS; gpr_slice v = GRPC_MDSTR_GZIP; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -234,10 +260,12 @@ static void BM_MetadataFromStaticMetadataStringsNotIndexed( } grpc_exec_ctx_finish(&exec_ctx); grpc_slice_unref(k); + track_counters.Finish(state); } BENCHMARK(BM_MetadataFromStaticMetadataStringsNotIndexed); static void BM_MetadataRefUnrefExternal(benchmark::State& state) { + TrackCounters track_counters; char backing_store[sizeof(grpc_mdelem_data)]; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_mdelem el = @@ -249,10 +277,12 @@ static void BM_MetadataRefUnrefExternal(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefExternal); static void BM_MetadataRefUnrefInterned(benchmark::State& state) { + TrackCounters track_counters; char backing_store[sizeof(grpc_mdelem_data)]; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key")); @@ -266,10 +296,12 @@ static void BM_MetadataRefUnrefInterned(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefInterned); static void BM_MetadataRefUnrefAllocated(benchmark::State& state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_mdelem el = grpc_mdelem_create(&exec_ctx, grpc_slice_from_static_string("a"), @@ -279,10 +311,12 @@ static void BM_MetadataRefUnrefAllocated(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefAllocated); static void BM_MetadataRefUnrefStatic(benchmark::State& state) { + TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_mdelem el = grpc_mdelem_create(&exec_ctx, GRPC_MDSTR_STATUS, GRPC_MDSTR_200, NULL); @@ -291,6 +325,7 @@ static void BM_MetadataRefUnrefStatic(benchmark::State& state) { } GRPC_MDELEM_UNREF(&exec_ctx, el); grpc_exec_ctx_finish(&exec_ctx); + track_counters.Finish(state); } BENCHMARK(BM_MetadataRefUnrefStatic); diff --git a/test/cpp/microbenchmarks/fullstack_context_mutators.h b/test/cpp/microbenchmarks/fullstack_context_mutators.h new file mode 100644 index 00000000000..676f9aa1cc6 --- /dev/null +++ b/test/cpp/microbenchmarks/fullstack_context_mutators.h @@ -0,0 +1,158 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef TEST_CPP_MICROBENCHMARKS_FULLSTACK_CONTEXT_MUTATORS_H +#define TEST_CPP_MICROBENCHMARKS_FULLSTACK_CONTEXT_MUTATORS_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test/cpp/microbenchmarks/helpers.h" + +namespace grpc { +namespace testing { + +/******************************************************************************* + * CONTEXT MUTATORS + */ + +static const int kPregenerateKeyCount = 100000; + +template +auto MakeVector(size_t length, F f) -> std::vector { + std::vector out; + out.reserve(length); + for (size_t i = 0; i < length; i++) { + out.push_back(f()); + } + return out; +} + +class NoOpMutator { + public: + template + NoOpMutator(ContextType* context) {} +}; + +template +class RandomBinaryMetadata { + public: + static const grpc::string& Key() { return kKey; } + + static const grpc::string& Value() { + return kValues[rand() % kValues.size()]; + } + + private: + static const grpc::string kKey; + static const std::vector kValues; + + static grpc::string GenerateOneString() { + grpc::string s; + s.reserve(length + 1); + for (int i = 0; i < length; i++) { + s += (char)rand(); + } + return s; + } +}; + +template +class RandomAsciiMetadata { + public: + static const grpc::string& Key() { return kKey; } + + static const grpc::string& Value() { + return kValues[rand() % kValues.size()]; + } + + private: + static const grpc::string kKey; + static const std::vector kValues; + + static grpc::string GenerateOneString() { + grpc::string s; + s.reserve(length + 1); + for (int i = 0; i < length; i++) { + s += (char)(rand() % 26 + 'a'); + } + return s; + } +}; + +template +class Client_AddMetadata : public NoOpMutator { + public: + Client_AddMetadata(ClientContext* context) : NoOpMutator(context) { + for (int i = 0; i < kNumKeys; i++) { + context->AddMetadata(Generator::Key(), Generator::Value()); + } + } +}; + +template +class Server_AddInitialMetadata : public NoOpMutator { + public: + Server_AddInitialMetadata(ServerContext* context) : NoOpMutator(context) { + for (int i = 0; i < kNumKeys; i++) { + context->AddInitialMetadata(Generator::Key(), Generator::Value()); + } + } +}; + +// static initialization + +template +const grpc::string RandomBinaryMetadata::kKey = "foo-bin"; + +template +const std::vector RandomBinaryMetadata::kValues = + MakeVector(kPregenerateKeyCount, GenerateOneString); + +template +const grpc::string RandomAsciiMetadata::kKey = "foo"; + +template +const std::vector RandomAsciiMetadata::kValues = + MakeVector(kPregenerateKeyCount, GenerateOneString); + +} // namespace testing +} // namespace grpc + +#endif diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h new file mode 100644 index 00000000000..dc297010599 --- /dev/null +++ b/test/cpp/microbenchmarks/fullstack_fixtures.h @@ -0,0 +1,244 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H +#define TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H + +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/endpoint.h" +#include "src/core/lib/iomgr/endpoint_pair.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/iomgr/tcp_posix.h" +#include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/completion_queue.h" +#include "src/core/lib/surface/server.h" +#include "test/core/util/passthru_endpoint.h" +#include "test/core/util/port.h" +} + +#include "test/cpp/microbenchmarks/helpers.h" + +namespace grpc { +namespace testing { + +static void ApplyCommonServerBuilderConfig(ServerBuilder* b) { + b->SetMaxReceiveMessageSize(INT_MAX); + b->SetMaxSendMessageSize(INT_MAX); +} + +static void ApplyCommonChannelArguments(ChannelArguments* c) { + c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX); + c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX); +} + +class BaseFixture : public TrackCounters {}; + +class FullstackFixture : public BaseFixture { + public: + FullstackFixture(Service* service, const grpc::string& address) { + ServerBuilder b; + b.AddListeningPort(address, InsecureServerCredentials()); + cq_ = b.AddCompletionQueue(true); + b.RegisterService(service); + ApplyCommonServerBuilderConfig(&b); + server_ = b.BuildAndStart(); + ChannelArguments args; + ApplyCommonChannelArguments(&args); + channel_ = CreateCustomChannel(address, InsecureChannelCredentials(), args); + } + + virtual ~FullstackFixture() { + server_->Shutdown(); + cq_->Shutdown(); + void* tag; + bool ok; + while (cq_->Next(&tag, &ok)) { + } + } + + ServerCompletionQueue* cq() { return cq_.get(); } + std::shared_ptr channel() { return channel_; } + + private: + std::unique_ptr server_; + std::unique_ptr cq_; + std::shared_ptr channel_; +}; + +class TCP : public FullstackFixture { + public: + TCP(Service* service) : FullstackFixture(service, MakeAddress()) {} + + private: + static grpc::string MakeAddress() { + int port = grpc_pick_unused_port_or_die(); + std::stringstream addr; + addr << "localhost:" << port; + return addr.str(); + } +}; + +class UDS : public FullstackFixture { + public: + UDS(Service* service) : FullstackFixture(service, MakeAddress()) {} + + private: + static grpc::string MakeAddress() { + int port = grpc_pick_unused_port_or_die(); // just for a unique id - not a + // real port + std::stringstream addr; + addr << "unix:/tmp/bm_fullstack." << port; + return addr.str(); + } +}; + +class EndpointPairFixture : public BaseFixture { + public: + EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints) + : endpoint_pair_(endpoints) { + ServerBuilder b; + cq_ = b.AddCompletionQueue(true); + b.RegisterService(service); + ApplyCommonServerBuilderConfig(&b); + server_ = b.BuildAndStart(); + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + + /* add server endpoint to server_ + * */ + { + const grpc_channel_args* server_args = + grpc_server_get_channel_args(server_->c_server()); + server_transport_ = grpc_create_chttp2_transport( + &exec_ctx, server_args, endpoints.server, 0 /* is_client */); + + grpc_pollset** pollsets; + size_t num_pollsets = 0; + grpc_server_get_pollsets(server_->c_server(), &pollsets, &num_pollsets); + + for (size_t i = 0; i < num_pollsets; i++) { + grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]); + } + + grpc_server_setup_transport(&exec_ctx, server_->c_server(), + server_transport_, NULL, server_args); + grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL); + } + + /* create channel */ + { + ChannelArguments args; + args.SetString(GRPC_ARG_DEFAULT_AUTHORITY, "test.authority"); + ApplyCommonChannelArguments(&args); + + grpc_channel_args c_args = args.c_channel_args(); + client_transport_ = + grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1); + GPR_ASSERT(client_transport_); + grpc_channel* channel = + grpc_channel_create(&exec_ctx, "target", &c_args, + GRPC_CLIENT_DIRECT_CHANNEL, client_transport_); + grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL); + + channel_ = CreateChannelInternal("", channel); + } + + grpc_exec_ctx_finish(&exec_ctx); + } + + virtual ~EndpointPairFixture() { + server_->Shutdown(); + cq_->Shutdown(); + void* tag; + bool ok; + while (cq_->Next(&tag, &ok)) { + } + } + + ServerCompletionQueue* cq() { return cq_.get(); } + std::shared_ptr channel() { return channel_; } + + protected: + grpc_endpoint_pair endpoint_pair_; + grpc_transport* client_transport_; + grpc_transport* server_transport_; + + private: + std::unique_ptr server_; + std::unique_ptr cq_; + std::shared_ptr channel_; +}; + +class SockPair : public EndpointPairFixture { + public: + SockPair(Service* service) + : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair( + "test", Library::get().rq(), 8192)) {} +}; + +class InProcessCHTTP2 : public EndpointPairFixture { + public: + InProcessCHTTP2(Service* service) + : EndpointPairFixture(service, MakeEndpoints()) {} + + void AddToLabel(std::ostream& out, benchmark::State& state) { + EndpointPairFixture::AddToLabel(out, state); + out << " writes/iter:" + << ((double)stats_.num_writes / (double)state.iterations()); + } + + private: + grpc_passthru_endpoint_stats stats_; + + grpc_endpoint_pair MakeEndpoints() { + grpc_endpoint_pair p; + grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(), + &stats_); + return p; + } +}; + +} // namespace testing +} // namespace grpc + +#endif diff --git a/test/cpp/microbenchmarks/helpers.cc b/test/cpp/microbenchmarks/helpers.cc new file mode 100644 index 00000000000..947e81ffd83 --- /dev/null +++ b/test/cpp/microbenchmarks/helpers.cc @@ -0,0 +1,65 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "test/cpp/microbenchmarks/helpers.h" + +void TrackCounters::Finish(benchmark::State &state) { + std::ostringstream out; + AddToLabel(out, state); + auto label = out.str(); + if (label.length() && label[0] == ' ') { + label = label.substr(1); + } + state.SetLabel(label); +} + +void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) { +#ifdef GPR_LOW_LEVEL_COUNTERS + out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) - + mu_locks_at_start_) / + (double)state.iterations()) + << " atm_cas/iter:" + << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) - + atm_cas_at_start_) / + (double)state.iterations()) + << " atm_add/iter:" + << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - + atm_add_at_start_) / + (double)state.iterations()); +#endif + grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); + out << " allocs/iter:" + << ((double)(counters_at_end.total_allocs_absolute - + counters_at_start_.total_allocs_absolute) / + (double)state.iterations()); +} diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h new file mode 100644 index 00000000000..42a8fbaf0bf --- /dev/null +++ b/test/cpp/microbenchmarks/helpers.h @@ -0,0 +1,91 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef TEST_CPP_MICROBENCHMARKS_COUNTERS_H +#define TEST_CPP_MICROBENCHMARKS_COUNTERS_H + +#include + +extern "C" { +#include +#include "test/core/util/memory_counters.h" +} + +#include +#include "third_party/benchmark/include/benchmark/benchmark.h" + +class Library { + public: + static Library& get() { + static Library lib; + return lib; + } + + grpc_resource_quota* rq() { return rq_; } + + private: + Library() { + grpc_memory_counters_init(); + init_lib_.init(); + rq_ = grpc_resource_quota_create("bm"); + } + + ~Library() { init_lib_.shutdown(); } + + grpc::internal::GrpcLibrary init_lib_; + grpc_resource_quota* rq_; +}; + +#ifdef GPR_LOW_LEVEL_COUNTERS +extern "C" gpr_atm gpr_mu_locks; +extern "C" gpr_atm gpr_counter_atm_cas; +extern "C" gpr_atm gpr_counter_atm_add; +#endif + +class TrackCounters { + public: + virtual void Finish(benchmark::State& state); + virtual void AddToLabel(std::ostream& out, benchmark::State& state); + + private: +#ifdef GPR_LOW_LEVEL_COUNTERS + const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks); + const size_t atm_cas_at_start_ = + gpr_atm_no_barrier_load(&gpr_counter_atm_cas); + const size_t atm_add_at_start_ = + gpr_atm_no_barrier_load(&gpr_counter_atm_add); +#endif + grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot(); +}; + +#endif diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 0f63612ed27..8a31a6e0552 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2420,6 +2420,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2440,6 +2441,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2460,6 +2462,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2480,6 +2483,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2500,6 +2504,7 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -2520,14 +2525,15 @@ "grpc", "grpc++", "grpc++_test_util", + "grpc_benchmark", "grpc_test_util" ], "headers": [], "is_filegroup": false, "language": "c++", - "name": "bm_fullstack", + "name": "bm_fullstack_streaming_ping_pong", "src": [ - "test/cpp/microbenchmarks/bm_fullstack.cc" + "test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc" ], "third_party": false, "type": "target" @@ -2538,6 +2544,70 @@ "gpr", "gpr_test_util", "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_fullstack_streaming_pump", + "src": [ + "test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_fullstack_trickle", + "src": [ + "test/cpp/microbenchmarks/bm_fullstack_trickle.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_fullstack_unary_ping_pong", + "src": [ + "test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc_benchmark", "grpc_test_util" ], "headers": [], @@ -5741,6 +5811,30 @@ "third_party": false, "type": "lib" }, + { + "deps": [ + "benchmark", + "grpc", + "grpc++", + "grpc_test_util" + ], + "headers": [ + "test/cpp/microbenchmarks/fullstack_context_mutators.h", + "test/cpp/microbenchmarks/fullstack_fixtures.h", + "test/cpp/microbenchmarks/helpers.h" + ], + "is_filegroup": false, + "language": "c++", + "name": "grpc_benchmark", + "src": [ + "test/cpp/microbenchmarks/fullstack_context_mutators.h", + "test/cpp/microbenchmarks/fullstack_fixtures.h", + "test/cpp/microbenchmarks/helpers.cc", + "test/cpp/microbenchmarks/helpers.h" + ], + "third_party": false, + "type": "lib" + }, { "deps": [ "grpc++", @@ -7946,7 +8040,7 @@ "headers": [ "test/core/end2end/cq_verifier.h", "test/core/end2end/fake_resolver.h", - "test/core/end2end/fixtures/http_proxy.h", + "test/core/end2end/fixtures/http_proxy_fixture.h", "test/core/end2end/fixtures/proxy.h", "test/core/iomgr/endpoint_tests.h", "test/core/util/debugger_macros.h", @@ -7968,8 +8062,8 @@ "test/core/end2end/cq_verifier.h", "test/core/end2end/fake_resolver.c", "test/core/end2end/fake_resolver.h", - "test/core/end2end/fixtures/http_proxy.c", - "test/core/end2end/fixtures/http_proxy.h", + "test/core/end2end/fixtures/http_proxy_fixture.c", + "test/core/end2end/fixtures/http_proxy_fixture.h", "test/core/end2end/fixtures/proxy.c", "test/core/end2end/fixtures/proxy.h", "test/core/iomgr/endpoint_tests.c", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 46c83384019..402dabc554d 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2712,7 +2712,88 @@ "flaky": false, "gtest": false, "language": "c++", - "name": "bm_fullstack", + "name": "bm_fullstack_streaming_ping_pong", + "platforms": [ + "linux", + "mac", + "posix" + ], + "timeout_seconds": 1200 + }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "excluded_poll_engines": [ + "poll", + "poll-cv" + ], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_fullstack_streaming_pump", + "platforms": [ + "linux", + "mac", + "posix" + ], + "timeout_seconds": 1200 + }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "excluded_poll_engines": [ + "poll", + "poll-cv" + ], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_fullstack_trickle", + "platforms": [ + "linux", + "mac", + "posix" + ], + "timeout_seconds": 1200 + }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "excluded_poll_engines": [ + "poll", + "poll-cv" + ], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_fullstack_unary_ping_pong", "platforms": [ "linux", "mac", diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 3a9461ecd3b..4fb1d5fc7c2 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -199,13 +199,17 @@ argp.add_argument('-c', '--collect', default=sorted(collectors.keys()), help='Which collectors should be run against each benchmark') argp.add_argument('-b', '--benchmarks', - default=['bm_fullstack', + default=['bm_fullstack_unary_ping_pong', + 'bm_fullstack_streaming_ping_pong', + 'bm_fullstack_streaming_pump', 'bm_closure', 'bm_cq', 'bm_call_create', 'bm_error', 'bm_chttp2_hpack', - 'bm_metadata'], + 'bm_metadata', + 'bm_fullstack_trickle', + ], nargs='+', type=str, help='Which microbenchmarks should be run') diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index d08ceb68287..e7c9fb71f33 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -182,7 +182,7 @@ - + @@ -317,7 +317,7 @@ - + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 3beaa80994f..3d36948aaef 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -22,7 +22,7 @@ test\core\end2end - + test\core\end2end\fixtures @@ -518,7 +518,7 @@ test\core\end2end - + test\core\end2end\fixtures diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj index 5a58cae83ff..af13acef455 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj @@ -149,7 +149,7 @@ - + @@ -168,7 +168,7 @@ - + diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters index 88c875cb011..4da043ea908 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters @@ -7,7 +7,7 @@ test\core\end2end - + test\core\end2end\fixtures @@ -54,7 +54,7 @@ test\core\end2end - + test\core\end2end\fixtures diff --git a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj new file mode 100644 index 00000000000..58586f0cb8d --- /dev/null +++ b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj @@ -0,0 +1,206 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {31FCED31-7D88-BE3D-2D61-0840F08E0850} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + grpc_benchmark + static + Debug + static + Debug + + + grpc_benchmark + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Windows + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Windows + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Windows + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Windows + true + false + true + true + + + + + + + + + + + + + + + {07978586-E47C-8709-A63E-895FBF3C3C7D} + + + {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters new file mode 100644 index 00000000000..8e865bcc1bb --- /dev/null +++ b/vsprojects/vcxproj/test/grpc_benchmark/grpc_benchmark.vcxproj.filters @@ -0,0 +1,32 @@ + + + + + test\cpp\microbenchmarks + + + + + test\cpp\microbenchmarks + + + test\cpp\microbenchmarks + + + test\cpp\microbenchmarks + + + + + + {46d1162d-13b8-d144-8b76-77a6d982a9f1} + + + {1d2b47d7-8fc3-a5b6-cc85-47e31600e9d7} + + + {2a1ac913-6c7b-fbd2-6e8f-1215e92b4af8} + + + + From be2807de9269cfb552eb7aea1b6078bbe47ac5cc Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 9 Mar 2017 10:15:41 +0100 Subject: [PATCH 083/188] Add newest dotnet SDK to C# dockerfile --- templates/tools/dockerfile/csharp_dotnetcli_deps.include | 8 ++++++-- .../interoptest/grpc_interop_csharpcoreclr/Dockerfile | 8 ++++++-- tools/dockerfile/test/csharp_coreclr_x64/Dockerfile | 8 ++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/templates/tools/dockerfile/csharp_dotnetcli_deps.include b/templates/tools/dockerfile/csharp_dotnetcli_deps.include index 430f3fa3f22..058ce15fb52 100644 --- a/templates/tools/dockerfile/csharp_dotnetcli_deps.include +++ b/templates/tools/dockerfile/csharp_dotnetcli_deps.include @@ -1,7 +1,11 @@ # Install dotnet SDK based on https://www.microsoft.com/net/core#debian RUN apt-get update && apt-get install -y curl libunwind8 gettext -RUN curl -sSL -o dotnet.tar.gz https://go.microsoft.com/fwlink/?LinkID=809130 -RUN mkdir -p /opt/dotnet && tar zxf dotnet.tar.gz -C /opt/dotnet +# dotnet-dev-1.0.0-preview2-003121 +RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=809130 +RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet +# dotnet-dev-1.0.1 +RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453 +RUN mkdir -p /opt/dotnet && tar zxf dotnet101.tar.gz -C /opt/dotnet RUN ln -s /opt/dotnet/dotnet /usr/local/bin # Trigger the population of the local package cache diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile index 91639829dcd..c26c9a2826e 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile @@ -99,8 +99,12 @@ RUN nuget update -self # Install dotnet SDK based on https://www.microsoft.com/net/core#debian RUN apt-get update && apt-get install -y curl libunwind8 gettext -RUN curl -sSL -o dotnet.tar.gz https://go.microsoft.com/fwlink/?LinkID=809130 -RUN mkdir -p /opt/dotnet && tar zxf dotnet.tar.gz -C /opt/dotnet +# dotnet-dev-1.0.0-preview2-003121 +RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=809130 +RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet +# dotnet-dev-1.0.1 +RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453 +RUN mkdir -p /opt/dotnet && tar zxf dotnet101.tar.gz -C /opt/dotnet RUN ln -s /opt/dotnet/dotnet /usr/local/bin # Trigger the population of the local package cache diff --git a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile index 91639829dcd..c26c9a2826e 100644 --- a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile @@ -99,8 +99,12 @@ RUN nuget update -self # Install dotnet SDK based on https://www.microsoft.com/net/core#debian RUN apt-get update && apt-get install -y curl libunwind8 gettext -RUN curl -sSL -o dotnet.tar.gz https://go.microsoft.com/fwlink/?LinkID=809130 -RUN mkdir -p /opt/dotnet && tar zxf dotnet.tar.gz -C /opt/dotnet +# dotnet-dev-1.0.0-preview2-003121 +RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=809130 +RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet +# dotnet-dev-1.0.1 +RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453 +RUN mkdir -p /opt/dotnet && tar zxf dotnet101.tar.gz -C /opt/dotnet RUN ln -s /opt/dotnet/dotnet /usr/local/bin # Trigger the population of the local package cache From f5ac0b95f240c50909fc0d8b1f16960c1b34828b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 9 Mar 2017 16:55:06 +0100 Subject: [PATCH 084/188] Add global.json file to pick dotnet core SDK version --- examples/csharp/helloworld-from-cli/global.json | 5 +++++ src/csharp/global.json | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 examples/csharp/helloworld-from-cli/global.json create mode 100644 src/csharp/global.json diff --git a/examples/csharp/helloworld-from-cli/global.json b/examples/csharp/helloworld-from-cli/global.json new file mode 100644 index 00000000000..32ff399ef94 --- /dev/null +++ b/examples/csharp/helloworld-from-cli/global.json @@ -0,0 +1,5 @@ +{ + "sdk": { + "version": "1.0.0-preview2-003121" + } +} \ No newline at end of file diff --git a/src/csharp/global.json b/src/csharp/global.json new file mode 100644 index 00000000000..32ff399ef94 --- /dev/null +++ b/src/csharp/global.json @@ -0,0 +1,5 @@ +{ + "sdk": { + "version": "1.0.0-preview2-003121" + } +} \ No newline at end of file From 77108238509e2c8c236541a0ef8cb7a9d7a34e0d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Mar 2017 10:04:50 -0800 Subject: [PATCH 085/188] Add speed of light measures --- test/cpp/microbenchmarks/bm_pollset.cc | 87 +++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 2 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc index 9d7fac39c77..da06eae5078 100644 --- a/test/cpp/microbenchmarks/bm_pollset.cc +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -35,10 +35,13 @@ #include #include +#include +#include extern "C" { #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/pollset.h" +#include "src/core/lib/iomgr/port.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" } @@ -47,6 +50,12 @@ extern "C" { #include +#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL +#include +#include +#include +#endif + auto& force_library_initialization = Library::get(); static void shutdown_ps(grpc_exec_ctx* exec_ctx, void* ps, grpc_error* error) { @@ -76,6 +85,46 @@ static void BM_CreateDestroyPollset(benchmark::State& state) { } BENCHMARK(BM_CreateDestroyPollset); +#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL +static void BM_PollEmptyPollset_SpeedOfLight(benchmark::State& state) { + // equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage + // what the speed of light would be if we abstracted perfectly + TrackCounters track_counters; + int epfd = epoll_create1(0); + GPR_ASSERT(epfd != -1); + size_t nev = state.range(0); + size_t nfd = state.range(1); + epoll_event* ev = new epoll_event[nev]; + std::vector fds; + for (size_t i = 0; i < nfd; i++) { + fds.push_back(eventfd(0, 0)); + epoll_event ev; + ev.events = EPOLLIN; + epoll_ctl(epfd, EPOLL_CTL_ADD, fds.back(), &ev); + } + while (state.KeepRunning()) { + epoll_wait(epfd, ev, nev, 0); + } + for (auto fd : fds) { + close(fd); + } + close(epfd); + delete[] ev; + track_counters.Finish(state); +} +BENCHMARK(BM_PollEmptyPollset_SpeedOfLight) + ->Args({1, 0}) + ->Args({1, 1}) + ->Args({1, 10}) + ->Args({1, 100}) + ->Args({1, 1000}) + ->Args({1, 10000}) + ->Args({1, 100000}) + ->Args({10, 1}) + ->Args({100, 1}) + ->Args({1000, 1}); +#endif + static void BM_PollEmptyPollset(benchmark::State& state) { TrackCounters track_counters; size_t ps_sz = grpc_pollset_size(); @@ -115,7 +164,41 @@ grpc_closure* MakeClosure(F f, grpc_closure_scheduler* scheduler) { return new C(f, scheduler); } -static void BM_PollAfterWakeup(benchmark::State& state) { +#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL +static void BM_SingleThreadPollOneFd_SpeedOfLight(benchmark::State& state) { + // equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage + // what the speed of light would be if we abstracted perfectly + TrackCounters track_counters; + int epfd = epoll_create1(0); + GPR_ASSERT(epfd != -1); + epoll_event ev[100]; + int fd = eventfd(0, EFD_NONBLOCK); + ev[0].events = EPOLLIN; + epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev[0]); + while (state.KeepRunning()) { + int err; + do { + err = eventfd_write(fd, 1); + } while (err < 0 && errno == EINTR); + GPR_ASSERT(err == 0); + do { + err = epoll_wait(epfd, ev, GPR_ARRAY_SIZE(ev), 0); + } while (err < 0 && errno == EINTR); + GPR_ASSERT(err == 1); + eventfd_t value; + do { + err = eventfd_read(fd, &value); + } while (err < 0 && errno == EINTR); + GPR_ASSERT(err == 0); + } + close(fd); + close(epfd); + track_counters.Finish(state); +} +BENCHMARK(BM_SingleThreadPollOneFd_SpeedOfLight); +#endif + +static void BM_SingleThreadPollOneFd(benchmark::State& state) { TrackCounters track_counters; size_t ps_sz = grpc_pollset_size(); grpc_pollset* ps = static_cast(gpr_zalloc(ps_sz)); @@ -159,6 +242,6 @@ static void BM_PollAfterWakeup(benchmark::State& state) { gpr_free(ps); track_counters.Finish(state); } -BENCHMARK(BM_PollAfterWakeup); +BENCHMARK(BM_SingleThreadPollOneFd); BENCHMARK_MAIN(); From d0cda5c40b08c9b4db394fc253b830be59c3b7b9 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 7 Mar 2017 18:04:52 -0800 Subject: [PATCH 086/188] Add uv resolver fallback for named ports, fix portability tests --- build.yaml | 2 + .../chttp2/transport/chttp2_transport.c | 8 +-- src/core/lib/iomgr/pollset_uv.c | 22 ++++---- src/core/lib/iomgr/resolve_address_uv.c | 52 ++++++++++++++++++- src/core/lib/iomgr/tcp_client_uv.c | 1 - test/core/iomgr/tcp_client_uv_test.c | 7 +-- test/core/iomgr/tcp_server_uv_test.c | 4 +- test/core/util/trickle_endpoint.c | 5 +- tools/run_tests/generated/tests.json | 4 +- tools/run_tests/run_tests_matrix.py | 6 +-- 10 files changed, 83 insertions(+), 28 deletions(-) diff --git a/build.yaml b/build.yaml index 4f5b8e07f85..38c9a2859d1 100644 --- a/build.yaml +++ b/build.yaml @@ -2562,6 +2562,8 @@ targets: - grpc - gpr_test_util - gpr + exclude_iomgrs: + - uv platforms: - mac - linux diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index da4c7dc7b23..a3684535ff0 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -511,6 +511,10 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_error *error) { if (!t->closed) { + if (!grpc_error_has_clear_grpc_status(error)) { + error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, + GRPC_STATUS_UNAVAILABLE); + } if (t->write_state != GRPC_CHTTP2_WRITE_STATE_IDLE) { if (t->close_transport_on_writes_finished == NULL) { t->close_transport_on_writes_finished = @@ -520,10 +524,6 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx, grpc_error_add_child(t->close_transport_on_writes_finished, error); return; } - if (!grpc_error_has_clear_grpc_status(error)) { - error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, - GRPC_STATUS_UNAVAILABLE); - } t->closed = 1; connectivity_state_set(exec_ctx, t, GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error), "close_transport"); diff --git a/src/core/lib/iomgr/pollset_uv.c b/src/core/lib/iomgr/pollset_uv.c index af33949c698..a2f81bcd78d 100644 --- a/src/core/lib/iomgr/pollset_uv.c +++ b/src/core/lib/iomgr/pollset_uv.c @@ -39,6 +39,7 @@ #include +#include #include #include @@ -61,25 +62,30 @@ gpr_mu grpc_polling_mu; immediately in the next loop iteration. Note: In the future, if there is a bug that involves missing wakeups in the future, try adding a uv_async_t to kick the loop differently */ -uv_timer_t dummy_uv_handle; +uv_timer_t *dummy_uv_handle; size_t grpc_pollset_size() { return sizeof(grpc_pollset); } void dummy_timer_cb(uv_timer_t *handle) {} +void dummy_handle_close_cb(uv_handle_t *handle) { gpr_free(handle); } + void grpc_pollset_global_init(void) { gpr_mu_init(&grpc_polling_mu); - uv_timer_init(uv_default_loop(), &dummy_uv_handle); + dummy_uv_handle = gpr_malloc(sizeof(uv_timer_t)); + uv_timer_init(uv_default_loop(), dummy_uv_handle); grpc_pollset_work_run_loop = 1; } -static void timer_close_cb(uv_handle_t *handle) { handle->data = (void *)1; } - void grpc_pollset_global_shutdown(void) { gpr_mu_destroy(&grpc_polling_mu); - uv_close((uv_handle_t *)&dummy_uv_handle, timer_close_cb); + uv_close((uv_handle_t *)dummy_uv_handle, dummy_handle_close_cb); } +static void timer_run_cb(uv_timer_t *timer) {} + +static void timer_close_cb(uv_handle_t *handle) { handle->data = (void *)1; } + void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) { *mu = &grpc_polling_mu; uv_timer_init(uv_default_loop(), &pollset->timer); @@ -95,7 +101,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, uv_run(uv_default_loop(), UV_RUN_NOWAIT); } else { // kick the loop once - uv_timer_start(&dummy_uv_handle, dummy_timer_cb, 0, 0); + uv_timer_start(dummy_uv_handle, dummy_timer_cb, 0, 0); } grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); } @@ -111,8 +117,6 @@ void grpc_pollset_destroy(grpc_pollset *pollset) { } } -static void timer_run_cb(uv_timer_t *timer) {} - grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { @@ -145,7 +149,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_error *grpc_pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - uv_timer_start(&dummy_uv_handle, dummy_timer_cb, 0, 0); + uv_timer_start(dummy_uv_handle, dummy_timer_cb, 0, 0); return GRPC_ERROR_NONE; } diff --git a/src/core/lib/iomgr/resolve_address_uv.c b/src/core/lib/iomgr/resolve_address_uv.c index 79ff9107389..4d715be94c7 100644 --- a/src/core/lib/iomgr/resolve_address_uv.c +++ b/src/core/lib/iomgr/resolve_address_uv.c @@ -40,6 +40,7 @@ #include #include #include +#include #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" @@ -54,8 +55,36 @@ typedef struct request { grpc_closure *on_done; grpc_resolved_addresses **addresses; struct addrinfo *hints; + char *host; + char *port; } request; +static int retry_named_port_failure(int status, request *r, + uv_getaddrinfo_cb getaddrinfo_cb) { + if (status != 0) { + // This loop is copied from resolve_address_posix.c + char *svc[][2] = {{"http", "80"}, {"https", "443"}}; + for (size_t i = 0; i < GPR_ARRAY_SIZE(svc); i++) { + if (strcmp(r->port, svc[i][0]) == 0) { + int retry_status; + uv_getaddrinfo_t *req = gpr_malloc(sizeof(uv_getaddrinfo_t)); + req->data = r; + retry_status = uv_getaddrinfo(uv_default_loop(), req, getaddrinfo_cb, + r->host, svc[i][1], r->hints); + if (retry_status < 0 || getaddrinfo_cb == NULL) { + // The callback will not be called + gpr_free(req); + } + return retry_status; + } + } + } + /* If this function calls uv_getaddrinfo, it will return that function's + return value. That function only returns numbers <=0, so we can safely + return 1 to indicate that we never retried */ + return 1; +} + static grpc_error *handle_addrinfo_result(int status, struct addrinfo *result, grpc_resolved_addresses **addresses) { struct addrinfo *resp; @@ -97,13 +126,21 @@ static void getaddrinfo_callback(uv_getaddrinfo_t *req, int status, request *r = (request *)req->data; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_error *error; + int retry_status; + + gpr_free(req); + retry_status = retry_named_port_failure(status, r, getaddrinfo_callback); + if (retry_status == 0) { + // The request is being retried. Nothing should be done here + return; + } + /* Either no retry was attempted, or the retry failed. Either way, the + original error probably has more interesting information */ error = handle_addrinfo_result(status, res, r->addresses); grpc_closure_sched(&exec_ctx, r->on_done, error); grpc_exec_ctx_finish(&exec_ctx); - gpr_free(r->hints); gpr_free(r); - gpr_free(req); uv_freeaddrinfo(res); } @@ -143,6 +180,7 @@ static grpc_error *blocking_resolve_address_impl( uv_getaddrinfo_t req; int s; grpc_error *err; + int retry_status; req.addrinfo = NULL; @@ -158,6 +196,12 @@ static grpc_error *blocking_resolve_address_impl( hints.ai_flags = AI_PASSIVE; /* for wildcard IP address */ s = uv_getaddrinfo(uv_default_loop(), &req, NULL, host, port, &hints); + request r = { + .addresses = addresses, .hints = &hints, .host = host, .port = port}; + retry_status = retry_named_port_failure(s, &r, NULL); + if (retry_status <= 0) { + s = retry_status; + } err = handle_addrinfo_result(s, req.addrinfo, addresses); done: @@ -200,6 +244,8 @@ static void resolve_address_impl(grpc_exec_ctx *exec_ctx, const char *name, r = gpr_malloc(sizeof(request)); r->on_done = on_done; r->addresses = addrs; + r->host = host; + r->port = port; req = gpr_malloc(sizeof(uv_getaddrinfo_t)); req->data = r; @@ -222,6 +268,8 @@ static void resolve_address_impl(grpc_exec_ctx *exec_ctx, const char *name, gpr_free(r); gpr_free(req); gpr_free(hints); + gpr_free(host); + gpr_free(port); } } diff --git a/src/core/lib/iomgr/tcp_client_uv.c b/src/core/lib/iomgr/tcp_client_uv.c index ae66577cafd..618483d9cb7 100644 --- a/src/core/lib/iomgr/tcp_client_uv.c +++ b/src/core/lib/iomgr/tcp_client_uv.c @@ -76,7 +76,6 @@ static void uv_tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: error=%s", connect->addr_name, str); - grpc_error_free_string(str); } if (error == GRPC_ERROR_NONE) { /* error == NONE implies that the timer ran out, and wasn't cancelled. If diff --git a/test/core/iomgr/tcp_client_uv_test.c b/test/core/iomgr/tcp_client_uv_test.c index f8938d0abb7..064119f11b9 100644 --- a/test/core/iomgr/tcp_client_uv_test.c +++ b/test/core/iomgr/tcp_client_uv_test.c @@ -58,7 +58,7 @@ static int g_connections_complete = 0; static grpc_endpoint *g_connecting = NULL; static gpr_timespec test_deadline(void) { - return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10); + return grpc_timeout_seconds_to_deadline(10); } static void finish_connection() { @@ -73,7 +73,8 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { GPR_ASSERT(g_connecting != NULL); GPR_ASSERT(error == GRPC_ERROR_NONE); - grpc_endpoint_shutdown(exec_ctx, g_connecting); + grpc_endpoint_shutdown(exec_ctx, g_connecting, + GRPC_ERROR_CREATE("must_succeed called")); grpc_endpoint_destroy(exec_ctx, g_connecting); g_connecting = NULL; finish_connection(); @@ -133,7 +134,7 @@ void test_succeeds(void) { "pollset_work", grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), - GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5)))); + grpc_timeout_seconds_to_deadline(5)))); gpr_mu_unlock(g_mu); grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(g_mu); diff --git a/test/core/iomgr/tcp_server_uv_test.c b/test/core/iomgr/tcp_server_uv_test.c index 7b458c90f3d..0fc74599ea5 100644 --- a/test/core/iomgr/tcp_server_uv_test.c +++ b/test/core/iomgr/tcp_server_uv_test.c @@ -115,7 +115,7 @@ static void server_weak_ref_set(server_weak_ref *weak_ref, static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_pollset *pollset, grpc_tcp_server_acceptor *acceptor) { - grpc_endpoint_shutdown(exec_ctx, tcp); + grpc_endpoint_shutdown(exec_ctx, tcp, GRPC_ERROR_CREATE("Connected")); grpc_endpoint_destroy(exec_ctx, tcp); on_connect_result temp_result; @@ -203,7 +203,7 @@ static void close_cb(uv_handle_t *handle) { gpr_free(handle); } static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, socklen_t remote_len, on_connect_result *result) { - gpr_timespec deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10); + gpr_timespec deadline = grpc_timeout_seconds_to_deadline(10); uv_tcp_t *client_handle = gpr_malloc(sizeof(uv_tcp_t)); uv_connect_t *req = gpr_malloc(sizeof(uv_connect_t)); int nconnects_before; diff --git a/test/core/util/trickle_endpoint.c b/test/core/util/trickle_endpoint.c index 7ab0488a660..0848147158a 100644 --- a/test/core/util/trickle_endpoint.c +++ b/test/core/util/trickle_endpoint.c @@ -31,6 +31,8 @@ * */ +#include "src/core/lib/iomgr/sockaddr.h" + #include "test/core/util/passthru_endpoint.h" #include @@ -40,9 +42,6 @@ #include #include #include - -#include "src/core/lib/iomgr/sockaddr.h" - #include "src/core/lib/slice/slice_internal.h" typedef struct { diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 46c83384019..2af5ec8df1b 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -1790,7 +1790,9 @@ ], "cpu_cost": 1.0, "exclude_configs": [], - "exclude_iomgrs": [], + "exclude_iomgrs": [ + "uv" + ], "flaky": false, "gtest": false, "language": "c", diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 5f5df70d1da..cfc7f2b6fdb 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -92,19 +92,19 @@ def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native', for config in configs: name = '%s_%s_%s_%s' % (language, platform, config, iomgr_platform) runtests_args = ['-l', language, - '-c', config] + '-c', config, + '--iomgr_platform', iomgr_platform] if arch or compiler: name += '_%s_%s' % (arch, compiler) runtests_args += ['--arch', arch, '--compiler', compiler] - runtests_args += extra_args if platform == 'linux': job = _docker_jobspec(name=name, runtests_args=runtests_args, inner_jobs=inner_jobs) else: job = _workspace_jobspec(name=name, runtests_args=runtests_args, inner_jobs=inner_jobs) - job.labels = [platform, config, language] + labels + job.labels = [platform, config, language, iomgr_platform] + labels result.append(job) return result From 343702bf616b21729930379c79d7caf8878751bd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Mar 2017 10:25:04 -0800 Subject: [PATCH 087/188] Write an essay --- .../ext/transport/chttp2/transport/parsing.c | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 7ed00522c3e..e7f2597f89c 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -381,16 +381,38 @@ static grpc_error *update_incoming_window(grpc_exec_ctx *exec_ctx, s->incoming_window_delta + t->settings[GRPC_ACKED_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]) { - char *msg; - gpr_asprintf(&msg, - "frame of size %d overflows incoming window of %" PRId64, - t->incoming_frame_size, - s->incoming_window_delta + - t->settings[GRPC_ACKED_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); - grpc_error *err = GRPC_ERROR_CREATE(msg); - gpr_free(msg); - return err; + if (incoming_frame_size <= + s->incoming_window_delta + + t->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]) { + gpr_log( + GPR_ERROR, + "Incoming frame of size %d exceeds incoming window size of %" PRId64 + ".\n" + "The (un-acked, future) window size would be %" PRId64 + " which is not exceeded.\n" + "This would usually cause a disconnection, but allowing it due to " + "broken HTTP2 implementations in the wild.\n" + "See (for example) https://github.com/netty/netty/issues/6520.", + t->incoming_frame_size, + s->incoming_window_delta + + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], + s->incoming_window_delta + + t->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); + } else { + char *msg; + gpr_asprintf(&msg, + "frame of size %d overflows incoming window of %" PRId64, + t->incoming_frame_size, + s->incoming_window_delta + + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); + grpc_error *err = GRPC_ERROR_CREATE(msg); + gpr_free(msg); + return err; + } } GRPC_CHTTP2_FLOW_DEBIT_STREAM_INCOMING_WINDOW_DELTA("parse", t, s, From a3e5801e2ef90988074cd6c1b74fb7d876f09533 Mon Sep 17 00:00:00 2001 From: Noah Eisen Date: Thu, 9 Mar 2017 10:32:09 -0800 Subject: [PATCH 088/188] s/realloc/gpr_realloc to fix mem bug --- src/core/lib/iomgr/error.c | 2 +- test/cpp/microbenchmarks/bm_error.cc | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index 42c2f3e2dde..f0b7e01d49a 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -210,7 +210,7 @@ static uint8_t get_placement(grpc_error **err, size_t size) { uint8_t slots = (uint8_t)(size / sizeof(intptr_t)); if ((*err)->arena_size + slots > (*err)->arena_capacity) { (*err)->arena_capacity = (uint8_t)(3 * (*err)->arena_capacity / 2); - *err = realloc( + *err = gpr_realloc( *err, sizeof(grpc_error) + (*err)->arena_capacity * sizeof(intptr_t)); } uint8_t placement = (*err)->arena_size; diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc index 6331b245923..76f168b75be 100644 --- a/test/cpp/microbenchmarks/bm_error.cc +++ b/test/cpp/microbenchmarks/bm_error.cc @@ -72,12 +72,14 @@ static void BM_ErrorCreateAndSetStatus(benchmark::State& state) { BENCHMARK(BM_ErrorCreateAndSetStatus); static void BM_ErrorCreateAndSetIntAndStr(benchmark::State& state) { + TrackCounters track_counters; while (state.KeepRunning()) { GRPC_ERROR_UNREF(grpc_error_set_str( grpc_error_set_int(GRPC_ERROR_CREATE("GOAWAY received"), GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)0), GRPC_ERROR_STR_RAW_BYTES, "raw bytes")); } + track_counters.Finish(state); } BENCHMARK(BM_ErrorCreateAndSetIntAndStr); From 2e659c25df3b385701398cfc43f6fcd37f8ed8a4 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 9 Mar 2017 13:41:05 -0800 Subject: [PATCH 089/188] Fix a comment --- src/core/lib/iomgr/wakeup_fd_posix.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/wakeup_fd_posix.h b/src/core/lib/iomgr/wakeup_fd_posix.h index 71d32d97ba7..c8dd242c753 100644 --- a/src/core/lib/iomgr/wakeup_fd_posix.h +++ b/src/core/lib/iomgr/wakeup_fd_posix.h @@ -46,7 +46,7 @@ * * Setup: * 1. Before calling anything, call global_init() at least once. - * 1. Call grpc_wakeup_fd_create() to get a wakeup_fd. + * 1. Call grpc_wakeup_fd_init() to set up a wakeup_fd. * 2. Add the result of GRPC_WAKEUP_FD_FD to the set of monitored file * descriptors for the poll() style API you are using. Monitor the file * descriptor for readability. From 648229ec976ed89d5713f689cf0c2f433cecb806 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 9 Mar 2017 14:18:27 -0800 Subject: [PATCH 090/188] fix in6_scope_id incompatibiliity issue on windows/mingw --- src/core/ext/client_channel/parse_address.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/ext/client_channel/parse_address.c b/src/core/ext/client_channel/parse_address.c index 8ae15fc72bc..cd1b2cd80cf 100644 --- a/src/core/ext/client_channel/parse_address.c +++ b/src/core/ext/client_channel/parse_address.c @@ -128,6 +128,7 @@ int parse_ipv6(grpc_uri *uri, grpc_resolved_address *resolved_addr) { GPR_ASSERT(host_end >= host); char host_without_scope[INET6_ADDRSTRLEN]; size_t host_without_scope_len = (size_t)(host_end - host); + uint32_t sin6_scope_id = 0; strncpy(host_without_scope, host, host_without_scope_len); host_without_scope[host_without_scope_len] = '\0'; if (inet_pton(AF_INET6, host_without_scope, &in6->sin6_addr) == 0) { @@ -136,10 +137,12 @@ int parse_ipv6(grpc_uri *uri, grpc_resolved_address *resolved_addr) { } if (gpr_parse_bytes_to_uint32(host_end + 1, strlen(host) - host_without_scope_len - 1, - &in6->sin6_scope_id) == 0) { + &sin6_scope_id) == 0) { gpr_log(GPR_ERROR, "invalid ipv6 scope id: '%s'", host_end + 1); goto done; } + // Handle "sin6_scope_id" being type "u_long". See grpc issue ##10027. + in6->sin6_scope_id = sin6_scope_id; } else { if (inet_pton(AF_INET6, host, &in6->sin6_addr) == 0) { gpr_log(GPR_ERROR, "invalid ipv6 address: '%s'", host); From 74cec10619c0998ba1507e76ee527bce47c660ce Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 9 Mar 2017 15:00:26 -0800 Subject: [PATCH 091/188] Bumped version to 1.3.0-dev --- CMakeLists.txt | 2 +- Makefile | 4 ++-- build.yaml | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.json | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Auth/project.json | 4 ++-- src/csharp/Grpc.Core.Testing/project.json | 4 ++-- src/csharp/Grpc.Core/VersionInfo.cs | 4 ++-- src/csharp/Grpc.Core/project.json | 2 +- src/csharp/Grpc.HealthCheck/project.json | 4 ++-- src/csharp/Grpc.Reflection/project.json | 4 ++-- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 4 ++-- src/node/health_check/package.json | 4 ++-- src/node/tools/package.json | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/php/composer.json | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 32 files changed, 41 insertions(+), 41 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d3a76a29d76..2c3f38e6172 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.2.0-dev") +set(PACKAGE_VERSION "1.3.0-dev") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 3cffa73224b..9eb3b1b7140 100644 --- a/Makefile +++ b/Makefile @@ -412,8 +412,8 @@ Q = @ endif CORE_VERSION = 3.0.0-dev -CPP_VERSION = 1.2.0-dev -CSHARP_VERSION = 1.2.0-dev +CPP_VERSION = 1.3.0-dev +CSHARP_VERSION = 1.3.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index dc69998cccb..5ba109bad22 100644 --- a/build.yaml +++ b/build.yaml @@ -14,7 +14,7 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 3.0.0-dev g_stands_for: green - version: 1.2.0-dev + version: 1.3.0-dev filegroups: - name: census public_headers: diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 909ea5af967..027babcda46 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -37,7 +37,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.2.0-dev' + version = '1.3.0-dev' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'http://www.grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 33ad74d2dfc..6ca9fcd3426 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.2.0-dev' + version = '1.3.0-dev' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'http://www.grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 35eb5587c85..51b52c0c440 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.2.0-dev' + version = '1.3.0-dev' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'http://www.grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 6ba0c01d39f..83a86803ebc 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -35,7 +35,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.2.0-dev' + version = '1.3.0-dev' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'http://www.grpc.io' diff --git a/package.json b/package.json index d729f3d8376..8c0854b1d28 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.2.0-dev", + "version": "1.3.0-dev", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "http://www.grpc.io/", diff --git a/package.xml b/package.xml index e16161d7a2e..4167bef26e3 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2017-03-01 - 1.2.0dev - 1.2.0dev + 1.3.0dev + 1.3.0dev beta diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 039c530cdc7..f5a0e4131de 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -37,5 +37,5 @@ #include namespace grpc { -grpc::string Version() { return "1.2.0-dev"; } +grpc::string Version() { return "1.3.0-dev"; } } diff --git a/src/csharp/Grpc.Auth/project.json b/src/csharp/Grpc.Auth/project.json index 170149ace52..370bf11b2dd 100644 --- a/src/csharp/Grpc.Auth/project.json +++ b/src/csharp/Grpc.Auth/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.0-dev", + "version": "1.3.0-dev", "title": "gRPC C# Auth", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.0-dev", + "Grpc.Core": "1.3.0-dev", "Google.Apis.Auth": "1.21.0" }, "frameworks": { diff --git a/src/csharp/Grpc.Core.Testing/project.json b/src/csharp/Grpc.Core.Testing/project.json index 02be9578126..38d5fab50e1 100644 --- a/src/csharp/Grpc.Core.Testing/project.json +++ b/src/csharp/Grpc.Core.Testing/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.0-dev", + "version": "1.3.0-dev", "title": "gRPC C# Core Testing", "authors": [ "Google Inc." ], "copyright": "Copyright 2017, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.0-dev" + "Grpc.Core": "1.3.0-dev" }, "frameworks": { "net45": { diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index f01a024db43..6012d904b69 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -48,11 +48,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "1.2.0.0"; + public const string CurrentAssemblyFileVersion = "1.3.0.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.2.0-dev"; + public const string CurrentVersion = "1.3.0-dev"; } } diff --git a/src/csharp/Grpc.Core/project.json b/src/csharp/Grpc.Core/project.json index 0e37ec8927f..a1306baa876 100644 --- a/src/csharp/Grpc.Core/project.json +++ b/src/csharp/Grpc.Core/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.0-dev", + "version": "1.3.0-dev", "title": "gRPC C# Core", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", diff --git a/src/csharp/Grpc.HealthCheck/project.json b/src/csharp/Grpc.HealthCheck/project.json index 9e9d245caec..e93d0bf81b3 100644 --- a/src/csharp/Grpc.HealthCheck/project.json +++ b/src/csharp/Grpc.HealthCheck/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.0-dev", + "version": "1.3.0-dev", "title": "gRPC C# Healthchecking", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.0-dev", + "Grpc.Core": "1.3.0-dev", "Google.Protobuf": "3.2.0" }, "frameworks": { diff --git a/src/csharp/Grpc.Reflection/project.json b/src/csharp/Grpc.Reflection/project.json index 8bfe722f780..014c78e489d 100644 --- a/src/csharp/Grpc.Reflection/project.json +++ b/src/csharp/Grpc.Reflection/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.0-dev", + "version": "1.3.0-dev", "title": "gRPC C# Reflection", "authors": [ "Google Inc." ], "copyright": "Copyright 2016, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.0-dev", + "Grpc.Core": "1.3.0-dev", "Google.Protobuf": "3.2.0" }, "frameworks": { diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index b99fdcbdfde..4fec2c71cf8 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -28,7 +28,7 @@ @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @rem Current package versions -set VERSION=1.2.0-dev +set VERSION=1.3.0-dev set PROTOBUF_VERSION=3.0.0 @rem Adjust the location of nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 442e3acad28..f51b42bc8c1 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -66,7 +66,7 @@ dotnet pack --configuration Release Grpc.Auth/project.json --output ../../artifa dotnet pack --configuration Release Grpc.HealthCheck/project.json --output ../../artifacts dotnet pack --configuration Release Grpc.Reflection/project.json --output ../../artifacts -nuget pack Grpc.nuspec -Version "1.2.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.2.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.3.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.3.0-dev" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index 8376339debe..e218f5a4063 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -1,6 +1,6 @@ { "name": "grpc-health-check", - "version": "1.2.0-dev", + "version": "1.3.0-dev", "author": "Google Inc.", "description": "Health check service for use with gRPC", "repository": { @@ -15,7 +15,7 @@ } ], "dependencies": { - "grpc": "^1.2.0-dev", + "grpc": "^1.3.0-dev", "lodash": "^3.9.3", "google-protobuf": "^3.0.0" }, diff --git a/src/node/tools/package.json b/src/node/tools/package.json index 53dd53f5391..3096c6e42a8 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.2.0-dev", + "version": "1.3.0-dev", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "http://www.grpc.io/", diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 1a3b775c609..ab8f82a39e7 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.2.0-dev' + v = '1.3.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index e569faa25bb..09155ee4d40 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -38,4 +38,4 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.2.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.3.0-dev" diff --git a/src/php/composer.json b/src/php/composer.json index 491e34795ac..2b140077cc5 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "BSD-3-Clause", - "version": "1.2.0", + "version": "1.3.0", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.1.0" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index c197e92ca58..267d848e743 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='1.2.0.dev0' +VERSION='1.3.0.dev0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index c1807e9f1c2..4ff5e266a1f 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION='1.2.0.dev0' +VERSION='1.3.0.dev0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 3778dcd3e0b..8ffc08c04b8 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION='1.2.0.dev0' +VERSION='1.3.0.dev0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 33824b6b8fc..ba82dce6f6d 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION='1.2.0.dev0' +VERSION='1.3.0.dev0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index ce0892decfb..9901158e735 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -29,5 +29,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.2.0.dev' + VERSION = '1.3.0.dev' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 8f3d2ba81cc..632c0100bd8 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -29,6 +29,6 @@ module GRPC module Tools - VERSION = '1.2.0.dev' + VERSION = '1.3.0.dev' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 263785b774b..ad35b90bad3 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.2.0.dev0' +VERSION='1.3.0.dev0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 965259746cf..cdf2a63580d 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.2.0-dev +PROJECT_NUMBER = 1.3.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index d98475fb00c..3135df39b83 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.2.0-dev +PROJECT_NUMBER = 1.3.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 904e7f53aeb585df4eacac5d549585654bed1fd8 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 9 Mar 2017 15:02:19 -0800 Subject: [PATCH 092/188] Introducing 1.2.0-pre1 --- CMakeLists.txt | 2 +- Makefile | 4 ++-- build.yaml | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.json | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Auth/project.json | 4 ++-- src/csharp/Grpc.Core.Testing/project.json | 4 ++-- src/csharp/Grpc.Core/VersionInfo.cs | 2 +- src/csharp/Grpc.Core/project.json | 2 +- src/csharp/Grpc.HealthCheck/project.json | 4 ++-- src/csharp/Grpc.Reflection/project.json | 4 ++-- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 4 ++-- src/node/health_check/package.json | 4 ++-- src/node/tools/package.json | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 31 files changed, 39 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d3a76a29d76..89a983ca4dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.2.0-dev") +set(PACKAGE_VERSION "1.2.0-pre1") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 3cffa73224b..c35fe90a17f 100644 --- a/Makefile +++ b/Makefile @@ -412,8 +412,8 @@ Q = @ endif CORE_VERSION = 3.0.0-dev -CPP_VERSION = 1.2.0-dev -CSHARP_VERSION = 1.2.0-dev +CPP_VERSION = 1.2.0-pre1 +CSHARP_VERSION = 1.2.0-pre1 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index dc69998cccb..dfd0619578f 100644 --- a/build.yaml +++ b/build.yaml @@ -14,7 +14,7 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 3.0.0-dev g_stands_for: green - version: 1.2.0-dev + version: 1.2.0-pre1 filegroups: - name: census public_headers: diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 909ea5af967..8b67a5dcdd2 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -37,7 +37,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.2.0-dev' + version = '1.2.0-pre1' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'http://www.grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 33ad74d2dfc..8fbdf7ad55d 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.2.0-dev' + version = '1.2.0-pre1' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'http://www.grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 35eb5587c85..e302e6482e3 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.2.0-dev' + version = '1.2.0-pre1' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'http://www.grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 6ba0c01d39f..64a6f0aba47 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -35,7 +35,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.2.0-dev' + version = '1.2.0-pre1' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'http://www.grpc.io' diff --git a/package.json b/package.json index d729f3d8376..2228884816f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.2.0-dev", + "version": "1.2.0-pre1", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "http://www.grpc.io/", diff --git a/package.xml b/package.xml index e16161d7a2e..94305025b5c 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2017-03-01 - 1.2.0dev - 1.2.0dev + 1.2.0RC1 + 1.2.0RC1 beta diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 039c530cdc7..30395fbac53 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -37,5 +37,5 @@ #include namespace grpc { -grpc::string Version() { return "1.2.0-dev"; } +grpc::string Version() { return "1.2.0-pre1"; } } diff --git a/src/csharp/Grpc.Auth/project.json b/src/csharp/Grpc.Auth/project.json index 170149ace52..d4fbdc1d56d 100644 --- a/src/csharp/Grpc.Auth/project.json +++ b/src/csharp/Grpc.Auth/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.0-dev", + "version": "1.2.0-pre1", "title": "gRPC C# Auth", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.0-dev", + "Grpc.Core": "1.2.0-pre1", "Google.Apis.Auth": "1.21.0" }, "frameworks": { diff --git a/src/csharp/Grpc.Core.Testing/project.json b/src/csharp/Grpc.Core.Testing/project.json index 02be9578126..9d1180d0549 100644 --- a/src/csharp/Grpc.Core.Testing/project.json +++ b/src/csharp/Grpc.Core.Testing/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.0-dev", + "version": "1.2.0-pre1", "title": "gRPC C# Core Testing", "authors": [ "Google Inc." ], "copyright": "Copyright 2017, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.0-dev" + "Grpc.Core": "1.2.0-pre1" }, "frameworks": { "net45": { diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index f01a024db43..b6e3bc5f8d2 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -53,6 +53,6 @@ namespace Grpc.Core /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.2.0-dev"; + public const string CurrentVersion = "1.2.0-pre1"; } } diff --git a/src/csharp/Grpc.Core/project.json b/src/csharp/Grpc.Core/project.json index 0e37ec8927f..51311010cd5 100644 --- a/src/csharp/Grpc.Core/project.json +++ b/src/csharp/Grpc.Core/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.0-dev", + "version": "1.2.0-pre1", "title": "gRPC C# Core", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", diff --git a/src/csharp/Grpc.HealthCheck/project.json b/src/csharp/Grpc.HealthCheck/project.json index 9e9d245caec..330002f3ad6 100644 --- a/src/csharp/Grpc.HealthCheck/project.json +++ b/src/csharp/Grpc.HealthCheck/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.0-dev", + "version": "1.2.0-pre1", "title": "gRPC C# Healthchecking", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.0-dev", + "Grpc.Core": "1.2.0-pre1", "Google.Protobuf": "3.2.0" }, "frameworks": { diff --git a/src/csharp/Grpc.Reflection/project.json b/src/csharp/Grpc.Reflection/project.json index 8bfe722f780..1a77a854c2d 100644 --- a/src/csharp/Grpc.Reflection/project.json +++ b/src/csharp/Grpc.Reflection/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.0-dev", + "version": "1.2.0-pre1", "title": "gRPC C# Reflection", "authors": [ "Google Inc." ], "copyright": "Copyright 2016, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.0-dev", + "Grpc.Core": "1.2.0-pre1", "Google.Protobuf": "3.2.0" }, "frameworks": { diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index b99fdcbdfde..fee896d883f 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -28,7 +28,7 @@ @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @rem Current package versions -set VERSION=1.2.0-dev +set VERSION=1.2.0-pre1 set PROTOBUF_VERSION=3.0.0 @rem Adjust the location of nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 442e3acad28..bc7350f3afc 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -66,7 +66,7 @@ dotnet pack --configuration Release Grpc.Auth/project.json --output ../../artifa dotnet pack --configuration Release Grpc.HealthCheck/project.json --output ../../artifacts dotnet pack --configuration Release Grpc.Reflection/project.json --output ../../artifacts -nuget pack Grpc.nuspec -Version "1.2.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.2.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.2.0-pre1" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.2.0-pre1" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index 8376339debe..302f2606c6b 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -1,6 +1,6 @@ { "name": "grpc-health-check", - "version": "1.2.0-dev", + "version": "1.2.0-pre1", "author": "Google Inc.", "description": "Health check service for use with gRPC", "repository": { @@ -15,7 +15,7 @@ } ], "dependencies": { - "grpc": "^1.2.0-dev", + "grpc": "^1.2.0-pre1", "lodash": "^3.9.3", "google-protobuf": "^3.0.0" }, diff --git a/src/node/tools/package.json b/src/node/tools/package.json index 53dd53f5391..78071ad55af 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.2.0-dev", + "version": "1.2.0-pre1", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "http://www.grpc.io/", diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 1a3b775c609..ccad095f17d 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.2.0-dev' + v = '1.2.0-pre1' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index e569faa25bb..2d9ecc6c308 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -38,4 +38,4 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.2.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.2.0-pre1" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index c197e92ca58..12c80fdbef3 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='1.2.0.dev0' +VERSION='1.2.0rc1' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index c1807e9f1c2..6d7907d3778 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION='1.2.0.dev0' +VERSION='1.2.0rc1' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 3778dcd3e0b..0372478401c 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION='1.2.0.dev0' +VERSION='1.2.0rc1' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 33824b6b8fc..5be5ecf254e 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION='1.2.0.dev0' +VERSION='1.2.0rc1' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index ce0892decfb..c5169e6ae3f 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -29,5 +29,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.2.0.dev' + VERSION = '1.2.0.pre1' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 8f3d2ba81cc..2970518f916 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -29,6 +29,6 @@ module GRPC module Tools - VERSION = '1.2.0.dev' + VERSION = '1.2.0.pre1' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 263785b774b..72c929889bb 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.2.0.dev0' +VERSION='1.2.0rc1' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 965259746cf..1588755fde6 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.2.0-dev +PROJECT_NUMBER = 1.2.0-pre1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index d98475fb00c..62d801054dc 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.2.0-dev +PROJECT_NUMBER = 1.2.0-pre1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 072c5af838aa9b6bcb98dbf4dd969708ba794152 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Mar 2017 10:25:04 -0800 Subject: [PATCH 093/188] Write an essay --- .../ext/transport/chttp2/transport/parsing.c | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 7ed00522c3e..e7f2597f89c 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -381,16 +381,38 @@ static grpc_error *update_incoming_window(grpc_exec_ctx *exec_ctx, s->incoming_window_delta + t->settings[GRPC_ACKED_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]) { - char *msg; - gpr_asprintf(&msg, - "frame of size %d overflows incoming window of %" PRId64, - t->incoming_frame_size, - s->incoming_window_delta + - t->settings[GRPC_ACKED_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); - grpc_error *err = GRPC_ERROR_CREATE(msg); - gpr_free(msg); - return err; + if (incoming_frame_size <= + s->incoming_window_delta + + t->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]) { + gpr_log( + GPR_ERROR, + "Incoming frame of size %d exceeds incoming window size of %" PRId64 + ".\n" + "The (un-acked, future) window size would be %" PRId64 + " which is not exceeded.\n" + "This would usually cause a disconnection, but allowing it due to " + "broken HTTP2 implementations in the wild.\n" + "See (for example) https://github.com/netty/netty/issues/6520.", + t->incoming_frame_size, + s->incoming_window_delta + + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], + s->incoming_window_delta + + t->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); + } else { + char *msg; + gpr_asprintf(&msg, + "frame of size %d overflows incoming window of %" PRId64, + t->incoming_frame_size, + s->incoming_window_delta + + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); + grpc_error *err = GRPC_ERROR_CREATE(msg); + gpr_free(msg); + return err; + } } GRPC_CHTTP2_FLOW_DEBIT_STREAM_INCOMING_WINDOW_DELTA("parse", t, s, From 9d68158d13dda1ca0437b6c773ca2c3276fe2f4c Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Thu, 9 Mar 2017 15:37:21 -0800 Subject: [PATCH 094/188] Change argument validation in Server#addService --- src/node/src/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/src/server.js b/src/node/src/server.js index 8a7eff507d0..a5a0ea26420 100644 --- a/src/node/src/server.js +++ b/src/node/src/server.js @@ -728,7 +728,7 @@ var defaultHandler = { * method implementation for the provided service. */ Server.prototype.addService = function(service, implementation) { - if (!_.isObjectLike(service) || !_.isObjectLike(implementation)) { + if (!_.isObject(service) || !_.isObject(implementation)) { throw new Error('addService requires two objects as arguments'); } if (_.keys(service).length === 0) { From 9f1e5ab3c99168590d481008a2a101758915878c Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Thu, 2 Mar 2017 23:29:52 +0000 Subject: [PATCH 095/188] Add test behavior to yapf_code.sh With this change yapf_code no longer changes the code under test when executed from within the sanity tests. --- .gitignore | 2 +- tools/distrib/yapf_code.sh | 59 ++++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 56e4b6d4b43..da2082c088c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ objs # Python items cython_debug/ python_build/ -python_format_venv/ +yapf_virtual_environment/ python_pylint_venv/ .coverage* .eggs diff --git a/tools/distrib/yapf_code.sh b/tools/distrib/yapf_code.sh index 007b14810eb..f28a1ce8bae 100755 --- a/tools/distrib/yapf_code.sh +++ b/tools/distrib/yapf_code.sh @@ -31,31 +31,48 @@ set -ex # change to root directory -cd $(dirname $0)/../.. +cd "$(dirname "${0}")/../.." -DIRS=src/python -EXCLUSIONS='src/python/grpcio/grpc_*.py src/python/grpcio_health_checking/grpc_*.py src/python/grpcio_reflection/grpc_*.py src/python/grpcio_tests/grpc_*.py' +DIRS=( + 'src/python' +) +EXCLUSIONS=( + 'grpcio/grpc_*.py' + 'grpcio_health_checking/grpc_*.py' + 'grpcio_reflection/grpc_*.py' + 'grpcio_tests/grpc_*.py' +) -VIRTUALENV=python_format_venv +VIRTUALENV=yapf_virtual_environment virtualenv $VIRTUALENV -PYTHON=`realpath $VIRTUALENV/bin/python` -$PYTHON -m pip install futures +PYTHON=$(realpath "${VIRTUALENV}/bin/python") +$PYTHON -m pip install --upgrade pip +$PYTHON -m pip install --upgrade futures $PYTHON -m pip install yapf==0.16.0 -exclusion_args="" -for exclusion in $EXCLUSIONS; do - exclusion_args="$exclusion_args --exclude $exclusion" -done +yapf() { + local exclusion exclusion_args=() + for exclusion in "${EXCLUSIONS[@]}"; do + exclusion_args+=( "--exclude" "$1/${exclusion}" ) + done + $PYTHON -m yapf -i -r --style=setup.cfg -p "${exclusion_args[@]}" "${1}" +} -script_result=0 -for dir in $DIRS; do - tempdir=`mktemp -d` - cp -RT $dir $tempdir - $PYTHON -m yapf -i -r -p $exclusion_args $dir - if ! diff -r $dir $tempdir; then - script_result=1 - fi - rm -rf $tempdir -done -exit $script_result +if [[ -z "${TEST}" ]]; then + for dir in "${DIRS[@]}"; do + yapf "${dir}" + done +else + ok=yes + for dir in "${DIRS[@]}"; do + tempdir=$(mktemp -d) + cp -RT "${dir}" "${tempdir}" + yapf "${tempdir}" + diff -ru "${dir}" "${tempdir}" || ok=no + rm -rf "${tempdir}" + done + if [[ ${ok} == no ]]; then + false + fi +fi From 2b182c2b4b82f3443111f79227de6a492fa22c78 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Mar 2017 16:19:58 -0800 Subject: [PATCH 096/188] Make ASAN happy --- test/cpp/microbenchmarks/bm_pollset.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc index da06eae5078..0f3d3cef66a 100644 --- a/test/cpp/microbenchmarks/bm_pollset.cc +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -150,14 +150,20 @@ static void BM_PollEmptyPollset(benchmark::State& state) { } BENCHMARK(BM_PollEmptyPollset); +class Closure : public grpc_closure { + public: + virtual ~Closure() {} +}; + template -grpc_closure* MakeClosure(F f, grpc_closure_scheduler* scheduler) { - struct C : public grpc_closure { +Closure* MakeClosure(F f, grpc_closure_scheduler* scheduler) { + struct C : public Closure { C(F f, grpc_closure_scheduler* scheduler) : f_(f) { grpc_closure_init(this, C::cbfn, this, scheduler); } static void cbfn(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - static_cast(arg)->f_(); + C* p = static_cast(arg); + p->f_(); } F f_; }; @@ -212,7 +218,7 @@ static void BM_SingleThreadPollOneFd(benchmark::State& state) { grpc_fd* wakeup = grpc_fd_create(wakeup_fd.read_fd, "wakeup_read"); grpc_pollset_add_fd(&exec_ctx, ps, wakeup); bool done = false; - grpc_closure* continue_closure = MakeClosure( + Closure* continue_closure = MakeClosure( [&]() { GRPC_ERROR_UNREF(grpc_wakeup_fd_consume_wakeup(&wakeup_fd)); if (!state.KeepRunning()) { @@ -241,6 +247,7 @@ static void BM_SingleThreadPollOneFd(benchmark::State& state) { grpc_wakeup_fd_destroy(&wakeup_fd); gpr_free(ps); track_counters.Finish(state); + delete continue_closure; } BENCHMARK(BM_SingleThreadPollOneFd); From f90d4c837283f86cc812c7ea39afb89ed64521c7 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Thu, 9 Mar 2017 16:35:21 -0800 Subject: [PATCH 097/188] Add copyright and license to bm_diff.py --- tools/profiling/microbenchmarks/bm_diff.py | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 5e1f68bc37d..59cc2d0e475 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -1,4 +1,32 @@ #!/usr/bin/env python2.7 +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import json From ceddd293919a9580cef928fa56a35480de278682 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Thu, 9 Mar 2017 17:04:24 -0800 Subject: [PATCH 098/188] Address github comments --- src/core/lib/iomgr/error.c | 17 +++++++++-------- src/core/lib/iomgr/error_internal.h | 4 ++-- src/core/lib/transport/error_utils.c | 4 ++-- test/core/iomgr/error_test.c | 2 +- test/cpp/microbenchmarks/bm_error.cc | 24 ++++++++++++++++++++++++ 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index f0b7e01d49a..c6851b0b6cb 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -155,7 +155,7 @@ grpc_error *grpc_error_ref(grpc_error *err) { static void unref_errs(grpc_error *err) { uint8_t slot = err->first_err; while (slot != UINT8_MAX) { - linked_error *lerr = (linked_error *)(err->arena + slot); + grpc_linked_error *lerr = (grpc_linked_error *)(err->arena + slot); GRPC_ERROR_UNREF(lerr->err); GPR_ASSERT(err->last_err == slot ? lerr->next == UINT8_MAX : lerr->next != UINT8_MAX); @@ -254,25 +254,26 @@ static void internal_set_time(grpc_error **err, grpc_error_times which, } static void internal_add_error(grpc_error **err, grpc_error *new) { - linked_error new_last = {new, UINT8_MAX}; - uint8_t slot = get_placement(err, sizeof(linked_error)); + grpc_linked_error new_last = {new, UINT8_MAX}; + uint8_t slot = get_placement(err, sizeof(grpc_linked_error)); if ((*err)->first_err == UINT8_MAX) { GPR_ASSERT((*err)->last_err == UINT8_MAX); (*err)->last_err = slot; (*err)->first_err = slot; } else { GPR_ASSERT((*err)->last_err != UINT8_MAX); - linked_error *old_last = (linked_error *)((*err)->arena + (*err)->last_err); + grpc_linked_error *old_last = + (grpc_linked_error *)((*err)->arena + (*err)->last_err); old_last->next = slot; (*err)->last_err = slot; } - memcpy((*err)->arena + slot, &new_last, sizeof(linked_error)); + memcpy((*err)->arena + slot, &new_last, sizeof(grpc_linked_error)); } #define SLOTS_PER_INT (sizeof(intptr_t) / sizeof(intptr_t)) #define SLOTS_PER_STR (sizeof(grpc_slice) / sizeof(intptr_t)) #define SLOTS_PER_TIME (sizeof(gpr_timespec) / sizeof(intptr_t)) -#define SLOTS_PER_LINKED_ERROR (sizeof(linked_error) / sizeof(intptr_t)) +#define SLOTS_PER_LINKED_ERROR (sizeof(grpc_linked_error) / sizeof(intptr_t)) // size of storing one int and two slices and a timespec. For line, desc, file, // and time created @@ -345,7 +346,7 @@ static void ref_strs(grpc_error *err) { static void ref_errs(grpc_error *err) { uint8_t slot = err->first_err; while (slot != UINT8_MAX) { - linked_error *lerr = (linked_error *)(err->arena + slot); + grpc_linked_error *lerr = (grpc_linked_error *)(err->arena + slot); GRPC_ERROR_REF(lerr->err); slot = lerr->next; } @@ -636,7 +637,7 @@ static void add_errs(grpc_error *err, char **s, size_t *sz, size_t *cap) { uint8_t slot = err->first_err; bool first = true; while (slot != UINT8_MAX) { - linked_error *lerr = (linked_error *)(err->arena + slot); + grpc_linked_error *lerr = (grpc_linked_error *)(err->arena + slot); if (!first) append_chr(',', s, sz, cap); first = false; const char *e = grpc_error_string(lerr->err); diff --git a/src/core/lib/iomgr/error_internal.h b/src/core/lib/iomgr/error_internal.h index bdebbe1792a..fb4814e41f2 100644 --- a/src/core/lib/iomgr/error_internal.h +++ b/src/core/lib/iomgr/error_internal.h @@ -39,9 +39,9 @@ #include -typedef struct linked_error linked_error; +typedef struct grpc_linked_error grpc_linked_error; -struct linked_error { +struct grpc_linked_error { grpc_error *err; uint8_t next; }; diff --git a/src/core/lib/transport/error_utils.c b/src/core/lib/transport/error_utils.c index 707aac024b2..ef55e561fbf 100644 --- a/src/core/lib/transport/error_utils.c +++ b/src/core/lib/transport/error_utils.c @@ -46,7 +46,7 @@ static grpc_error *recursively_find_error_with_field(grpc_error *error, // Otherwise, search through its children. uint8_t slot = error->first_err; while (slot != UINT8_MAX) { - linked_error *lerr = (linked_error *)(error->arena + slot); + grpc_linked_error *lerr = (grpc_linked_error *)(error->arena + slot); grpc_error *result = recursively_find_error_with_field(lerr->err, which); if (result) return result; slot = lerr->next; @@ -114,7 +114,7 @@ bool grpc_error_has_clear_grpc_status(grpc_error *error) { } uint8_t slot = error->first_err; while (slot != UINT8_MAX) { - linked_error *lerr = (linked_error *)(error->arena + slot); + grpc_linked_error *lerr = (grpc_linked_error *)(error->arena + slot); if (grpc_error_has_clear_grpc_status(lerr->err)) { return true; } diff --git a/test/core/iomgr/error_test.c b/test/core/iomgr/error_test.c index d8289ac1029..2a6b1b17fde 100644 --- a/test/core/iomgr/error_test.c +++ b/test/core/iomgr/error_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc index 76f168b75be..c4f6aa19d5b 100644 --- a/test/cpp/microbenchmarks/bm_error.cc +++ b/test/cpp/microbenchmarks/bm_error.cc @@ -83,6 +83,30 @@ static void BM_ErrorCreateAndSetIntAndStr(benchmark::State& state) { } BENCHMARK(BM_ErrorCreateAndSetIntAndStr); +static void BM_ErrorCreateAndSetIntLoop(benchmark::State& state) { + TrackCounters track_counters; + grpc_error* error = GRPC_ERROR_CREATE("Error"); + int n = 0; + while (state.KeepRunning()) { + error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, n++); + } + GRPC_ERROR_UNREF(error); + track_counters.Finish(state); +} +BENCHMARK(BM_ErrorCreateAndSetIntLoop); + +static void BM_ErrorCreateAndSetStrLoop(benchmark::State& state) { + TrackCounters track_counters; + grpc_error* error = GRPC_ERROR_CREATE("Error"); + const char* str = "hello"; + while (state.KeepRunning()) { + error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, str); + } + GRPC_ERROR_UNREF(error); + track_counters.Finish(state); +} +BENCHMARK(BM_ErrorCreateAndSetStrLoop); + static void BM_ErrorRefUnref(benchmark::State& state) { TrackCounters track_counters; grpc_error* error = GRPC_ERROR_CREATE("Error"); From e3edfe29869086dcc575c8054bc27aabf7fb4077 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Fri, 10 Mar 2017 00:55:11 -0800 Subject: [PATCH 099/188] Give execute permission for Kokoro shell scripts --- tools/internal_ci/linux/grpc_master_sanitizers.sh | 0 tools/internal_ci/linux/grpc_portability.sh | 0 tools/internal_ci/linux/grpc_portability_build_only.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/internal_ci/linux/grpc_master_sanitizers.sh mode change 100644 => 100755 tools/internal_ci/linux/grpc_portability.sh mode change 100644 => 100755 tools/internal_ci/linux/grpc_portability_build_only.sh diff --git a/tools/internal_ci/linux/grpc_master_sanitizers.sh b/tools/internal_ci/linux/grpc_master_sanitizers.sh old mode 100644 new mode 100755 diff --git a/tools/internal_ci/linux/grpc_portability.sh b/tools/internal_ci/linux/grpc_portability.sh old mode 100644 new mode 100755 diff --git a/tools/internal_ci/linux/grpc_portability_build_only.sh b/tools/internal_ci/linux/grpc_portability_build_only.sh old mode 100644 new mode 100755 From d159af31fcf300cfe2d7b9b41d789cd664ee4f1b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 8 Mar 2017 16:12:01 +0100 Subject: [PATCH 100/188] return tuple from jobset.run when skipping tests --- tools/run_tests/python_utils/jobset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index f3047431e2a..5d812f28ee9 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -476,13 +476,13 @@ def run(cmdlines, skip_jobs=False, quiet_success=False): if skip_jobs: - results = {} + resultset = {} skipped_job_result = JobResult() skipped_job_result.state = 'SKIPPED' for job in cmdlines: message('SKIPPED', job.shortname, do_newline=True) - results[job.shortname] = [skipped_job_result] - return results + resultset[job.shortname] = [skipped_job_result] + return 0, resultset js = Jobset(check_cancelled, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, newline_on_success, travis, stop_on_failure, add_env, From 7134a316f62a094afcfc461e947c91b36706cce0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 7 Mar 2017 19:00:25 +0100 Subject: [PATCH 101/188] add --manual_run option for running interop tests --- tools/run_tests/run_interop_tests.py | 118 +++++++++++++++++++++------ 1 file changed, 94 insertions(+), 24 deletions(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 0d5bec1d67f..161fa81956b 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -501,6 +501,28 @@ def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): return docker_cmdline +def manual_cmdline(docker_cmdline): + """Returns docker cmdline adjusted for manual invocation.""" + print_cmdline = [] + for item in docker_cmdline: + if item.startswith('--name='): + continue + # add quotes when necessary + if any(character.isspace() for character in item): + item = "\"%s\"" % item + print_cmdline.append(item) + return ' '.join(print_cmdline) + + +def write_cmdlog_maybe(cmdlog, filename): + """Returns docker cmdline adjusted for manual invocation.""" + if cmdlog: + with open(filename, 'w') as logfile: + logfile.write('#!/bin/bash\n') + logfile.writelines("%s\n" % line for line in cmdlog) + print('Command log written to file %s' % filename) + + def bash_cmdline(cmdline): """Creates bash -c cmdline from args list.""" # Use login shell: @@ -551,7 +573,8 @@ def _job_kill_handler(job): def cloud_to_prod_jobspec(language, test_case, server_host_name, - server_host_detail, docker_image=None, auth=False): + server_host_detail, docker_image=None, auth=False, + manual_cmd_log=None): """Creates jobspec for cloud-to-prod interop test""" container_name = None cmdargs = [ @@ -576,7 +599,9 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name, cwd=cwd, environ=environ, docker_args=['--net=host', - '--name', container_name]) + '--name=%s' % container_name]) + if manual_cmd_log is not None: + manual_cmd_log.append(manual_cmdline(cmdline)) cwd = None environ = None @@ -597,7 +622,8 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name, def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, - server_port, docker_image=None, insecure=False): + server_port, docker_image=None, insecure=False, + manual_cmd_log=None): """Creates jobspec for cloud-to-cloud interop test""" interop_only_options = [ '--server_host_override=foo.test.google.fr', @@ -628,7 +654,9 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, environ=environ, cwd=cwd, docker_args=['--net=host', - '--name', container_name]) + '--name=%s' % container_name]) + if manual_cmd_log is not None: + manual_cmd_log.append(manual_cmdline(cmdline)) cwd = None test_job = jobset.JobSpec( @@ -646,7 +674,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, return test_job -def server_jobspec(language, docker_image, insecure=False): +def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): """Create jobspec for running a server""" container_name = dockerjob.random_name('interop_server_%s' % language.safename) cmdline = bash_cmdline( @@ -676,7 +704,9 @@ def server_jobspec(language, docker_image, insecure=False): cwd=language.server_cwd, environ=environ, docker_args=port_args + - ['--name', container_name]) + ['--name=%s' % container_name]) + if manual_cmd_log is not None: + manual_cmd_log.append(manual_cmdline(docker_cmdline)) server_job = jobset.JobSpec( cmdline=docker_cmdline, environ=environ, @@ -806,6 +836,14 @@ argp.add_argument('--allow_flakes', action='store_const', const=True, help='Allow flaky tests to show as passing (re-runs failed tests up to five times)') +argp.add_argument('--manual_run', + default=False, + action='store_const', + const=True, + help='Prepare things for running interop tests manually. ' + + 'Preserve docker images after building them and skip ' + 'actually running the tests. Only print commands to run by ' + + 'hand.') argp.add_argument('--http2_interop', default=False, action='store_const', @@ -837,6 +875,10 @@ if args.use_docker: print('copied to the docker environment.') time.sleep(5) +if args.manual_run and not args.use_docker: + print('--manual_run is only supported with --use_docker option enabled.') + sys.exit(1) + if not args.use_docker and servers: print('Running interop servers is only supported with --use_docker option enabled.') sys.exit(1) @@ -887,24 +929,36 @@ if args.use_docker: dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) +server_manual_cmd_log = [] if args.manual_run else None +client_manual_cmd_log = [] if args.manual_run else None + # Start interop servers. -server_jobs={} -server_addresses={} +server_jobs = {} +server_addresses = {} try: for s in servers: lang = str(s) spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang), - args.insecure) - job = dockerjob.DockerJob(spec) - server_jobs[lang] = job - server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT)) + args.insecure, manual_cmd_log=server_manual_cmd_log) + if not args.manual_run: + job = dockerjob.DockerJob(spec) + server_jobs[lang] = job + server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT)) + else: + # don't run the server, set server port to a placeholder value + server_addresses[lang] = ('localhost', '${SERVER_PORT}') if args.http2_badserver_interop: # launch a HTTP2 server emulator that creates edge cases lang = str(http2InteropServer) - spec = server_jobspec(http2InteropServer, docker_images.get(lang)) - job = dockerjob.DockerJob(spec) - server_jobs[lang] = job + spec = server_jobspec(http2InteropServer, docker_images.get(lang), + manual_cmd_log=server_manual_cmd_log) + if not args.manual_run: + job = dockerjob.DockerJob(spec) + server_jobs[lang] = job + else: + # don't run the server, set server port to a placeholder value + server_addresses[lang] = ('localhost', '${SERVER_PORT}') jobs = [] if args.cloud_to_prod: @@ -918,7 +972,8 @@ try: test_job = cloud_to_prod_jobspec( language, test_case, server_host_name, prod_servers[server_host_name], - docker_image=docker_images.get(str(language))) + docker_image=docker_images.get(str(language)), + manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) if args.http2_interop: @@ -926,7 +981,8 @@ try: test_job = cloud_to_prod_jobspec( http2Interop, test_case, server_host_name, prod_servers[server_host_name], - docker_image=docker_images.get(str(http2Interop))) + docker_image=docker_images.get(str(http2Interop)), + manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) if args.cloud_to_prod_auth: @@ -939,7 +995,8 @@ try: test_job = cloud_to_prod_jobspec( language, test_case, server_host_name, prod_servers[server_host_name], - docker_image=docker_images.get(str(language)), auth=True) + docker_image=docker_images.get(str(language)), auth=True, + manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) for server in args.override_server: @@ -963,7 +1020,8 @@ try: server_host, server_port, docker_image=docker_images.get(str(language)), - insecure=args.insecure) + insecure=args.insecure, + manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) if args.http2_interop: @@ -977,7 +1035,8 @@ try: server_host, server_port, docker_image=docker_images.get(str(http2Interop)), - insecure=args.insecure) + insecure=args.insecure, + manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) if args.http2_badserver_interop: @@ -988,7 +1047,8 @@ try: str(http2InteropServer), 'localhost', _DEFAULT_SERVER_PORT, - docker_image=docker_images.get(str(language))) + docker_image=docker_images.get(str(language)), + manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) if not jobs: @@ -997,13 +1057,20 @@ try: dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) + if args.manual_run: + print('All tests will skipped --manual_run option is active.') + num_failures, resultset = jobset.run(jobs, newline_on_success=True, - maxjobs=args.jobs) + maxjobs=args.jobs, + skip_jobs=args.manual_run) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) + write_cmdlog_maybe(server_manual_cmd_log, 'interop_server_cmds.sh') + write_cmdlog_maybe(client_manual_cmd_log, 'interop_client_cmds.sh') + report_utils.render_junit_xml_report(resultset, 'report.xml') for name, job in resultset.items(): @@ -1029,5 +1096,8 @@ finally: dockerjob.finish_jobs([j for j in server_jobs.itervalues()]) for image in docker_images.itervalues(): - print('Removing docker image %s' % image) - dockerjob.remove_image(image) + if not args.manual_run: + print('Removing docker image %s' % image) + dockerjob.remove_image(image) + else: + print('Preserving docker image: %s' % image) From 32af871b1a597edd0cd72746b3ef2b637bedda6f Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Wed, 8 Mar 2017 23:48:30 -0800 Subject: [PATCH 102/188] Fix chttp2 ping parser --- src/core/ext/transport/chttp2/transport/frame_ping.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index f487533c414..9b4b1a7b841 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -91,7 +91,7 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, grpc_chttp2_ping_parser *p = parser; while (p->byte != 8 && cur != end) { - p->opaque_8bytes |= (((uint64_t)*cur) << (8 * p->byte)); + p->opaque_8bytes |= (((uint64_t)*cur) << (56 - 8 * p->byte)); cur++; p->byte++; } From 123c72bc3c7594e31620e603dce4322c4d7c57b1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 10 Mar 2017 07:33:27 -0800 Subject: [PATCH 103/188] Handle a race between cancellation of a parent call and adding a child --- src/core/lib/surface/call.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index eafeef0a207..01efcafceba 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -163,7 +163,7 @@ struct grpc_call { bool requested_final_op; gpr_atm any_ops_sent_atm; gpr_atm received_final_op_atm; - + /* have we received initial metadata */ bool has_initial_md_been_received; @@ -343,6 +343,10 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, } if (args->propagation_mask & GRPC_PROPAGATE_CANCELLATION) { call->cancellation_is_inherited = 1; + if (gpr_atm_acq_load(&args->parent_call->received_final_op_atm)) { + cancel_with_error(exec_ctx, call, STATUS_FROM_API_OVERRIDE, + GRPC_ERROR_CANCELLED); + } } if (args->parent_call->first_child == NULL) { @@ -490,7 +494,8 @@ void grpc_call_destroy(grpc_call *c) { GPR_ASSERT(!c->destroy_called); c->destroy_called = 1; - cancel = gpr_atm_acq_load(&c->any_ops_sent_atm) && !gpr_atm_acq_load(&c->received_final_op_atm); + cancel = gpr_atm_acq_load(&c->any_ops_sent_atm) && + !gpr_atm_acq_load(&c->received_final_op_atm); if (cancel) { cancel_with_error(&exec_ctx, c, STATUS_FROM_API_OVERRIDE, GRPC_ERROR_CANCELLED); @@ -1103,7 +1108,8 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, next_child_call = child_call->sibling_next; if (child_call->cancellation_is_inherited) { GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel"); - grpc_call_cancel(child_call, NULL); + cancel_with_error(exec_ctx, call, STATUS_FROM_API_OVERRIDE, + GRPC_ERROR_CANCELLED); GRPC_CALL_INTERNAL_UNREF(exec_ctx, child_call, "propagate_cancel"); } child_call = next_child_call; From 3a35189f9b47696149f48505d0a8f18dfd7f9f89 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 10 Mar 2017 07:39:26 -0800 Subject: [PATCH 104/188] Fix build --- CMakeLists.txt | 2 ++ Makefile | 6 +++--- build.yaml | 2 ++ tools/run_tests/generated/sources_and_headers.json | 2 ++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 933426d73ca..df1565676b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8044,7 +8044,9 @@ target_link_libraries(bm_pollset ${_gRPC_ALLTARGETS_LIBRARIES} grpc_benchmark benchmark + grpc++_test_util grpc_test_util + grpc++ grpc gpr_test_util gpr diff --git a/Makefile b/Makefile index 33c330d02ed..63c76a023c4 100644 --- a/Makefile +++ b/Makefile @@ -13097,16 +13097,16 @@ $(BINDIR)/$(CONFIG)/bm_pollset: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/bm_pollset: $(PROTOBUF_DEP) $(BM_POLLSET_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/bm_pollset: $(PROTOBUF_DEP) $(BM_POLLSET_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(BM_POLLSET_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_pollset + $(Q) $(LDXX) $(LDFLAGS) $(BM_POLLSET_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_pollset endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_pollset.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_pollset.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_pollset: $(BM_POLLSET_OBJS:.o=.dep) diff --git a/build.yaml b/build.yaml index 5a8f57a2c71..5294929431f 100644 --- a/build.yaml +++ b/build.yaml @@ -3276,7 +3276,9 @@ targets: deps: - grpc_benchmark - benchmark + - grpc++_test_util - grpc_test_util + - grpc++ - grpc - gpr_test_util - gpr diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 58d929806fd..c0cd6c04964 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2645,6 +2645,8 @@ "gpr", "gpr_test_util", "grpc", + "grpc++", + "grpc++_test_util", "grpc_benchmark", "grpc_test_util" ], From ea264b88f62b0778cd360375f2404af439a07291 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 10 Mar 2017 08:35:26 -0800 Subject: [PATCH 105/188] Ensure GRPC_TIMER_END gets called Not calling this breaks our latency tracing/analysis code --- src/core/lib/iomgr/error.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index 6c4d0157c20..cbaf9a42141 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -263,7 +263,7 @@ static grpc_error *copy_error_and_unref(grpc_error *in) { else out = GRPC_ERROR_CREATE("unknown"); } else if (gpr_ref_is_unique(&in->refs)) { - return in; + out = in; } else { out = gpr_malloc(sizeof(*out)); #ifdef GRPC_ERROR_REFCOUNT_DEBUG From e736b705ccbffcf3914397638a5bf2f1f77abec3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 10 Mar 2017 09:52:25 -0800 Subject: [PATCH 106/188] Add Zalloc to bq conversion table --- tools/profiling/microbenchmarks/bm_json.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index 2548ed736f5..604823712f9 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -78,22 +78,26 @@ _BM_SPECS = { 'tpl': ['fixture'], 'dyn': [], }, - 'BM_IsolatedFilter' : { + 'BM_IsolatedFilter': { 'tpl': ['fixture', 'client_mutator'], 'dyn': [], }, - 'BM_HpackEncoderEncodeHeader' : { + 'BM_HpackEncoderEncodeHeader': { 'tpl': ['fixture'], 'dyn': ['end_of_stream', 'request_size'], }, - 'BM_HpackParserParseHeader' : { + 'BM_HpackParserParseHeader': { 'tpl': ['fixture'], 'dyn': [], }, - 'BM_CallCreateDestroy' : { + 'BM_CallCreateDestroy': { 'tpl': ['fixture'], 'dyn': [], }, + 'BM_Zalloc': { + 'tpl': [], + 'dyn': ['request_size'], + }, } def numericalize(s): From 47f37f3db33bb30496f662e77aceb87a752ac705 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 10 Mar 2017 10:44:56 -0800 Subject: [PATCH 107/188] Change execution order --- tools/run_tests/run_microbenchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index f41bde113a8..94711109680 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -231,8 +231,8 @@ argp.add_argument('--summary_time', args = argp.parse_args() try: - for bm_name in args.benchmarks: - for collect in args.collect: + for collect in args.collect: + for bm_name in args.benchmarks: collectors[collect](bm_name, args) if args.diff_perf: if 'summary' not in args.collect: From 958ded975f7e5dc0dee7767af63613466ee9ed4a Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 6 Mar 2017 00:54:05 -0800 Subject: [PATCH 108/188] PHP benchmarking performance client, proxied through Ruby Requires synchronous and single-threaded, so only 1 channel and 1 outstanding RPC per channel for now. No open-loop for now --- src/php/tests/qps/client.php | 166 +++++++ src/php/tests/qps/composer.json | 11 + .../Src/Proto/Grpc/Testing/Control.php | 127 ++++++ .../Src/Proto/Grpc/Testing/Messages.php | 69 +++ .../Src/Proto/Grpc/Testing/Payloads.php | 37 ++ .../Src/Proto/Grpc/Testing/ProxyService.php | 34 ++ .../Src/Proto/Grpc/Testing/Services.php | 45 ++ .../Src/Proto/Grpc/Testing/Stats.php | 44 ++ .../Grpc/Testing/BenchmarkServiceClient.php | 78 ++++ .../generated_code/Grpc/Testing/BoolValue.php | 62 +++ .../Grpc/Testing/ByteBufferParams.php | 65 +++ .../Grpc/Testing/ChannelArg.php | 84 ++++ .../Grpc/Testing/ClientArgs.php | 63 +++ .../Grpc/Testing/ClientConfig.php | 407 +++++++++++++++++ .../Grpc/Testing/ClientStats.php | 164 +++++++ .../Grpc/Testing/ClientStatus.php | 44 ++ .../Grpc/Testing/ClientType.php | 34 ++ .../Grpc/Testing/ClosedLoopParams.php | 28 ++ .../Grpc/Testing/ComplexProtoParams.php | 28 ++ .../Grpc/Testing/CoreRequest.php | 23 + .../Grpc/Testing/CoreResponse.php | 56 +++ .../Grpc/Testing/EchoStatus.php | 70 +++ .../Grpc/Testing/HistogramData.php | 153 +++++++ .../Grpc/Testing/HistogramParams.php | 93 ++++ .../Grpc/Testing/LoadParams.php | 63 +++ .../qps/generated_code/Grpc/Testing/Mark.php | 60 +++ .../generated_code/Grpc/Testing/Payload.php | 96 ++++ .../Grpc/Testing/PayloadConfig.php | 80 ++++ .../Grpc/Testing/PayloadType.php | 26 ++ .../Grpc/Testing/PoissonParams.php | 61 +++ .../Grpc/Testing/ProxyClientServiceClient.php | 72 +++ .../generated_code/Grpc/Testing/ProxyStat.php | 44 ++ .../Grpc/Testing/ReconnectInfo.php | 71 +++ .../Grpc/Testing/ReconnectParams.php | 49 ++ .../Grpc/Testing/RequestResultCount.php | 65 +++ .../Grpc/Testing/ResponseParameters.php | 138 ++++++ .../generated_code/Grpc/Testing/RpcType.php | 21 + .../generated_code/Grpc/Testing/Scenario.php | 291 ++++++++++++ .../Grpc/Testing/ScenarioResult.php | 312 +++++++++++++ .../Grpc/Testing/ScenarioResultSummary.php | 430 ++++++++++++++++++ .../generated_code/Grpc/Testing/Scenarios.php | 48 ++ .../Grpc/Testing/SecurityParams.php | 69 +++ .../Grpc/Testing/ServerArgs.php | 63 +++ .../Grpc/Testing/ServerConfig.php | 305 +++++++++++++ .../Grpc/Testing/ServerStats.php | 191 ++++++++ .../Grpc/Testing/ServerStatus.php | 110 +++++ .../Grpc/Testing/ServerType.php | 33 ++ .../Grpc/Testing/SimpleProtoParams.php | 65 +++ .../Grpc/Testing/SimpleRequest.php | 306 +++++++++++++ .../Grpc/Testing/SimpleResponse.php | 129 ++++++ .../Testing/StreamingInputCallRequest.php | 102 +++++ .../Testing/StreamingInputCallResponse.php | 60 +++ .../Testing/StreamingOutputCallRequest.php | 171 +++++++ .../Testing/StreamingOutputCallResponse.php | 60 +++ .../qps/generated_code/Grpc/Testing/Void.php | 23 + .../Grpc/Testing/WorkerServiceClient.php | 111 +++++ src/proto/grpc/testing/proxy-service.proto | 44 ++ src/ruby/qps/proxy-worker.rb | 160 +++++++ .../proto/grpc/testing/proxy-service_pb.rb | 17 + .../grpc/testing/proxy-service_services_pb.rb | 55 +++ 60 files changed, 5986 insertions(+) create mode 100644 src/php/tests/qps/client.php create mode 100644 src/php/tests/qps/composer.json create mode 100644 src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Control.php create mode 100644 src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Messages.php create mode 100644 src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Payloads.php create mode 100644 src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/ProxyService.php create mode 100644 src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Services.php create mode 100644 src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Stats.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/BenchmarkServiceClient.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/BoolValue.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ByteBufferParams.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ChannelArg.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ClientArgs.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ClientConfig.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ClientStats.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ClientStatus.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ClientType.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ClosedLoopParams.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ComplexProtoParams.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/CoreRequest.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/CoreResponse.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/EchoStatus.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/HistogramData.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/HistogramParams.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/LoadParams.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/Mark.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/Payload.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/PayloadConfig.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/PayloadType.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/PoissonParams.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ProxyClientServiceClient.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ProxyStat.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ReconnectInfo.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ReconnectParams.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/RequestResultCount.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ResponseParameters.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/RpcType.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/Scenario.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ScenarioResult.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ScenarioResultSummary.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/Scenarios.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/SecurityParams.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ServerArgs.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ServerConfig.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ServerStats.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ServerStatus.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/ServerType.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/SimpleProtoParams.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/SimpleRequest.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/SimpleResponse.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/StreamingInputCallRequest.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/StreamingInputCallResponse.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/StreamingOutputCallRequest.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/StreamingOutputCallResponse.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/Void.php create mode 100644 src/php/tests/qps/generated_code/Grpc/Testing/WorkerServiceClient.php create mode 100644 src/proto/grpc/testing/proxy-service.proto create mode 100755 src/ruby/qps/proxy-worker.rb create mode 100644 src/ruby/qps/src/proto/grpc/testing/proxy-service_pb.rb create mode 100644 src/ruby/qps/src/proto/grpc/testing/proxy-service_services_pb.rb diff --git a/src/php/tests/qps/client.php b/src/php/tests/qps/client.php new file mode 100644 index 00000000000..d9ca35ba43d --- /dev/null +++ b/src/php/tests/qps/client.php @@ -0,0 +1,166 @@ +code !== Grpc\STATUS_OK) { + echo "Call did not complete successfully. Status object:\n"; + var_dump($status); + exit(1); + } +} + +/* Start the actual client */ + +function qps_client_main($proxy_address) { + echo "Initiating php client\n"; + + $proxystubopts = []; + $proxystubopts['credentials'] = Grpc\ChannelCredentials::createInsecure(); + $proxystub = new Grpc\Testing\ProxyClientServiceClient($proxy_address, $proxystubopts); + list($config, $status) = $proxystub->GetConfig(new Grpc\Testing\Void())->wait(); + hardAssertIfStatusOk($status); + hardAssert($config->getClientChannels() == 1, "Only 1 channel supported"); + hardAssert($config->getOutstandingRpcsPerChannel() == 1, "Only 1 outstanding RPC supported"); + + echo "Got configuration from proxy, target is " . $config->getServerTargets()[0] . "\n"; + + $stubopts = []; + if ($config->getSecurityParams()) { + if ($config->getSecurityParams()->getUseTestCa()) { + $stubopts['credentials'] = Grpc\ChannelCredentials::createSsl( + file_get_contents(dirname(__FILE__).'/../data/ca.pem')); + } else { + $stubopts['credentials'] = Grpc\ChannelCredentials::createSsl(null); + } + $override = $config->getSecurityParams()->getServerHostOverride(); + if ($override) { + $stubopts['grpc.ssl_target_name_override'] = $override; + $stubopts['grpc.default_authority'] = $override; + } + } else { + $stubopts['credentials'] = Grpc\ChannelCredentials::createInsecure(); + } + echo "Initiating php benchmarking client\n"; + + $stub = new Grpc\Testing\BenchmarkServiceClient( + $config->getServerTargets()[0], $stubopts); + $req = new Grpc\Testing\SimpleRequest(); + + $req->setResponseType(Grpc\Testing\PayloadType::COMPRESSABLE); + $req->setResponseSize($config->getPayloadConfig()->getSimpleParams()->getRespSize()); + $payload = new Grpc\Testing\Payload(); + $payload->setType(Grpc\Testing\PayloadType::COMPRESSABLE); + $payload->setBody(str_repeat("\0", $config->getPayloadConfig()->getSimpleParams()->getReqSize())); + $req->setPayload($payload); + + /* TODO(stanley-cheung): Enable the following by removing the 0&& once protobuf + * properly supports oneof in PHP */ + if (0 && $config->getLoadParams()->getLoad() == "poisson") { + $poisson = true; + $lamrecip = 1.0/($config->getLoadParams()->getPoisson()->getOfferedLoad()); + $issue = microtime(true) + $lamrecip * -log(1.0-rand()/(getrandmax()+1)); + } else { + $poisson = false; + } + $metric = new Grpc\Testing\ProxyStat; + $telemetry = $proxystub->ReportTime(); + if ($config->getRpcType() == Grpc\Testing\RpcType::UNARY) { + while (1) { + if ($poisson) { + time_sleep_until($issue); + $issue = $issue + $lamrecip * -log(1.0-rand()/(getrandmax()+1)); + } + $startreq = microtime(true); + list($resp,$status) = $stub->UnaryCall($req)->wait(); + hardAssertIfStatusOk($status); + $metric->setLatency(microtime(true)-$startreq); + $telemetry->write($metric); + } + } else { + $stream = $stub->StreamingCall(); + while (1) { + if ($poisson) { + time_sleep_until($issue); + $issue = $issue + $lamrecip * -log(1.0-rand()/(getrandmax()+1)); + } + $startreq = microtime(true); + $stream->write($req); + $resp = $stream->read(); + $metric->setLatency(microtime(true)-$startreq); + $telemetry->write($metric); + } + } +} + +ini_set('display_startup_errors', 1); +ini_set('display_errors', 1); +error_reporting(-1); +qps_client_main($argv[1]); diff --git a/src/php/tests/qps/composer.json b/src/php/tests/qps/composer.json new file mode 100644 index 00000000000..0fc87098f57 --- /dev/null +++ b/src/php/tests/qps/composer.json @@ -0,0 +1,11 @@ +{ + "minimum-stability": "dev", + "require": { + "grpc/grpc": "dev-master" + }, + "autoload": { + "psr-4": { + "": "generated_code/" + } + } +} diff --git a/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Control.php b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Control.php new file mode 100644 index 00000000000..efca18a0cb3 --- /dev/null +++ b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Control.php @@ -0,0 +1,127 @@ +internalAddGeneratedFile(hex2bin( + "0add170a247372632f70726f746f2f677270632f74657374696e672f636f" . + "6e74726f6c2e70726f746f120c677270632e74657374696e671a25737263" . + "2f70726f746f2f677270632f74657374696e672f7061796c6f6164732e70" . + "726f746f1a227372632f70726f746f2f677270632f74657374696e672f73" . + "746174732e70726f746f22250a0d506f6973736f6e506172616d7312140a" . + "0c6f6666657265645f6c6f616418012001280122120a10436c6f7365644c" . + "6f6f70506172616d73227b0a0a4c6f6164506172616d7312350a0b636c6f" . + "7365645f6c6f6f7018012001280b321e2e677270632e74657374696e672e" . + "436c6f7365644c6f6f70506172616d734800122e0a07706f6973736f6e18" . + "022001280b321b2e677270632e74657374696e672e506f6973736f6e5061" . + "72616d73480042060a046c6f616422430a0e536563757269747950617261" . + "6d7312130a0b7573655f746573745f6361180120012808121c0a14736572" . + "7665725f686f73745f6f76657272696465180220012809224d0a0a436861" . + "6e6e656c417267120c0a046e616d6518012001280912130a097374725f76" . + "616c7565180220012809480012130a09696e745f76616c75651803200128" . + "05480042070a0576616c756522a0040a0c436c69656e74436f6e66696712" . + "160a0e7365727665725f74617267657473180120032809122d0a0b636c69" . + "656e745f7479706518022001280e32182e677270632e74657374696e672e" . + "436c69656e745479706512350a0f73656375726974795f706172616d7318" . + "032001280b321c2e677270632e74657374696e672e536563757269747950" . + "6172616d7312240a1c6f75747374616e64696e675f727063735f7065725f" . + "6368616e6e656c18042001280512170a0f636c69656e745f6368616e6e65" . + "6c73180520012805121c0a146173796e635f636c69656e745f7468726561" . + "647318072001280512270a087270635f7479706518082001280e32152e67" . + "7270632e74657374696e672e52706354797065122d0a0b6c6f61645f7061" . + "72616d73180a2001280b32182e677270632e74657374696e672e4c6f6164" . + "506172616d7312330a0e7061796c6f61645f636f6e666967180b2001280b" . + "321b2e677270632e74657374696e672e5061796c6f6164436f6e66696712" . + "370a10686973746f6772616d5f706172616d73180c2001280b321d2e6772" . + "70632e74657374696e672e486973746f6772616d506172616d7312110a09" . + "636f72655f6c697374180d2003280512120a0a636f72655f6c696d697418" . + "0e2001280512180a106f746865725f636c69656e745f617069180f200128" . + "09122e0a0c6368616e6e656c5f6172677318102003280b32182e67727063" . + "2e74657374696e672e4368616e6e656c41726722380a0c436c69656e7453" . + "746174757312280a05737461747318012001280b32192e677270632e7465" . + "7374696e672e436c69656e74537461747322150a044d61726b120d0a0572" . + "6573657418012001280822680a0a436c69656e7441726773122b0a057365" . + "74757018012001280b321a2e677270632e74657374696e672e436c69656e" . + "74436f6e666967480012220a046d61726b18022001280b32122e67727063" . + "2e74657374696e672e4d61726b480042090a076172677479706522b4020a" . + "0c536572766572436f6e666967122d0a0b7365727665725f747970651801" . + "2001280e32182e677270632e74657374696e672e53657276657254797065" . + "12350a0f73656375726974795f706172616d7318022001280b321c2e6772" . + "70632e74657374696e672e5365637572697479506172616d73120c0a0470" . + "6f7274180420012805121c0a146173796e635f7365727665725f74687265" . + "61647318072001280512120a0a636f72655f6c696d697418082001280512" . + "330a0e7061796c6f61645f636f6e66696718092001280b321b2e67727063" . + "2e74657374696e672e5061796c6f6164436f6e66696712110a09636f7265" . + "5f6c697374180a2003280512180a106f746865725f7365727665725f6170" . + "69180b20012809121c0a137265736f757263655f71756f74615f73697a65" . + "18e9072001280522680a0a53657276657241726773122b0a057365747570" . + "18012001280b321a2e677270632e74657374696e672e536572766572436f" . + "6e666967480012220a046d61726b18022001280b32122e677270632e7465" . + "7374696e672e4d61726b480042090a076172677479706522550a0c536572" . + "76657253746174757312280a05737461747318012001280b32192e677270" . + "632e74657374696e672e5365727665725374617473120c0a04706f727418" . + "0220012805120d0a05636f726573180320012805220d0a0b436f72655265" . + "7175657374221d0a0c436f7265526573706f6e7365120d0a05636f726573" . + "18012001280522060a04566f696422fd010a085363656e6172696f120c0a" . + "046e616d6518012001280912310a0d636c69656e745f636f6e6669671802" . + "2001280b321a2e677270632e74657374696e672e436c69656e74436f6e66" . + "696712130a0b6e756d5f636c69656e747318032001280512310a0d736572" . + "7665725f636f6e66696718042001280b321a2e677270632e74657374696e" . + "672e536572766572436f6e66696712130a0b6e756d5f7365727665727318" . + "052001280512160a0e7761726d75705f7365636f6e647318062001280512" . + "190a1162656e63686d61726b5f7365636f6e647318072001280512200a18" . + "737061776e5f6c6f63616c5f776f726b65725f636f756e74180820012805" . + "22360a095363656e6172696f7312290a097363656e6172696f7318012003" . + "280b32162e677270632e74657374696e672e5363656e6172696f22f8020a" . + "155363656e6172696f526573756c7453756d6d617279120b0a0371707318" . + "0120012801121b0a137170735f7065725f7365727665725f636f72651802" . + "20012801121a0a127365727665725f73797374656d5f74696d6518032001" . + "280112180a107365727665725f757365725f74696d65180420012801121a" . + "0a12636c69656e745f73797374656d5f74696d6518052001280112180a10" . + "636c69656e745f757365725f74696d6518062001280112120a0a6c617465" . + "6e63795f353018072001280112120a0a6c6174656e63795f393018082001" . + "280112120a0a6c6174656e63795f393518092001280112120a0a6c617465" . + "6e63795f3939180a2001280112130a0b6c6174656e63795f393939180b20" . + "01280112180a107365727665725f6370755f7573616765180c2001280112" . + "260a1e7375636365737366756c5f72657175657374735f7065725f736563" . + "6f6e64180d2001280112220a1a6661696c65645f72657175657374735f70" . + "65725f7365636f6e64180e200128012283030a0e5363656e6172696f5265" . + "73756c7412280a087363656e6172696f18012001280b32162e677270632e" . + "74657374696e672e5363656e6172696f122e0a096c6174656e6369657318" . + "022001280b321b2e677270632e74657374696e672e486973746f6772616d" . + "44617461122f0a0c636c69656e745f737461747318032003280b32192e67" . + "7270632e74657374696e672e436c69656e745374617473122f0a0c736572" . + "7665725f737461747318042003280b32192e677270632e74657374696e67" . + "2e536572766572537461747312140a0c7365727665725f636f7265731805" . + "2003280512340a0773756d6d61727918062001280b32232e677270632e74" . + "657374696e672e5363656e6172696f526573756c7453756d6d6172791216" . + "0a0e636c69656e745f7375636365737318072003280812160a0e73657276" . + "65725f7375636365737318082003280812390a0f726571756573745f7265" . + "73756c747318092003280b32202e677270632e74657374696e672e526571" . + "75657374526573756c74436f756e742a410a0a436c69656e745479706512" . + "0f0a0b53594e435f434c49454e54100012100a0c4153594e435f434c4945" . + "4e54100112100a0c4f544845525f434c49454e5410022a5b0a0a53657276" . + "657254797065120f0a0b53594e435f534552564552100012100a0c415359" . + "4e435f534552564552100112180a144153594e435f47454e455249435f53" . + "4552564552100212100a0c4f544845525f53455256455210032a230a0752" . + "70635479706512090a05554e4152591000120d0a0953545245414d494e47" . + "1001620670726f746f33" + )); + + static::$is_initialized = true; + } +} + diff --git a/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Messages.php b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Messages.php new file mode 100644 index 00000000000..c0880026264 --- /dev/null +++ b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Messages.php @@ -0,0 +1,69 @@ +internalAddGeneratedFile(hex2bin( + "0ad50a0a257372632f70726f746f2f677270632f74657374696e672f6d65" . + "7373616765732e70726f746f120c677270632e74657374696e67221a0a09" . + "426f6f6c56616c7565120d0a0576616c756518012001280822400a075061" . + "796c6f616412270a047479706518012001280e32192e677270632e746573" . + "74696e672e5061796c6f616454797065120c0a04626f647918022001280c" . + "222b0a0a4563686f537461747573120c0a04636f6465180120012805120f" . + "0a076d65737361676518022001280922ce020a0d53696d706c6552657175" . + "65737412300a0d726573706f6e73655f7479706518012001280e32192e67" . + "7270632e74657374696e672e5061796c6f61645479706512150a0d726573" . + "706f6e73655f73697a6518022001280512260a077061796c6f6164180320" . + "01280b32152e677270632e74657374696e672e5061796c6f616412150a0d" . + "66696c6c5f757365726e616d6518042001280812180a1066696c6c5f6f61" . + "7574685f73636f706518052001280812340a13726573706f6e73655f636f" . + "6d7072657373656418062001280b32172e677270632e74657374696e672e" . + "426f6f6c56616c756512310a0f726573706f6e73655f7374617475731807" . + "2001280b32182e677270632e74657374696e672e4563686f537461747573" . + "12320a116578706563745f636f6d7072657373656418082001280b32172e" . + "677270632e74657374696e672e426f6f6c56616c7565225f0a0e53696d70" . + "6c65526573706f6e736512260a077061796c6f616418012001280b32152e" . + "677270632e74657374696e672e5061796c6f616412100a08757365726e61" . + "6d6518022001280912130a0b6f617574685f73636f706518032001280922" . + "770a1953747265616d696e67496e70757443616c6c526571756573741226" . + "0a077061796c6f616418012001280b32152e677270632e74657374696e67" . + "2e5061796c6f616412320a116578706563745f636f6d7072657373656418" . + "022001280b32172e677270632e74657374696e672e426f6f6c56616c7565" . + "223d0a1a53747265616d696e67496e70757443616c6c526573706f6e7365" . + "121f0a17616767726567617465645f7061796c6f61645f73697a65180120" . + "01280522640a12526573706f6e7365506172616d6574657273120c0a0473" . + "697a6518012001280512130a0b696e74657276616c5f7573180220012805" . + "122b0a0a636f6d7072657373656418032001280b32172e677270632e7465" . + "7374696e672e426f6f6c56616c756522e8010a1a53747265616d696e674f" . + "757470757443616c6c5265717565737412300a0d726573706f6e73655f74" . + "79706518012001280e32192e677270632e74657374696e672e5061796c6f" . + "616454797065123d0a13726573706f6e73655f706172616d657465727318" . + "022003280b32202e677270632e74657374696e672e526573706f6e736550" . + "6172616d657465727312260a077061796c6f616418032001280b32152e67" . + "7270632e74657374696e672e5061796c6f616412310a0f726573706f6e73" . + "655f73746174757318072001280b32182e677270632e74657374696e672e" . + "4563686f53746174757322450a1b53747265616d696e674f757470757443" . + "616c6c526573706f6e736512260a077061796c6f616418012001280b3215" . + "2e677270632e74657374696e672e5061796c6f616422330a0f5265636f6e" . + "6e656374506172616d7312200a186d61785f7265636f6e6e6563745f6261" . + "636b6f66665f6d7318012001280522330a0d5265636f6e6e656374496e66" . + "6f120e0a0670617373656418012001280812120a0a6261636b6f66665f6d" . + "731802200328052a1f0a0b5061796c6f61645479706512100a0c434f4d50" . + "5245535341424c451000620670726f746f33" + )); + + static::$is_initialized = true; + } +} + diff --git a/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Payloads.php b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Payloads.php new file mode 100644 index 00000000000..279fe00ac82 --- /dev/null +++ b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Payloads.php @@ -0,0 +1,37 @@ +internalAddGeneratedFile(hex2bin( + "0a93030a257372632f70726f746f2f677270632f74657374696e672f7061" . + "796c6f6164732e70726f746f120c677270632e74657374696e6722370a10" . + "42797465427566666572506172616d7312100a087265715f73697a651801" . + "2001280512110a09726573705f73697a6518022001280522380a1153696d" . + "706c6550726f746f506172616d7312100a087265715f73697a6518012001" . + "280512110a09726573705f73697a6518022001280522140a12436f6d706c" . + "657850726f746f506172616d7322ca010a0d5061796c6f6164436f6e6669" . + "6712380a0e627974656275665f706172616d7318012001280b321e2e6772" . + "70632e74657374696e672e42797465427566666572506172616d73480012" . + "380a0d73696d706c655f706172616d7318022001280b321f2e677270632e" . + "74657374696e672e53696d706c6550726f746f506172616d734800123a0a" . + "0e636f6d706c65785f706172616d7318032001280b32202e677270632e74" . + "657374696e672e436f6d706c657850726f746f506172616d73480042090a" . + "077061796c6f6164620670726f746f33" + )); + + static::$is_initialized = true; + } +} + diff --git a/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/ProxyService.php b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/ProxyService.php new file mode 100644 index 00000000000..e35944e1d82 --- /dev/null +++ b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/ProxyService.php @@ -0,0 +1,34 @@ +internalAddGeneratedFile(hex2bin( + "0a97020a2a7372632f70726f746f2f677270632f74657374696e672f7072" . + "6f78792d736572766963652e70726f746f120c677270632e74657374696e" . + "671a247372632f70726f746f2f677270632f74657374696e672f636f6e74" . + "726f6c2e70726f746f221c0a0950726f787953746174120f0a076c617465" . + "6e6379180120012801328e010a1250726f7879436c69656e745365727669" . + "6365123b0a09476574436f6e66696712122e677270632e74657374696e67" . + "2e566f69641a1a2e677270632e74657374696e672e436c69656e74436f6e" . + "666967123b0a0a5265706f727454696d6512172e677270632e7465737469" . + "6e672e50726f7879537461741a122e677270632e74657374696e672e566f" . + "69642801620670726f746f33" + )); + + static::$is_initialized = true; + } +} + diff --git a/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Services.php b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Services.php new file mode 100644 index 00000000000..7a9439a5b93 --- /dev/null +++ b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Services.php @@ -0,0 +1,45 @@ +internalAddGeneratedFile(hex2bin( + "0ad1040a257372632f70726f746f2f677270632f74657374696e672f7365" . + "7276696365732e70726f746f120c677270632e74657374696e671a257372" . + "632f70726f746f2f677270632f74657374696e672f6d657373616765732e" . + "70726f746f1a247372632f70726f746f2f677270632f74657374696e672f" . + "636f6e74726f6c2e70726f746f32aa010a1042656e63686d61726b536572" . + "7669636512460a09556e61727943616c6c121b2e677270632e7465737469" . + "6e672e53696d706c65526571756573741a1c2e677270632e74657374696e" . + "672e53696d706c65526573706f6e7365124e0a0d53747265616d696e6743" . + "616c6c121b2e677270632e74657374696e672e53696d706c655265717565" . + "73741a1c2e677270632e74657374696e672e53696d706c65526573706f6e" . + "7365280130013297020a0d576f726b65725365727669636512450a095275" . + "6e53657276657212182e677270632e74657374696e672e53657276657241" . + "7267731a1a2e677270632e74657374696e672e5365727665725374617475" . + "732801300112450a0952756e436c69656e7412182e677270632e74657374" . + "696e672e436c69656e74417267731a1a2e677270632e74657374696e672e" . + "436c69656e745374617475732801300112420a09436f7265436f756e7412" . + "192e677270632e74657374696e672e436f7265526571756573741a1a2e67" . + "7270632e74657374696e672e436f7265526573706f6e736512340a0a5175" . + "6974576f726b657212122e677270632e74657374696e672e566f69641a12" . + "2e677270632e74657374696e672e566f6964620670726f746f33" + )); + + static::$is_initialized = true; + } +} + diff --git a/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Stats.php b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Stats.php new file mode 100644 index 00000000000..99c0000a52c --- /dev/null +++ b/src/php/tests/qps/generated_code/GPBMetadata/Src/Proto/Grpc/Testing/Stats.php @@ -0,0 +1,44 @@ +internalAddGeneratedFile(hex2bin( + "0adf040a227372632f70726f746f2f677270632f74657374696e672f7374" . + "6174732e70726f746f120c677270632e74657374696e67227a0a0b536572" . + "766572537461747312140a0c74696d655f656c6170736564180120012801" . + "12110a0974696d655f7573657218022001280112130a0b74696d655f7379" . + "7374656d18032001280112160a0e746f74616c5f6370755f74696d651804" . + "2001280412150a0d69646c655f6370755f74696d65180520012804223b0a" . + "0f486973746f6772616d506172616d7312120a0a7265736f6c7574696f6e" . + "18012001280112140a0c6d61785f706f737369626c651802200128012277" . + "0a0d486973746f6772616d44617461120e0a066275636b65741801200328" . + "0d12100a086d696e5f7365656e18022001280112100a086d61785f736565" . + "6e180320012801120b0a0373756d18042001280112160a0e73756d5f6f66" . + "5f73717561726573180520012801120d0a05636f756e7418062001280122" . + "380a1252657175657374526573756c74436f756e7412130a0b7374617475" . + "735f636f6465180120012805120d0a05636f756e7418022001280322b601" . + "0a0b436c69656e745374617473122e0a096c6174656e6369657318012001" . + "280b321b2e677270632e74657374696e672e486973746f6772616d446174" . + "6112140a0c74696d655f656c617073656418022001280112110a0974696d" . + "655f7573657218032001280112130a0b74696d655f73797374656d180420" . + "01280112390a0f726571756573745f726573756c747318052003280b3220" . + "2e677270632e74657374696e672e52657175657374526573756c74436f75" . + "6e74620670726f746f33" + )); + + static::$is_initialized = true; + } +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/BenchmarkServiceClient.php b/src/php/tests/qps/generated_code/Grpc/Testing/BenchmarkServiceClient.php new file mode 100644 index 00000000000..daf17800cdc --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/BenchmarkServiceClient.php @@ -0,0 +1,78 @@ +_simpleRequest('/grpc.testing.BenchmarkService/UnaryCall', + $argument, + ['\Grpc\Testing\SimpleResponse', 'decode'], + $metadata, $options); + } + + /** + * One request followed by one response. + * The server returns the client payload as-is. + * @param array $metadata metadata + * @param array $options call options + */ + public function StreamingCall($metadata = [], $options = []) { + return $this->_bidiRequest('/grpc.testing.BenchmarkService/StreamingCall', + ['\Grpc\Testing\SimpleResponse','decode'], + $metadata, $options); + } + + } + +} diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/BoolValue.php b/src/php/tests/qps/generated_code/Grpc/Testing/BoolValue.php new file mode 100644 index 00000000000..f0497accfb2 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/BoolValue.php @@ -0,0 +1,62 @@ + + * TODO(dgq): Go back to using well-known types once + * https://github.com/grpc/grpc/issues/6980 has been fixed. + * import "google/protobuf/wrappers.proto"; + * + * + * Protobuf type grpc.testing.BoolValue + */ +class BoolValue extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * The bool value.
+     *
+ * + * bool value = 1; + */ + private $value = false; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * 
+     * The bool value.
+     *
+ * + * bool value = 1; + */ + public function getValue() + { + return $this->value; + } + + /** + * 
+     * The bool value.
+     *
+ * + * bool value = 1; + */ + public function setValue($var) + { + GPBUtil::checkBool($var); + $this->value = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ByteBufferParams.php b/src/php/tests/qps/generated_code/Grpc/Testing/ByteBufferParams.php new file mode 100644 index 00000000000..0057d387488 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ByteBufferParams.php @@ -0,0 +1,65 @@ +grpc.testing.ByteBufferParams + */ +class ByteBufferParams extends \Google\Protobuf\Internal\Message +{ + /** + * int32 req_size = 1; + */ + private $req_size = 0; + /** + * int32 resp_size = 2; + */ + private $resp_size = 0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Payloads::initOnce(); + parent::__construct(); + } + + /** + * int32 req_size = 1; + */ + public function getReqSize() + { + return $this->req_size; + } + + /** + * int32 req_size = 1; + */ + public function setReqSize($var) + { + GPBUtil::checkInt32($var); + $this->req_size = $var; + } + + /** + * int32 resp_size = 2; + */ + public function getRespSize() + { + return $this->resp_size; + } + + /** + * int32 resp_size = 2; + */ + public function setRespSize($var) + { + GPBUtil::checkInt32($var); + $this->resp_size = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ChannelArg.php b/src/php/tests/qps/generated_code/Grpc/Testing/ChannelArg.php new file mode 100644 index 00000000000..d2fe3ae5ffc --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ChannelArg.php @@ -0,0 +1,84 @@ +grpc.testing.ChannelArg + */ +class ChannelArg extends \Google\Protobuf\Internal\Message +{ + /** + * string name = 1; + */ + private $name = ''; + protected $value; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * string name = 1; + */ + public function getName() + { + return $this->name; + } + + /** + * string name = 1; + */ + public function setName($var) + { + GPBUtil::checkString($var, True); + $this->name = $var; + } + + /** + * string str_value = 2; + */ + public function getStrValue() + { + return $this->readOneof(2); + } + + /** + * string str_value = 2; + */ + public function setStrValue($var) + { + GPBUtil::checkString($var, True); + $this->writeOneof(2, $var); + } + + /** + * int32 int_value = 3; + */ + public function getIntValue() + { + return $this->readOneof(3); + } + + /** + * int32 int_value = 3; + */ + public function setIntValue($var) + { + GPBUtil::checkInt32($var); + $this->writeOneof(3, $var); + } + + public function getValue() + { + return $this->whichOneof("value"); + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ClientArgs.php b/src/php/tests/qps/generated_code/Grpc/Testing/ClientArgs.php new file mode 100644 index 00000000000..c878c5a7bc0 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ClientArgs.php @@ -0,0 +1,63 @@ +grpc.testing.ClientArgs + */ +class ClientArgs extends \Google\Protobuf\Internal\Message +{ + protected $argtype; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * .grpc.testing.ClientConfig setup = 1; + */ + public function getSetup() + { + return $this->readOneof(1); + } + + /** + * .grpc.testing.ClientConfig setup = 1; + */ + public function setSetup(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\ClientConfig::class); + $this->writeOneof(1, $var); + } + + /** + * .grpc.testing.Mark mark = 2; + */ + public function getMark() + { + return $this->readOneof(2); + } + + /** + * .grpc.testing.Mark mark = 2; + */ + public function setMark(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\Mark::class); + $this->writeOneof(2, $var); + } + + public function getArgtype() + { + return $this->whichOneof("argtype"); + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ClientConfig.php b/src/php/tests/qps/generated_code/Grpc/Testing/ClientConfig.php new file mode 100644 index 00000000000..52d6a75fb0e --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ClientConfig.php @@ -0,0 +1,407 @@ +grpc.testing.ClientConfig + */ +class ClientConfig extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * List of targets to connect to. At least one target needs to be specified.
+     *
+ * + * repeated string server_targets = 1; + */ + private $server_targets; + /** + * .grpc.testing.ClientType client_type = 2; + */ + private $client_type = 0; + /** + * .grpc.testing.SecurityParams security_params = 3; + */ + private $security_params = null; + /** + * 
+     * How many concurrent RPCs to start for each channel.
+     * For synchronous client, use a separate thread for each outstanding RPC.
+     *
+ * + * int32 outstanding_rpcs_per_channel = 4; + */ + private $outstanding_rpcs_per_channel = 0; + /** + * 
+     * Number of independent client channels to create.
+     * i-th channel will connect to server_target[i % server_targets.size()]
+     *
+ * + * int32 client_channels = 5; + */ + private $client_channels = 0; + /** + * 
+     * Only for async client. Number of threads to use to start/manage RPCs.
+     *
+ * + * int32 async_client_threads = 7; + */ + private $async_client_threads = 0; + /** + * .grpc.testing.RpcType rpc_type = 8; + */ + private $rpc_type = 0; + /** + * 
+     * The requested load for the entire client (aggregated over all the threads).
+     *
+ * + * .grpc.testing.LoadParams load_params = 10; + */ + private $load_params = null; + /** + * .grpc.testing.PayloadConfig payload_config = 11; + */ + private $payload_config = null; + /** + * .grpc.testing.HistogramParams histogram_params = 12; + */ + private $histogram_params = null; + /** + * 
+     * Specify the cores we should run the client on, if desired
+     *
+ * + * repeated int32 core_list = 13; + */ + private $core_list; + /** + * int32 core_limit = 14; + */ + private $core_limit = 0; + /** + * 
+     * If we use an OTHER_CLIENT client_type, this string gives more detail
+     *
+ * + * string other_client_api = 15; + */ + private $other_client_api = ''; + /** + * repeated .grpc.testing.ChannelArg channel_args = 16; + */ + private $channel_args; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * 
+     * List of targets to connect to. At least one target needs to be specified.
+     *
+ * + * repeated string server_targets = 1; + */ + public function getServerTargets() + { + return $this->server_targets; + } + + /** + * 
+     * List of targets to connect to. At least one target needs to be specified.
+     *
+ * + * repeated string server_targets = 1; + */ + public function setServerTargets(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::STRING); + $this->server_targets = $var; + } + + /** + * .grpc.testing.ClientType client_type = 2; + */ + public function getClientType() + { + return $this->client_type; + } + + /** + * .grpc.testing.ClientType client_type = 2; + */ + public function setClientType($var) + { + GPBUtil::checkEnum($var, \Grpc\Testing\ClientType::class); + $this->client_type = $var; + } + + /** + * .grpc.testing.SecurityParams security_params = 3; + */ + public function getSecurityParams() + { + return $this->security_params; + } + + /** + * .grpc.testing.SecurityParams security_params = 3; + */ + public function setSecurityParams(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\SecurityParams::class); + $this->security_params = $var; + } + + /** + * 
+     * How many concurrent RPCs to start for each channel.
+     * For synchronous client, use a separate thread for each outstanding RPC.
+     *
+ * + * int32 outstanding_rpcs_per_channel = 4; + */ + public function getOutstandingRpcsPerChannel() + { + return $this->outstanding_rpcs_per_channel; + } + + /** + * 
+     * How many concurrent RPCs to start for each channel.
+     * For synchronous client, use a separate thread for each outstanding RPC.
+     *
+ * + * int32 outstanding_rpcs_per_channel = 4; + */ + public function setOutstandingRpcsPerChannel($var) + { + GPBUtil::checkInt32($var); + $this->outstanding_rpcs_per_channel = $var; + } + + /** + * 
+     * Number of independent client channels to create.
+     * i-th channel will connect to server_target[i % server_targets.size()]
+     *
+ * + * int32 client_channels = 5; + */ + public function getClientChannels() + { + return $this->client_channels; + } + + /** + * 
+     * Number of independent client channels to create.
+     * i-th channel will connect to server_target[i % server_targets.size()]
+     *
+ * + * int32 client_channels = 5; + */ + public function setClientChannels($var) + { + GPBUtil::checkInt32($var); + $this->client_channels = $var; + } + + /** + * 
+     * Only for async client. Number of threads to use to start/manage RPCs.
+     *
+ * + * int32 async_client_threads = 7; + */ + public function getAsyncClientThreads() + { + return $this->async_client_threads; + } + + /** + * 
+     * Only for async client. Number of threads to use to start/manage RPCs.
+     *
+ * + * int32 async_client_threads = 7; + */ + public function setAsyncClientThreads($var) + { + GPBUtil::checkInt32($var); + $this->async_client_threads = $var; + } + + /** + * .grpc.testing.RpcType rpc_type = 8; + */ + public function getRpcType() + { + return $this->rpc_type; + } + + /** + * .grpc.testing.RpcType rpc_type = 8; + */ + public function setRpcType($var) + { + GPBUtil::checkEnum($var, \Grpc\Testing\RpcType::class); + $this->rpc_type = $var; + } + + /** + * 
+     * The requested load for the entire client (aggregated over all the threads).
+     *
+ * + * .grpc.testing.LoadParams load_params = 10; + */ + public function getLoadParams() + { + return $this->load_params; + } + + /** + * 
+     * The requested load for the entire client (aggregated over all the threads).
+     *
+ * + * .grpc.testing.LoadParams load_params = 10; + */ + public function setLoadParams(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\LoadParams::class); + $this->load_params = $var; + } + + /** + * .grpc.testing.PayloadConfig payload_config = 11; + */ + public function getPayloadConfig() + { + return $this->payload_config; + } + + /** + * .grpc.testing.PayloadConfig payload_config = 11; + */ + public function setPayloadConfig(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\PayloadConfig::class); + $this->payload_config = $var; + } + + /** + * .grpc.testing.HistogramParams histogram_params = 12; + */ + public function getHistogramParams() + { + return $this->histogram_params; + } + + /** + * .grpc.testing.HistogramParams histogram_params = 12; + */ + public function setHistogramParams(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\HistogramParams::class); + $this->histogram_params = $var; + } + + /** + * 
+     * Specify the cores we should run the client on, if desired
+     *
+ * + * repeated int32 core_list = 13; + */ + public function getCoreList() + { + return $this->core_list; + } + + /** + * 
+     * Specify the cores we should run the client on, if desired
+     *
+ * + * repeated int32 core_list = 13; + */ + public function setCoreList(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::INT32); + $this->core_list = $var; + } + + /** + * int32 core_limit = 14; + */ + public function getCoreLimit() + { + return $this->core_limit; + } + + /** + * int32 core_limit = 14; + */ + public function setCoreLimit($var) + { + GPBUtil::checkInt32($var); + $this->core_limit = $var; + } + + /** + * 
+     * If we use an OTHER_CLIENT client_type, this string gives more detail
+     *
+ * + * string other_client_api = 15; + */ + public function getOtherClientApi() + { + return $this->other_client_api; + } + + /** + * 
+     * If we use an OTHER_CLIENT client_type, this string gives more detail
+     *
+ * + * string other_client_api = 15; + */ + public function setOtherClientApi($var) + { + GPBUtil::checkString($var, True); + $this->other_client_api = $var; + } + + /** + * repeated .grpc.testing.ChannelArg channel_args = 16; + */ + public function getChannelArgs() + { + return $this->channel_args; + } + + /** + * repeated .grpc.testing.ChannelArg channel_args = 16; + */ + public function setChannelArgs(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::MESSAGE, \Grpc\Testing\ChannelArg::class); + $this->channel_args = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ClientStats.php b/src/php/tests/qps/generated_code/Grpc/Testing/ClientStats.php new file mode 100644 index 00000000000..8b9a0c33a46 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ClientStats.php @@ -0,0 +1,164 @@ +grpc.testing.ClientStats + */ +class ClientStats extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * Latency histogram. Data points are in nanoseconds.
+     *
+ * + * .grpc.testing.HistogramData latencies = 1; + */ + private $latencies = null; + /** + * 
+     * See ServerStats for details.
+     *
+ * + * double time_elapsed = 2; + */ + private $time_elapsed = 0.0; + /** + * double time_user = 3; + */ + private $time_user = 0.0; + /** + * double time_system = 4; + */ + private $time_system = 0.0; + /** + * 
+     * Number of failed requests (one row per status code seen)
+     *
+ * + * repeated .grpc.testing.RequestResultCount request_results = 5; + */ + private $request_results; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Stats::initOnce(); + parent::__construct(); + } + + /** + * 
+     * Latency histogram. Data points are in nanoseconds.
+     *
+ * + * .grpc.testing.HistogramData latencies = 1; + */ + public function getLatencies() + { + return $this->latencies; + } + + /** + * 
+     * Latency histogram. Data points are in nanoseconds.
+     *
+ * + * .grpc.testing.HistogramData latencies = 1; + */ + public function setLatencies(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\HistogramData::class); + $this->latencies = $var; + } + + /** + * 
+     * See ServerStats for details.
+     *
+ * + * double time_elapsed = 2; + */ + public function getTimeElapsed() + { + return $this->time_elapsed; + } + + /** + * 
+     * See ServerStats for details.
+     *
+ * + * double time_elapsed = 2; + */ + public function setTimeElapsed($var) + { + GPBUtil::checkDouble($var); + $this->time_elapsed = $var; + } + + /** + * double time_user = 3; + */ + public function getTimeUser() + { + return $this->time_user; + } + + /** + * double time_user = 3; + */ + public function setTimeUser($var) + { + GPBUtil::checkDouble($var); + $this->time_user = $var; + } + + /** + * double time_system = 4; + */ + public function getTimeSystem() + { + return $this->time_system; + } + + /** + * double time_system = 4; + */ + public function setTimeSystem($var) + { + GPBUtil::checkDouble($var); + $this->time_system = $var; + } + + /** + * 
+     * Number of failed requests (one row per status code seen)
+     *
+ * + * repeated .grpc.testing.RequestResultCount request_results = 5; + */ + public function getRequestResults() + { + return $this->request_results; + } + + /** + * 
+     * Number of failed requests (one row per status code seen)
+     *
+ * + * repeated .grpc.testing.RequestResultCount request_results = 5; + */ + public function setRequestResults(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::MESSAGE, \Grpc\Testing\RequestResultCount::class); + $this->request_results = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ClientStatus.php b/src/php/tests/qps/generated_code/Grpc/Testing/ClientStatus.php new file mode 100644 index 00000000000..a59f87a9628 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ClientStatus.php @@ -0,0 +1,44 @@ +grpc.testing.ClientStatus + */ +class ClientStatus extends \Google\Protobuf\Internal\Message +{ + /** + * .grpc.testing.ClientStats stats = 1; + */ + private $stats = null; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * .grpc.testing.ClientStats stats = 1; + */ + public function getStats() + { + return $this->stats; + } + + /** + * .grpc.testing.ClientStats stats = 1; + */ + public function setStats(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\ClientStats::class); + $this->stats = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ClientType.php b/src/php/tests/qps/generated_code/Grpc/Testing/ClientType.php new file mode 100644 index 00000000000..4f59da992f9 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ClientType.php @@ -0,0 +1,34 @@ +grpc.testing.ClientType + */ +class ClientType +{ + /** + * 
+     * Many languages support a basic distinction between using
+     * sync or async client, and this allows the specification
+     *
+ * + * SYNC_CLIENT = 0; + */ + const SYNC_CLIENT = 0; + /** + * ASYNC_CLIENT = 1; + */ + const ASYNC_CLIENT = 1; + /** + * 
+     * used for some language-specific variants
+     *
+ * + * OTHER_CLIENT = 2; + */ + const OTHER_CLIENT = 2; +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ClosedLoopParams.php b/src/php/tests/qps/generated_code/Grpc/Testing/ClosedLoopParams.php new file mode 100644 index 00000000000..53f2948af28 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ClosedLoopParams.php @@ -0,0 +1,28 @@ + + * Once an RPC finishes, immediately start a new one. + * No configuration parameters needed. + * + * + * Protobuf type grpc.testing.ClosedLoopParams + */ +class ClosedLoopParams extends \Google\Protobuf\Internal\Message +{ + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ComplexProtoParams.php b/src/php/tests/qps/generated_code/Grpc/Testing/ComplexProtoParams.php new file mode 100644 index 00000000000..6d990f1b064 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ComplexProtoParams.php @@ -0,0 +1,28 @@ + + * TODO (vpai): Fill this in once the details of complex, representative + * protos are decided + * + * + * Protobuf type grpc.testing.ComplexProtoParams + */ +class ComplexProtoParams extends \Google\Protobuf\Internal\Message +{ + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Payloads::initOnce(); + parent::__construct(); + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/CoreRequest.php b/src/php/tests/qps/generated_code/Grpc/Testing/CoreRequest.php new file mode 100644 index 00000000000..2e078b3fcdb --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/CoreRequest.php @@ -0,0 +1,23 @@ +grpc.testing.CoreRequest + */ +class CoreRequest extends \Google\Protobuf\Internal\Message +{ + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/CoreResponse.php b/src/php/tests/qps/generated_code/Grpc/Testing/CoreResponse.php new file mode 100644 index 00000000000..85cb3418ada --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/CoreResponse.php @@ -0,0 +1,56 @@ +grpc.testing.CoreResponse + */ +class CoreResponse extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * Number of cores available on the server
+     *
+ * + * int32 cores = 1; + */ + private $cores = 0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * 
+     * Number of cores available on the server
+     *
+ * + * int32 cores = 1; + */ + public function getCores() + { + return $this->cores; + } + + /** + * 
+     * Number of cores available on the server
+     *
+ * + * int32 cores = 1; + */ + public function setCores($var) + { + GPBUtil::checkInt32($var); + $this->cores = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/EchoStatus.php b/src/php/tests/qps/generated_code/Grpc/Testing/EchoStatus.php new file mode 100644 index 00000000000..27340fb0efe --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/EchoStatus.php @@ -0,0 +1,70 @@ + + * A protobuf representation for grpc status. This is used by test + * clients to specify a status that the server should attempt to return. + * + * + * Protobuf type grpc.testing.EchoStatus + */ +class EchoStatus extends \Google\Protobuf\Internal\Message +{ + /** + * int32 code = 1; + */ + private $code = 0; + /** + * string message = 2; + */ + private $message = ''; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * int32 code = 1; + */ + public function getCode() + { + return $this->code; + } + + /** + * int32 code = 1; + */ + public function setCode($var) + { + GPBUtil::checkInt32($var); + $this->code = $var; + } + + /** + * string message = 2; + */ + public function getMessage() + { + return $this->message; + } + + /** + * string message = 2; + */ + public function setMessage($var) + { + GPBUtil::checkString($var, True); + $this->message = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/HistogramData.php b/src/php/tests/qps/generated_code/Grpc/Testing/HistogramData.php new file mode 100644 index 00000000000..056da6e5de5 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/HistogramData.php @@ -0,0 +1,153 @@ + + * Histogram data based on grpc/support/histogram.c + * + * + * Protobuf type grpc.testing.HistogramData + */ +class HistogramData extends \Google\Protobuf\Internal\Message +{ + /** + * repeated uint32 bucket = 1; + */ + private $bucket; + /** + * double min_seen = 2; + */ + private $min_seen = 0.0; + /** + * double max_seen = 3; + */ + private $max_seen = 0.0; + /** + * double sum = 4; + */ + private $sum = 0.0; + /** + * double sum_of_squares = 5; + */ + private $sum_of_squares = 0.0; + /** + * double count = 6; + */ + private $count = 0.0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Stats::initOnce(); + parent::__construct(); + } + + /** + * repeated uint32 bucket = 1; + */ + public function getBucket() + { + return $this->bucket; + } + + /** + * repeated uint32 bucket = 1; + */ + public function setBucket(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::UINT32); + $this->bucket = $var; + } + + /** + * double min_seen = 2; + */ + public function getMinSeen() + { + return $this->min_seen; + } + + /** + * double min_seen = 2; + */ + public function setMinSeen($var) + { + GPBUtil::checkDouble($var); + $this->min_seen = $var; + } + + /** + * double max_seen = 3; + */ + public function getMaxSeen() + { + return $this->max_seen; + } + + /** + * double max_seen = 3; + */ + public function setMaxSeen($var) + { + GPBUtil::checkDouble($var); + $this->max_seen = $var; + } + + /** + * double sum = 4; + */ + public function getSum() + { + return $this->sum; + } + + /** + * double sum = 4; + */ + public function setSum($var) + { + GPBUtil::checkDouble($var); + $this->sum = $var; + } + + /** + * double sum_of_squares = 5; + */ + public function getSumOfSquares() + { + return $this->sum_of_squares; + } + + /** + * double sum_of_squares = 5; + */ + public function setSumOfSquares($var) + { + GPBUtil::checkDouble($var); + $this->sum_of_squares = $var; + } + + /** + * double count = 6; + */ + public function getCount() + { + return $this->count; + } + + /** + * double count = 6; + */ + public function setCount($var) + { + GPBUtil::checkDouble($var); + $this->count = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/HistogramParams.php b/src/php/tests/qps/generated_code/Grpc/Testing/HistogramParams.php new file mode 100644 index 00000000000..836c94b01d8 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/HistogramParams.php @@ -0,0 +1,93 @@ + + * Histogram params based on grpc/support/histogram.c + * + * + * Protobuf type grpc.testing.HistogramParams + */ +class HistogramParams extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * first bucket is [0, 1 + resolution)
+     *
+ * + * double resolution = 1; + */ + private $resolution = 0.0; + /** + * 
+     * use enough buckets to allow this value
+     *
+ * + * double max_possible = 2; + */ + private $max_possible = 0.0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Stats::initOnce(); + parent::__construct(); + } + + /** + * 
+     * first bucket is [0, 1 + resolution)
+     *
+ * + * double resolution = 1; + */ + public function getResolution() + { + return $this->resolution; + } + + /** + * 
+     * first bucket is [0, 1 + resolution)
+     *
+ * + * double resolution = 1; + */ + public function setResolution($var) + { + GPBUtil::checkDouble($var); + $this->resolution = $var; + } + + /** + * 
+     * use enough buckets to allow this value
+     *
+ * + * double max_possible = 2; + */ + public function getMaxPossible() + { + return $this->max_possible; + } + + /** + * 
+     * use enough buckets to allow this value
+     *
+ * + * double max_possible = 2; + */ + public function setMaxPossible($var) + { + GPBUtil::checkDouble($var); + $this->max_possible = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/LoadParams.php b/src/php/tests/qps/generated_code/Grpc/Testing/LoadParams.php new file mode 100644 index 00000000000..1f32e49c8aa --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/LoadParams.php @@ -0,0 +1,63 @@ +grpc.testing.LoadParams + */ +class LoadParams extends \Google\Protobuf\Internal\Message +{ + protected $load; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * .grpc.testing.ClosedLoopParams closed_loop = 1; + */ + public function getClosedLoop() + { + return $this->readOneof(1); + } + + /** + * .grpc.testing.ClosedLoopParams closed_loop = 1; + */ + public function setClosedLoop(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\ClosedLoopParams::class); + $this->writeOneof(1, $var); + } + + /** + * .grpc.testing.PoissonParams poisson = 2; + */ + public function getPoisson() + { + return $this->readOneof(2); + } + + /** + * .grpc.testing.PoissonParams poisson = 2; + */ + public function setPoisson(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\PoissonParams::class); + $this->writeOneof(2, $var); + } + + public function getLoad() + { + return $this->whichOneof("load"); + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/Mark.php b/src/php/tests/qps/generated_code/Grpc/Testing/Mark.php new file mode 100644 index 00000000000..ce006efacd8 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/Mark.php @@ -0,0 +1,60 @@ + + * Request current stats + * + * + * Protobuf type grpc.testing.Mark + */ +class Mark extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * if true, the stats will be reset after taking their snapshot.
+     *
+ * + * bool reset = 1; + */ + private $reset = false; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * 
+     * if true, the stats will be reset after taking their snapshot.
+     *
+ * + * bool reset = 1; + */ + public function getReset() + { + return $this->reset; + } + + /** + * 
+     * if true, the stats will be reset after taking their snapshot.
+     *
+ * + * bool reset = 1; + */ + public function setReset($var) + { + GPBUtil::checkBool($var); + $this->reset = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/Payload.php b/src/php/tests/qps/generated_code/Grpc/Testing/Payload.php new file mode 100644 index 00000000000..d17c271af74 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/Payload.php @@ -0,0 +1,96 @@ + + * A block of data, to simply increase gRPC message size. + * + * + * Protobuf type grpc.testing.Payload + */ +class Payload extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * DEPRECATED, don't use. To be removed shortly.
+     * The type of data in body.
+     *
+ * + * .grpc.testing.PayloadType type = 1; + */ + private $type = 0; + /** + * 
+     * Primary contents of payload.
+     *
+ * + * bytes body = 2; + */ + private $body = ''; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * 
+     * DEPRECATED, don't use. To be removed shortly.
+     * The type of data in body.
+     *
+ * + * .grpc.testing.PayloadType type = 1; + */ + public function getType() + { + return $this->type; + } + + /** + * 
+     * DEPRECATED, don't use. To be removed shortly.
+     * The type of data in body.
+     *
+ * + * .grpc.testing.PayloadType type = 1; + */ + public function setType($var) + { + GPBUtil::checkEnum($var, \Grpc\Testing\PayloadType::class); + $this->type = $var; + } + + /** + * 
+     * Primary contents of payload.
+     *
+ * + * bytes body = 2; + */ + public function getBody() + { + return $this->body; + } + + /** + * 
+     * Primary contents of payload.
+     *
+ * + * bytes body = 2; + */ + public function setBody($var) + { + GPBUtil::checkString($var, False); + $this->body = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/PayloadConfig.php b/src/php/tests/qps/generated_code/Grpc/Testing/PayloadConfig.php new file mode 100644 index 00000000000..a2fe7109ba7 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/PayloadConfig.php @@ -0,0 +1,80 @@ +grpc.testing.PayloadConfig + */ +class PayloadConfig extends \Google\Protobuf\Internal\Message +{ + protected $payload; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Payloads::initOnce(); + parent::__construct(); + } + + /** + * .grpc.testing.ByteBufferParams bytebuf_params = 1; + */ + public function getBytebufParams() + { + return $this->readOneof(1); + } + + /** + * .grpc.testing.ByteBufferParams bytebuf_params = 1; + */ + public function setBytebufParams(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\ByteBufferParams::class); + $this->writeOneof(1, $var); + } + + /** + * .grpc.testing.SimpleProtoParams simple_params = 2; + */ + public function getSimpleParams() + { + return $this->readOneof(2); + } + + /** + * .grpc.testing.SimpleProtoParams simple_params = 2; + */ + public function setSimpleParams(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\SimpleProtoParams::class); + $this->writeOneof(2, $var); + } + + /** + * .grpc.testing.ComplexProtoParams complex_params = 3; + */ + public function getComplexParams() + { + return $this->readOneof(3); + } + + /** + * .grpc.testing.ComplexProtoParams complex_params = 3; + */ + public function setComplexParams(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\ComplexProtoParams::class); + $this->writeOneof(3, $var); + } + + public function getPayload() + { + return $this->whichOneof("payload"); + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/PayloadType.php b/src/php/tests/qps/generated_code/Grpc/Testing/PayloadType.php new file mode 100644 index 00000000000..189ef034b47 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/PayloadType.php @@ -0,0 +1,26 @@ + + * DEPRECATED, don't use. To be removed shortly. + * The type of payload that should be returned. + * + * + * Protobuf enum grpc.testing.PayloadType + */ +class PayloadType +{ + /** + * 
+     * Compressable text format.
+     *
+ * + * COMPRESSABLE = 0; + */ + const COMPRESSABLE = 0; +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/PoissonParams.php b/src/php/tests/qps/generated_code/Grpc/Testing/PoissonParams.php new file mode 100644 index 00000000000..d64edd45f03 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/PoissonParams.php @@ -0,0 +1,61 @@ + + * Parameters of poisson process distribution, which is a good representation + * of activity coming in from independent identical stationary sources. + * + * + * Protobuf type grpc.testing.PoissonParams + */ +class PoissonParams extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * The rate of arrivals (a.k.a. lambda parameter of the exp distribution).
+     *
+ * + * double offered_load = 1; + */ + private $offered_load = 0.0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * 
+     * The rate of arrivals (a.k.a. lambda parameter of the exp distribution).
+     *
+ * + * double offered_load = 1; + */ + public function getOfferedLoad() + { + return $this->offered_load; + } + + /** + * 
+     * The rate of arrivals (a.k.a. lambda parameter of the exp distribution).
+     *
+ * + * double offered_load = 1; + */ + public function setOfferedLoad($var) + { + GPBUtil::checkDouble($var); + $this->offered_load = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ProxyClientServiceClient.php b/src/php/tests/qps/generated_code/Grpc/Testing/ProxyClientServiceClient.php new file mode 100644 index 00000000000..23c041b4709 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ProxyClientServiceClient.php @@ -0,0 +1,72 @@ +_simpleRequest('/grpc.testing.ProxyClientService/GetConfig', + $argument, + ['\Grpc\Testing\ClientConfig', 'decode'], + $metadata, $options); + } + + /** + * @param array $metadata metadata + * @param array $options call options + */ + public function ReportTime($metadata = [], $options = []) { + return $this->_clientStreamRequest('/grpc.testing.ProxyClientService/ReportTime', + ['\Grpc\Testing\Void','decode'], + $metadata, $options); + } + + } + +} diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ProxyStat.php b/src/php/tests/qps/generated_code/Grpc/Testing/ProxyStat.php new file mode 100644 index 00000000000..ed43be99cef --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ProxyStat.php @@ -0,0 +1,44 @@ +grpc.testing.ProxyStat + */ +class ProxyStat extends \Google\Protobuf\Internal\Message +{ + /** + * double latency = 1; + */ + private $latency = 0.0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\ProxyService::initOnce(); + parent::__construct(); + } + + /** + * double latency = 1; + */ + public function getLatency() + { + return $this->latency; + } + + /** + * double latency = 1; + */ + public function setLatency($var) + { + GPBUtil::checkDouble($var); + $this->latency = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ReconnectInfo.php b/src/php/tests/qps/generated_code/Grpc/Testing/ReconnectInfo.php new file mode 100644 index 00000000000..dfaaa606c3b --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ReconnectInfo.php @@ -0,0 +1,71 @@ + + * For reconnect interop test only. + * Server tells client whether its reconnects are following the spec and the + * reconnect backoffs it saw. + * + * + * Protobuf type grpc.testing.ReconnectInfo + */ +class ReconnectInfo extends \Google\Protobuf\Internal\Message +{ + /** + * bool passed = 1; + */ + private $passed = false; + /** + * repeated int32 backoff_ms = 2; + */ + private $backoff_ms; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * bool passed = 1; + */ + public function getPassed() + { + return $this->passed; + } + + /** + * bool passed = 1; + */ + public function setPassed($var) + { + GPBUtil::checkBool($var); + $this->passed = $var; + } + + /** + * repeated int32 backoff_ms = 2; + */ + public function getBackoffMs() + { + return $this->backoff_ms; + } + + /** + * repeated int32 backoff_ms = 2; + */ + public function setBackoffMs(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::INT32); + $this->backoff_ms = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ReconnectParams.php b/src/php/tests/qps/generated_code/Grpc/Testing/ReconnectParams.php new file mode 100644 index 00000000000..97158557836 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ReconnectParams.php @@ -0,0 +1,49 @@ + + * For reconnect interop test only. + * Client tells server what reconnection parameters it used. + * + * + * Protobuf type grpc.testing.ReconnectParams + */ +class ReconnectParams extends \Google\Protobuf\Internal\Message +{ + /** + * int32 max_reconnect_backoff_ms = 1; + */ + private $max_reconnect_backoff_ms = 0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * int32 max_reconnect_backoff_ms = 1; + */ + public function getMaxReconnectBackoffMs() + { + return $this->max_reconnect_backoff_ms; + } + + /** + * int32 max_reconnect_backoff_ms = 1; + */ + public function setMaxReconnectBackoffMs($var) + { + GPBUtil::checkInt32($var); + $this->max_reconnect_backoff_ms = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/RequestResultCount.php b/src/php/tests/qps/generated_code/Grpc/Testing/RequestResultCount.php new file mode 100644 index 00000000000..1be42b2ac91 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/RequestResultCount.php @@ -0,0 +1,65 @@ +grpc.testing.RequestResultCount + */ +class RequestResultCount extends \Google\Protobuf\Internal\Message +{ + /** + * int32 status_code = 1; + */ + private $status_code = 0; + /** + * int64 count = 2; + */ + private $count = 0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Stats::initOnce(); + parent::__construct(); + } + + /** + * int32 status_code = 1; + */ + public function getStatusCode() + { + return $this->status_code; + } + + /** + * int32 status_code = 1; + */ + public function setStatusCode($var) + { + GPBUtil::checkInt32($var); + $this->status_code = $var; + } + + /** + * int64 count = 2; + */ + public function getCount() + { + return $this->count; + } + + /** + * int64 count = 2; + */ + public function setCount($var) + { + GPBUtil::checkInt64($var); + $this->count = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ResponseParameters.php b/src/php/tests/qps/generated_code/Grpc/Testing/ResponseParameters.php new file mode 100644 index 00000000000..b7a8e5ece71 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ResponseParameters.php @@ -0,0 +1,138 @@ + + * Configuration for a particular response. + * + * + * Protobuf type grpc.testing.ResponseParameters + */ +class ResponseParameters extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * Desired payload sizes in responses from the server.
+     *
+ * + * int32 size = 1; + */ + private $size = 0; + /** + * 
+     * Desired interval between consecutive responses in the response stream in
+     * microseconds.
+     *
+ * + * int32 interval_us = 2; + */ + private $interval_us = 0; + /** + * 
+     * Whether to request the server to compress the response. This field is
+     * "nullable" in order to interoperate seamlessly with clients not able to
+     * implement the full compression tests by introspecting the call to verify
+     * the response's compression status.
+     *
+ * + * .grpc.testing.BoolValue compressed = 3; + */ + private $compressed = null; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * 
+     * Desired payload sizes in responses from the server.
+     *
+ * + * int32 size = 1; + */ + public function getSize() + { + return $this->size; + } + + /** + * 
+     * Desired payload sizes in responses from the server.
+     *
+ * + * int32 size = 1; + */ + public function setSize($var) + { + GPBUtil::checkInt32($var); + $this->size = $var; + } + + /** + * 
+     * Desired interval between consecutive responses in the response stream in
+     * microseconds.
+     *
+ * + * int32 interval_us = 2; + */ + public function getIntervalUs() + { + return $this->interval_us; + } + + /** + * 
+     * Desired interval between consecutive responses in the response stream in
+     * microseconds.
+     *
+ * + * int32 interval_us = 2; + */ + public function setIntervalUs($var) + { + GPBUtil::checkInt32($var); + $this->interval_us = $var; + } + + /** + * 
+     * Whether to request the server to compress the response. This field is
+     * "nullable" in order to interoperate seamlessly with clients not able to
+     * implement the full compression tests by introspecting the call to verify
+     * the response's compression status.
+     *
+ * + * .grpc.testing.BoolValue compressed = 3; + */ + public function getCompressed() + { + return $this->compressed; + } + + /** + * 
+     * Whether to request the server to compress the response. This field is
+     * "nullable" in order to interoperate seamlessly with clients not able to
+     * implement the full compression tests by introspecting the call to verify
+     * the response's compression status.
+     *
+ * + * .grpc.testing.BoolValue compressed = 3; + */ + public function setCompressed(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\BoolValue::class); + $this->compressed = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/RpcType.php b/src/php/tests/qps/generated_code/Grpc/Testing/RpcType.php new file mode 100644 index 00000000000..2e664fff477 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/RpcType.php @@ -0,0 +1,21 @@ +grpc.testing.RpcType + */ +class RpcType +{ + /** + * UNARY = 0; + */ + const UNARY = 0; + /** + * STREAMING = 1; + */ + const STREAMING = 1; +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/Scenario.php b/src/php/tests/qps/generated_code/Grpc/Testing/Scenario.php new file mode 100644 index 00000000000..136ed299ea8 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/Scenario.php @@ -0,0 +1,291 @@ + + * A single performance scenario: input to qps_json_driver + * + * + * Protobuf type grpc.testing.Scenario + */ +class Scenario extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * Human readable name for this scenario
+     *
+ * + * string name = 1; + */ + private $name = ''; + /** + * 
+     * Client configuration
+     *
+ * + * .grpc.testing.ClientConfig client_config = 2; + */ + private $client_config = null; + /** + * 
+     * Number of clients to start for the test
+     *
+ * + * int32 num_clients = 3; + */ + private $num_clients = 0; + /** + * 
+     * Server configuration
+     *
+ * + * .grpc.testing.ServerConfig server_config = 4; + */ + private $server_config = null; + /** + * 
+     * Number of servers to start for the test
+     *
+ * + * int32 num_servers = 5; + */ + private $num_servers = 0; + /** + * 
+     * Warmup period, in seconds
+     *
+ * + * int32 warmup_seconds = 6; + */ + private $warmup_seconds = 0; + /** + * 
+     * Benchmark time, in seconds
+     *
+ * + * int32 benchmark_seconds = 7; + */ + private $benchmark_seconds = 0; + /** + * 
+     * Number of workers to spawn locally (usually zero)
+     *
+ * + * int32 spawn_local_worker_count = 8; + */ + private $spawn_local_worker_count = 0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * 
+     * Human readable name for this scenario
+     *
+ * + * string name = 1; + */ + public function getName() + { + return $this->name; + } + + /** + * 
+     * Human readable name for this scenario
+     *
+ * + * string name = 1; + */ + public function setName($var) + { + GPBUtil::checkString($var, True); + $this->name = $var; + } + + /** + * 
+     * Client configuration
+     *
+ * + * .grpc.testing.ClientConfig client_config = 2; + */ + public function getClientConfig() + { + return $this->client_config; + } + + /** + * 
+     * Client configuration
+     *
+ * + * .grpc.testing.ClientConfig client_config = 2; + */ + public function setClientConfig(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\ClientConfig::class); + $this->client_config = $var; + } + + /** + * 
+     * Number of clients to start for the test
+     *
+ * + * int32 num_clients = 3; + */ + public function getNumClients() + { + return $this->num_clients; + } + + /** + * 
+     * Number of clients to start for the test
+     *
+ * + * int32 num_clients = 3; + */ + public function setNumClients($var) + { + GPBUtil::checkInt32($var); + $this->num_clients = $var; + } + + /** + * 
+     * Server configuration
+     *
+ * + * .grpc.testing.ServerConfig server_config = 4; + */ + public function getServerConfig() + { + return $this->server_config; + } + + /** + * 
+     * Server configuration
+     *
+ * + * .grpc.testing.ServerConfig server_config = 4; + */ + public function setServerConfig(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\ServerConfig::class); + $this->server_config = $var; + } + + /** + * 
+     * Number of servers to start for the test
+     *
+ * + * int32 num_servers = 5; + */ + public function getNumServers() + { + return $this->num_servers; + } + + /** + * 
+     * Number of servers to start for the test
+     *
+ * + * int32 num_servers = 5; + */ + public function setNumServers($var) + { + GPBUtil::checkInt32($var); + $this->num_servers = $var; + } + + /** + * 
+     * Warmup period, in seconds
+     *
+ * + * int32 warmup_seconds = 6; + */ + public function getWarmupSeconds() + { + return $this->warmup_seconds; + } + + /** + * 
+     * Warmup period, in seconds
+     *
+ * + * int32 warmup_seconds = 6; + */ + public function setWarmupSeconds($var) + { + GPBUtil::checkInt32($var); + $this->warmup_seconds = $var; + } + + /** + * 
+     * Benchmark time, in seconds
+     *
+ * + * int32 benchmark_seconds = 7; + */ + public function getBenchmarkSeconds() + { + return $this->benchmark_seconds; + } + + /** + * 
+     * Benchmark time, in seconds
+     *
+ * + * int32 benchmark_seconds = 7; + */ + public function setBenchmarkSeconds($var) + { + GPBUtil::checkInt32($var); + $this->benchmark_seconds = $var; + } + + /** + * 
+     * Number of workers to spawn locally (usually zero)
+     *
+ * + * int32 spawn_local_worker_count = 8; + */ + public function getSpawnLocalWorkerCount() + { + return $this->spawn_local_worker_count; + } + + /** + * 
+     * Number of workers to spawn locally (usually zero)
+     *
+ * + * int32 spawn_local_worker_count = 8; + */ + public function setSpawnLocalWorkerCount($var) + { + GPBUtil::checkInt32($var); + $this->spawn_local_worker_count = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ScenarioResult.php b/src/php/tests/qps/generated_code/Grpc/Testing/ScenarioResult.php new file mode 100644 index 00000000000..809cd96244d --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ScenarioResult.php @@ -0,0 +1,312 @@ + + * Results of a single benchmark scenario. + * + * + * Protobuf type grpc.testing.ScenarioResult + */ +class ScenarioResult extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * Inputs used to run the scenario.
+     *
+ * + * .grpc.testing.Scenario scenario = 1; + */ + private $scenario = null; + /** + * 
+     * Histograms from all clients merged into one histogram.
+     *
+ * + * .grpc.testing.HistogramData latencies = 2; + */ + private $latencies = null; + /** + * 
+     * Client stats for each client
+     *
+ * + * repeated .grpc.testing.ClientStats client_stats = 3; + */ + private $client_stats; + /** + * 
+     * Server stats for each server
+     *
+ * + * repeated .grpc.testing.ServerStats server_stats = 4; + */ + private $server_stats; + /** + * 
+     * Number of cores available to each server
+     *
+ * + * repeated int32 server_cores = 5; + */ + private $server_cores; + /** + * 
+     * An after-the-fact computed summary
+     *
+ * + * .grpc.testing.ScenarioResultSummary summary = 6; + */ + private $summary = null; + /** + * 
+     * Information on success or failure of each worker
+     *
+ * + * repeated bool client_success = 7; + */ + private $client_success; + /** + * repeated bool server_success = 8; + */ + private $server_success; + /** + * 
+     * Number of failed requests (one row per status code seen)
+     *
+ * + * repeated .grpc.testing.RequestResultCount request_results = 9; + */ + private $request_results; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * 
+     * Inputs used to run the scenario.
+     *
+ * + * .grpc.testing.Scenario scenario = 1; + */ + public function getScenario() + { + return $this->scenario; + } + + /** + * 
+     * Inputs used to run the scenario.
+     *
+ * + * .grpc.testing.Scenario scenario = 1; + */ + public function setScenario(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\Scenario::class); + $this->scenario = $var; + } + + /** + * 
+     * Histograms from all clients merged into one histogram.
+     *
+ * + * .grpc.testing.HistogramData latencies = 2; + */ + public function getLatencies() + { + return $this->latencies; + } + + /** + * 
+     * Histograms from all clients merged into one histogram.
+     *
+ * + * .grpc.testing.HistogramData latencies = 2; + */ + public function setLatencies(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\HistogramData::class); + $this->latencies = $var; + } + + /** + * 
+     * Client stats for each client
+     *
+ * + * repeated .grpc.testing.ClientStats client_stats = 3; + */ + public function getClientStats() + { + return $this->client_stats; + } + + /** + * 
+     * Client stats for each client
+     *
+ * + * repeated .grpc.testing.ClientStats client_stats = 3; + */ + public function setClientStats(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::MESSAGE, \Grpc\Testing\ClientStats::class); + $this->client_stats = $var; + } + + /** + * 
+     * Server stats for each server
+     *
+ * + * repeated .grpc.testing.ServerStats server_stats = 4; + */ + public function getServerStats() + { + return $this->server_stats; + } + + /** + * 
+     * Server stats for each server
+     *
+ * + * repeated .grpc.testing.ServerStats server_stats = 4; + */ + public function setServerStats(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::MESSAGE, \Grpc\Testing\ServerStats::class); + $this->server_stats = $var; + } + + /** + * 
+     * Number of cores available to each server
+     *
+ * + * repeated int32 server_cores = 5; + */ + public function getServerCores() + { + return $this->server_cores; + } + + /** + * 
+     * Number of cores available to each server
+     *
+ * + * repeated int32 server_cores = 5; + */ + public function setServerCores(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::INT32); + $this->server_cores = $var; + } + + /** + * 
+     * An after-the-fact computed summary
+     *
+ * + * .grpc.testing.ScenarioResultSummary summary = 6; + */ + public function getSummary() + { + return $this->summary; + } + + /** + * 
+     * An after-the-fact computed summary
+     *
+ * + * .grpc.testing.ScenarioResultSummary summary = 6; + */ + public function setSummary(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\ScenarioResultSummary::class); + $this->summary = $var; + } + + /** + * 
+     * Information on success or failure of each worker
+     *
+ * + * repeated bool client_success = 7; + */ + public function getClientSuccess() + { + return $this->client_success; + } + + /** + * 
+     * Information on success or failure of each worker
+     *
+ * + * repeated bool client_success = 7; + */ + public function setClientSuccess(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::BOOL); + $this->client_success = $var; + } + + /** + * repeated bool server_success = 8; + */ + public function getServerSuccess() + { + return $this->server_success; + } + + /** + * repeated bool server_success = 8; + */ + public function setServerSuccess(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::BOOL); + $this->server_success = $var; + } + + /** + * 
+     * Number of failed requests (one row per status code seen)
+     *
+ * + * repeated .grpc.testing.RequestResultCount request_results = 9; + */ + public function getRequestResults() + { + return $this->request_results; + } + + /** + * 
+     * Number of failed requests (one row per status code seen)
+     *
+ * + * repeated .grpc.testing.RequestResultCount request_results = 9; + */ + public function setRequestResults(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::MESSAGE, \Grpc\Testing\RequestResultCount::class); + $this->request_results = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ScenarioResultSummary.php b/src/php/tests/qps/generated_code/Grpc/Testing/ScenarioResultSummary.php new file mode 100644 index 00000000000..7520cff78e8 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ScenarioResultSummary.php @@ -0,0 +1,430 @@ + + * Basic summary that can be computed from ClientStats and ServerStats + * once the scenario has finished. + * + * + * Protobuf type grpc.testing.ScenarioResultSummary + */ +class ScenarioResultSummary extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * Total number of operations per second over all clients.
+     *
+ * + * double qps = 1; + */ + private $qps = 0.0; + /** + * 
+     * QPS per one server core.
+     *
+ * + * double qps_per_server_core = 2; + */ + private $qps_per_server_core = 0.0; + /** + * 
+     * server load based on system_time (0.85 => 85%)
+     *
+ * + * double server_system_time = 3; + */ + private $server_system_time = 0.0; + /** + * 
+     * server load based on user_time (0.85 => 85%)
+     *
+ * + * double server_user_time = 4; + */ + private $server_user_time = 0.0; + /** + * 
+     * client load based on system_time (0.85 => 85%)
+     *
+ * + * double client_system_time = 5; + */ + private $client_system_time = 0.0; + /** + * 
+     * client load based on user_time (0.85 => 85%)
+     *
+ * + * double client_user_time = 6; + */ + private $client_user_time = 0.0; + /** + * 
+     * X% latency percentiles (in nanoseconds)
+     *
+ * + * double latency_50 = 7; + */ + private $latency_50 = 0.0; + /** + * double latency_90 = 8; + */ + private $latency_90 = 0.0; + /** + * double latency_95 = 9; + */ + private $latency_95 = 0.0; + /** + * double latency_99 = 10; + */ + private $latency_99 = 0.0; + /** + * double latency_999 = 11; + */ + private $latency_999 = 0.0; + /** + * 
+     * server cpu usage percentage
+     *
+ * + * double server_cpu_usage = 12; + */ + private $server_cpu_usage = 0.0; + /** + * 
+     * Number of requests that succeeded/failed
+     *
+ * + * double successful_requests_per_second = 13; + */ + private $successful_requests_per_second = 0.0; + /** + * double failed_requests_per_second = 14; + */ + private $failed_requests_per_second = 0.0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * 
+     * Total number of operations per second over all clients.
+     *
+ * + * double qps = 1; + */ + public function getQps() + { + return $this->qps; + } + + /** + * 
+     * Total number of operations per second over all clients.
+     *
+ * + * double qps = 1; + */ + public function setQps($var) + { + GPBUtil::checkDouble($var); + $this->qps = $var; + } + + /** + * 
+     * QPS per one server core.
+     *
+ * + * double qps_per_server_core = 2; + */ + public function getQpsPerServerCore() + { + return $this->qps_per_server_core; + } + + /** + * 
+     * QPS per one server core.
+     *
+ * + * double qps_per_server_core = 2; + */ + public function setQpsPerServerCore($var) + { + GPBUtil::checkDouble($var); + $this->qps_per_server_core = $var; + } + + /** + * 
+     * server load based on system_time (0.85 => 85%)
+     *
+ * + * double server_system_time = 3; + */ + public function getServerSystemTime() + { + return $this->server_system_time; + } + + /** + * 
+     * server load based on system_time (0.85 => 85%)
+     *
+ * + * double server_system_time = 3; + */ + public function setServerSystemTime($var) + { + GPBUtil::checkDouble($var); + $this->server_system_time = $var; + } + + /** + * 
+     * server load based on user_time (0.85 => 85%)
+     *
+ * + * double server_user_time = 4; + */ + public function getServerUserTime() + { + return $this->server_user_time; + } + + /** + * 
+     * server load based on user_time (0.85 => 85%)
+     *
+ * + * double server_user_time = 4; + */ + public function setServerUserTime($var) + { + GPBUtil::checkDouble($var); + $this->server_user_time = $var; + } + + /** + * 
+     * client load based on system_time (0.85 => 85%)
+     *
+ * + * double client_system_time = 5; + */ + public function getClientSystemTime() + { + return $this->client_system_time; + } + + /** + * 
+     * client load based on system_time (0.85 => 85%)
+     *
+ * + * double client_system_time = 5; + */ + public function setClientSystemTime($var) + { + GPBUtil::checkDouble($var); + $this->client_system_time = $var; + } + + /** + * 
+     * client load based on user_time (0.85 => 85%)
+     *
+ * + * double client_user_time = 6; + */ + public function getClientUserTime() + { + return $this->client_user_time; + } + + /** + * 
+     * client load based on user_time (0.85 => 85%)
+     *
+ * + * double client_user_time = 6; + */ + public function setClientUserTime($var) + { + GPBUtil::checkDouble($var); + $this->client_user_time = $var; + } + + /** + * 
+     * X% latency percentiles (in nanoseconds)
+     *
+ * + * double latency_50 = 7; + */ + public function getLatency50() + { + return $this->latency_50; + } + + /** + * 
+     * X% latency percentiles (in nanoseconds)
+     *
+ * + * double latency_50 = 7; + */ + public function setLatency50($var) + { + GPBUtil::checkDouble($var); + $this->latency_50 = $var; + } + + /** + * double latency_90 = 8; + */ + public function getLatency90() + { + return $this->latency_90; + } + + /** + * double latency_90 = 8; + */ + public function setLatency90($var) + { + GPBUtil::checkDouble($var); + $this->latency_90 = $var; + } + + /** + * double latency_95 = 9; + */ + public function getLatency95() + { + return $this->latency_95; + } + + /** + * double latency_95 = 9; + */ + public function setLatency95($var) + { + GPBUtil::checkDouble($var); + $this->latency_95 = $var; + } + + /** + * double latency_99 = 10; + */ + public function getLatency99() + { + return $this->latency_99; + } + + /** + * double latency_99 = 10; + */ + public function setLatency99($var) + { + GPBUtil::checkDouble($var); + $this->latency_99 = $var; + } + + /** + * double latency_999 = 11; + */ + public function getLatency999() + { + return $this->latency_999; + } + + /** + * double latency_999 = 11; + */ + public function setLatency999($var) + { + GPBUtil::checkDouble($var); + $this->latency_999 = $var; + } + + /** + * 
+     * server cpu usage percentage
+     *
+ * + * double server_cpu_usage = 12; + */ + public function getServerCpuUsage() + { + return $this->server_cpu_usage; + } + + /** + * 
+     * server cpu usage percentage
+     *
+ * + * double server_cpu_usage = 12; + */ + public function setServerCpuUsage($var) + { + GPBUtil::checkDouble($var); + $this->server_cpu_usage = $var; + } + + /** + * 
+     * Number of requests that succeeded/failed
+     *
+ * + * double successful_requests_per_second = 13; + */ + public function getSuccessfulRequestsPerSecond() + { + return $this->successful_requests_per_second; + } + + /** + * 
+     * Number of requests that succeeded/failed
+     *
+ * + * double successful_requests_per_second = 13; + */ + public function setSuccessfulRequestsPerSecond($var) + { + GPBUtil::checkDouble($var); + $this->successful_requests_per_second = $var; + } + + /** + * double failed_requests_per_second = 14; + */ + public function getFailedRequestsPerSecond() + { + return $this->failed_requests_per_second; + } + + /** + * double failed_requests_per_second = 14; + */ + public function setFailedRequestsPerSecond($var) + { + GPBUtil::checkDouble($var); + $this->failed_requests_per_second = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/Scenarios.php b/src/php/tests/qps/generated_code/Grpc/Testing/Scenarios.php new file mode 100644 index 00000000000..278f555b760 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/Scenarios.php @@ -0,0 +1,48 @@ + + * A set of scenarios to be run with qps_json_driver + * + * + * Protobuf type grpc.testing.Scenarios + */ +class Scenarios extends \Google\Protobuf\Internal\Message +{ + /** + * repeated .grpc.testing.Scenario scenarios = 1; + */ + private $scenarios; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * repeated .grpc.testing.Scenario scenarios = 1; + */ + public function getScenarios() + { + return $this->scenarios; + } + + /** + * repeated .grpc.testing.Scenario scenarios = 1; + */ + public function setScenarios(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::MESSAGE, \Grpc\Testing\Scenario::class); + $this->scenarios = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/SecurityParams.php b/src/php/tests/qps/generated_code/Grpc/Testing/SecurityParams.php new file mode 100644 index 00000000000..27a5b95cc94 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/SecurityParams.php @@ -0,0 +1,69 @@ + + * presence of SecurityParams implies use of TLS + * + * + * Protobuf type grpc.testing.SecurityParams + */ +class SecurityParams extends \Google\Protobuf\Internal\Message +{ + /** + * bool use_test_ca = 1; + */ + private $use_test_ca = false; + /** + * string server_host_override = 2; + */ + private $server_host_override = ''; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * bool use_test_ca = 1; + */ + public function getUseTestCa() + { + return $this->use_test_ca; + } + + /** + * bool use_test_ca = 1; + */ + public function setUseTestCa($var) + { + GPBUtil::checkBool($var); + $this->use_test_ca = $var; + } + + /** + * string server_host_override = 2; + */ + public function getServerHostOverride() + { + return $this->server_host_override; + } + + /** + * string server_host_override = 2; + */ + public function setServerHostOverride($var) + { + GPBUtil::checkString($var, True); + $this->server_host_override = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ServerArgs.php b/src/php/tests/qps/generated_code/Grpc/Testing/ServerArgs.php new file mode 100644 index 00000000000..0d84b80124a --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ServerArgs.php @@ -0,0 +1,63 @@ +grpc.testing.ServerArgs + */ +class ServerArgs extends \Google\Protobuf\Internal\Message +{ + protected $argtype; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * .grpc.testing.ServerConfig setup = 1; + */ + public function getSetup() + { + return $this->readOneof(1); + } + + /** + * .grpc.testing.ServerConfig setup = 1; + */ + public function setSetup(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\ServerConfig::class); + $this->writeOneof(1, $var); + } + + /** + * .grpc.testing.Mark mark = 2; + */ + public function getMark() + { + return $this->readOneof(2); + } + + /** + * .grpc.testing.Mark mark = 2; + */ + public function setMark(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\Mark::class); + $this->writeOneof(2, $var); + } + + public function getArgtype() + { + return $this->whichOneof("argtype"); + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ServerConfig.php b/src/php/tests/qps/generated_code/Grpc/Testing/ServerConfig.php new file mode 100644 index 00000000000..e2bcede48cb --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ServerConfig.php @@ -0,0 +1,305 @@ +grpc.testing.ServerConfig + */ +class ServerConfig extends \Google\Protobuf\Internal\Message +{ + /** + * .grpc.testing.ServerType server_type = 1; + */ + private $server_type = 0; + /** + * .grpc.testing.SecurityParams security_params = 2; + */ + private $security_params = null; + /** + * 
+     * Port on which to listen. Zero means pick unused port.
+     *
+ * + * int32 port = 4; + */ + private $port = 0; + /** + * 
+     * Only for async server. Number of threads used to serve the requests.
+     *
+ * + * int32 async_server_threads = 7; + */ + private $async_server_threads = 0; + /** + * 
+     * Specify the number of cores to limit server to, if desired
+     *
+ * + * int32 core_limit = 8; + */ + private $core_limit = 0; + /** + * 
+     * payload config, used in generic server.
+     * Note this must NOT be used in proto (non-generic) servers. For proto servers,
+     * 'response sizes' must be configured from the 'response_size' field of the
+     * 'SimpleRequest' objects in RPC requests.
+     *
+ * + * .grpc.testing.PayloadConfig payload_config = 9; + */ + private $payload_config = null; + /** + * 
+     * Specify the cores we should run the server on, if desired
+     *
+ * + * repeated int32 core_list = 10; + */ + private $core_list; + /** + * 
+     * If we use an OTHER_SERVER client_type, this string gives more detail
+     *
+ * + * string other_server_api = 11; + */ + private $other_server_api = ''; + /** + * 
+     * Buffer pool size (no buffer pool specified if unset)
+     *
+ * + * int32 resource_quota_size = 1001; + */ + private $resource_quota_size = 0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * .grpc.testing.ServerType server_type = 1; + */ + public function getServerType() + { + return $this->server_type; + } + + /** + * .grpc.testing.ServerType server_type = 1; + */ + public function setServerType($var) + { + GPBUtil::checkEnum($var, \Grpc\Testing\ServerType::class); + $this->server_type = $var; + } + + /** + * .grpc.testing.SecurityParams security_params = 2; + */ + public function getSecurityParams() + { + return $this->security_params; + } + + /** + * .grpc.testing.SecurityParams security_params = 2; + */ + public function setSecurityParams(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\SecurityParams::class); + $this->security_params = $var; + } + + /** + * 
+     * Port on which to listen. Zero means pick unused port.
+     *
+ * + * int32 port = 4; + */ + public function getPort() + { + return $this->port; + } + + /** + * 
+     * Port on which to listen. Zero means pick unused port.
+     *
+ * + * int32 port = 4; + */ + public function setPort($var) + { + GPBUtil::checkInt32($var); + $this->port = $var; + } + + /** + * 
+     * Only for async server. Number of threads used to serve the requests.
+     *
+ * + * int32 async_server_threads = 7; + */ + public function getAsyncServerThreads() + { + return $this->async_server_threads; + } + + /** + * 
+     * Only for async server. Number of threads used to serve the requests.
+     *
+ * + * int32 async_server_threads = 7; + */ + public function setAsyncServerThreads($var) + { + GPBUtil::checkInt32($var); + $this->async_server_threads = $var; + } + + /** + * 
+     * Specify the number of cores to limit server to, if desired
+     *
+ * + * int32 core_limit = 8; + */ + public function getCoreLimit() + { + return $this->core_limit; + } + + /** + * 
+     * Specify the number of cores to limit server to, if desired
+     *
+ * + * int32 core_limit = 8; + */ + public function setCoreLimit($var) + { + GPBUtil::checkInt32($var); + $this->core_limit = $var; + } + + /** + * 
+     * payload config, used in generic server.
+     * Note this must NOT be used in proto (non-generic) servers. For proto servers,
+     * 'response sizes' must be configured from the 'response_size' field of the
+     * 'SimpleRequest' objects in RPC requests.
+     *
+ * + * .grpc.testing.PayloadConfig payload_config = 9; + */ + public function getPayloadConfig() + { + return $this->payload_config; + } + + /** + * 
+     * payload config, used in generic server.
+     * Note this must NOT be used in proto (non-generic) servers. For proto servers,
+     * 'response sizes' must be configured from the 'response_size' field of the
+     * 'SimpleRequest' objects in RPC requests.
+     *
+ * + * .grpc.testing.PayloadConfig payload_config = 9; + */ + public function setPayloadConfig(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\PayloadConfig::class); + $this->payload_config = $var; + } + + /** + * 
+     * Specify the cores we should run the server on, if desired
+     *
+ * + * repeated int32 core_list = 10; + */ + public function getCoreList() + { + return $this->core_list; + } + + /** + * 
+     * Specify the cores we should run the server on, if desired
+     *
+ * + * repeated int32 core_list = 10; + */ + public function setCoreList(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::INT32); + $this->core_list = $var; + } + + /** + * 
+     * If we use an OTHER_SERVER client_type, this string gives more detail
+     *
+ * + * string other_server_api = 11; + */ + public function getOtherServerApi() + { + return $this->other_server_api; + } + + /** + * 
+     * If we use an OTHER_SERVER client_type, this string gives more detail
+     *
+ * + * string other_server_api = 11; + */ + public function setOtherServerApi($var) + { + GPBUtil::checkString($var, True); + $this->other_server_api = $var; + } + + /** + * 
+     * Buffer pool size (no buffer pool specified if unset)
+     *
+ * + * int32 resource_quota_size = 1001; + */ + public function getResourceQuotaSize() + { + return $this->resource_quota_size; + } + + /** + * 
+     * Buffer pool size (no buffer pool specified if unset)
+     *
+ * + * int32 resource_quota_size = 1001; + */ + public function setResourceQuotaSize($var) + { + GPBUtil::checkInt32($var); + $this->resource_quota_size = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ServerStats.php b/src/php/tests/qps/generated_code/Grpc/Testing/ServerStats.php new file mode 100644 index 00000000000..98b2af764c9 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ServerStats.php @@ -0,0 +1,191 @@ +grpc.testing.ServerStats + */ +class ServerStats extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * wall clock time change in seconds since last reset
+     *
+ * + * double time_elapsed = 1; + */ + private $time_elapsed = 0.0; + /** + * 
+     * change in user time (in seconds) used by the server since last reset
+     *
+ * + * double time_user = 2; + */ + private $time_user = 0.0; + /** + * 
+     * change in server time (in seconds) used by the server process and all
+     * threads since last reset
+     *
+ * + * double time_system = 3; + */ + private $time_system = 0.0; + /** + * 
+     * change in total cpu time of the server (data from proc/stat)
+     *
+ * + * uint64 total_cpu_time = 4; + */ + private $total_cpu_time = 0; + /** + * 
+     * change in idle time of the server (data from proc/stat)
+     *
+ * + * uint64 idle_cpu_time = 5; + */ + private $idle_cpu_time = 0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Stats::initOnce(); + parent::__construct(); + } + + /** + * 
+     * wall clock time change in seconds since last reset
+     *
+ * + * double time_elapsed = 1; + */ + public function getTimeElapsed() + { + return $this->time_elapsed; + } + + /** + * 
+     * wall clock time change in seconds since last reset
+     *
+ * + * double time_elapsed = 1; + */ + public function setTimeElapsed($var) + { + GPBUtil::checkDouble($var); + $this->time_elapsed = $var; + } + + /** + * 
+     * change in user time (in seconds) used by the server since last reset
+     *
+ * + * double time_user = 2; + */ + public function getTimeUser() + { + return $this->time_user; + } + + /** + * 
+     * change in user time (in seconds) used by the server since last reset
+     *
+ * + * double time_user = 2; + */ + public function setTimeUser($var) + { + GPBUtil::checkDouble($var); + $this->time_user = $var; + } + + /** + * 
+     * change in server time (in seconds) used by the server process and all
+     * threads since last reset
+     *
+ * + * double time_system = 3; + */ + public function getTimeSystem() + { + return $this->time_system; + } + + /** + * 
+     * change in server time (in seconds) used by the server process and all
+     * threads since last reset
+     *
+ * + * double time_system = 3; + */ + public function setTimeSystem($var) + { + GPBUtil::checkDouble($var); + $this->time_system = $var; + } + + /** + * 
+     * change in total cpu time of the server (data from proc/stat)
+     *
+ * + * uint64 total_cpu_time = 4; + */ + public function getTotalCpuTime() + { + return $this->total_cpu_time; + } + + /** + * 
+     * change in total cpu time of the server (data from proc/stat)
+     *
+ * + * uint64 total_cpu_time = 4; + */ + public function setTotalCpuTime($var) + { + GPBUtil::checkUint64($var); + $this->total_cpu_time = $var; + } + + /** + * 
+     * change in idle time of the server (data from proc/stat)
+     *
+ * + * uint64 idle_cpu_time = 5; + */ + public function getIdleCpuTime() + { + return $this->idle_cpu_time; + } + + /** + * 
+     * change in idle time of the server (data from proc/stat)
+     *
+ * + * uint64 idle_cpu_time = 5; + */ + public function setIdleCpuTime($var) + { + GPBUtil::checkUint64($var); + $this->idle_cpu_time = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ServerStatus.php b/src/php/tests/qps/generated_code/Grpc/Testing/ServerStatus.php new file mode 100644 index 00000000000..d293f03fbdf --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ServerStatus.php @@ -0,0 +1,110 @@ +grpc.testing.ServerStatus + */ +class ServerStatus extends \Google\Protobuf\Internal\Message +{ + /** + * .grpc.testing.ServerStats stats = 1; + */ + private $stats = null; + /** + * 
+     * the port bound by the server
+     *
+ * + * int32 port = 2; + */ + private $port = 0; + /** + * 
+     * Number of cores available to the server
+     *
+ * + * int32 cores = 3; + */ + private $cores = 0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + + /** + * .grpc.testing.ServerStats stats = 1; + */ + public function getStats() + { + return $this->stats; + } + + /** + * .grpc.testing.ServerStats stats = 1; + */ + public function setStats(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\ServerStats::class); + $this->stats = $var; + } + + /** + * 
+     * the port bound by the server
+     *
+ * + * int32 port = 2; + */ + public function getPort() + { + return $this->port; + } + + /** + * 
+     * the port bound by the server
+     *
+ * + * int32 port = 2; + */ + public function setPort($var) + { + GPBUtil::checkInt32($var); + $this->port = $var; + } + + /** + * 
+     * Number of cores available to the server
+     *
+ * + * int32 cores = 3; + */ + public function getCores() + { + return $this->cores; + } + + /** + * 
+     * Number of cores available to the server
+     *
+ * + * int32 cores = 3; + */ + public function setCores($var) + { + GPBUtil::checkInt32($var); + $this->cores = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/ServerType.php b/src/php/tests/qps/generated_code/Grpc/Testing/ServerType.php new file mode 100644 index 00000000000..605c83c3f76 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/ServerType.php @@ -0,0 +1,33 @@ +grpc.testing.ServerType + */ +class ServerType +{ + /** + * SYNC_SERVER = 0; + */ + const SYNC_SERVER = 0; + /** + * ASYNC_SERVER = 1; + */ + const ASYNC_SERVER = 1; + /** + * ASYNC_GENERIC_SERVER = 2; + */ + const ASYNC_GENERIC_SERVER = 2; + /** + * 
+     * used for some language-specific variants
+     *
+ * + * OTHER_SERVER = 3; + */ + const OTHER_SERVER = 3; +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/SimpleProtoParams.php b/src/php/tests/qps/generated_code/Grpc/Testing/SimpleProtoParams.php new file mode 100644 index 00000000000..29834a3be71 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/SimpleProtoParams.php @@ -0,0 +1,65 @@ +grpc.testing.SimpleProtoParams + */ +class SimpleProtoParams extends \Google\Protobuf\Internal\Message +{ + /** + * int32 req_size = 1; + */ + private $req_size = 0; + /** + * int32 resp_size = 2; + */ + private $resp_size = 0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Payloads::initOnce(); + parent::__construct(); + } + + /** + * int32 req_size = 1; + */ + public function getReqSize() + { + return $this->req_size; + } + + /** + * int32 req_size = 1; + */ + public function setReqSize($var) + { + GPBUtil::checkInt32($var); + $this->req_size = $var; + } + + /** + * int32 resp_size = 2; + */ + public function getRespSize() + { + return $this->resp_size; + } + + /** + * int32 resp_size = 2; + */ + public function setRespSize($var) + { + GPBUtil::checkInt32($var); + $this->resp_size = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/SimpleRequest.php b/src/php/tests/qps/generated_code/Grpc/Testing/SimpleRequest.php new file mode 100644 index 00000000000..f84c95319f4 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/SimpleRequest.php @@ -0,0 +1,306 @@ + + * Unary request. + * + * + * Protobuf type grpc.testing.SimpleRequest + */ +class SimpleRequest extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * DEPRECATED, don't use. To be removed shortly.
+     * Desired payload type in the response from the server.
+     * If response_type is RANDOM, server randomly chooses one from other formats.
+     *
+ * + * .grpc.testing.PayloadType response_type = 1; + */ + private $response_type = 0; + /** + * 
+     * Desired payload size in the response from the server.
+     *
+ * + * int32 response_size = 2; + */ + private $response_size = 0; + /** + * 
+     * Optional input payload sent along with the request.
+     *
+ * + * .grpc.testing.Payload payload = 3; + */ + private $payload = null; + /** + * 
+     * Whether SimpleResponse should include username.
+     *
+ * + * bool fill_username = 4; + */ + private $fill_username = false; + /** + * 
+     * Whether SimpleResponse should include OAuth scope.
+     *
+ * + * bool fill_oauth_scope = 5; + */ + private $fill_oauth_scope = false; + /** + * 
+     * Whether to request the server to compress the response. This field is
+     * "nullable" in order to interoperate seamlessly with clients not able to
+     * implement the full compression tests by introspecting the call to verify
+     * the response's compression status.
+     *
+ * + * .grpc.testing.BoolValue response_compressed = 6; + */ + private $response_compressed = null; + /** + * 
+     * Whether server should return a given status
+     *
+ * + * .grpc.testing.EchoStatus response_status = 7; + */ + private $response_status = null; + /** + * 
+     * Whether the server should expect this request to be compressed.
+     *
+ * + * .grpc.testing.BoolValue expect_compressed = 8; + */ + private $expect_compressed = null; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * 
+     * DEPRECATED, don't use. To be removed shortly.
+     * Desired payload type in the response from the server.
+     * If response_type is RANDOM, server randomly chooses one from other formats.
+     *
+ * + * .grpc.testing.PayloadType response_type = 1; + */ + public function getResponseType() + { + return $this->response_type; + } + + /** + * 
+     * DEPRECATED, don't use. To be removed shortly.
+     * Desired payload type in the response from the server.
+     * If response_type is RANDOM, server randomly chooses one from other formats.
+     *
+ * + * .grpc.testing.PayloadType response_type = 1; + */ + public function setResponseType($var) + { + GPBUtil::checkEnum($var, \Grpc\Testing\PayloadType::class); + $this->response_type = $var; + } + + /** + * 
+     * Desired payload size in the response from the server.
+     *
+ * + * int32 response_size = 2; + */ + public function getResponseSize() + { + return $this->response_size; + } + + /** + * 
+     * Desired payload size in the response from the server.
+     *
+ * + * int32 response_size = 2; + */ + public function setResponseSize($var) + { + GPBUtil::checkInt32($var); + $this->response_size = $var; + } + + /** + * 
+     * Optional input payload sent along with the request.
+     *
+ * + * .grpc.testing.Payload payload = 3; + */ + public function getPayload() + { + return $this->payload; + } + + /** + * 
+     * Optional input payload sent along with the request.
+     *
+ * + * .grpc.testing.Payload payload = 3; + */ + public function setPayload(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\Payload::class); + $this->payload = $var; + } + + /** + * 
+     * Whether SimpleResponse should include username.
+     *
+ * + * bool fill_username = 4; + */ + public function getFillUsername() + { + return $this->fill_username; + } + + /** + * 
+     * Whether SimpleResponse should include username.
+     *
+ * + * bool fill_username = 4; + */ + public function setFillUsername($var) + { + GPBUtil::checkBool($var); + $this->fill_username = $var; + } + + /** + * 
+     * Whether SimpleResponse should include OAuth scope.
+     *
+ * + * bool fill_oauth_scope = 5; + */ + public function getFillOauthScope() + { + return $this->fill_oauth_scope; + } + + /** + * 
+     * Whether SimpleResponse should include OAuth scope.
+     *
+ * + * bool fill_oauth_scope = 5; + */ + public function setFillOauthScope($var) + { + GPBUtil::checkBool($var); + $this->fill_oauth_scope = $var; + } + + /** + * 
+     * Whether to request the server to compress the response. This field is
+     * "nullable" in order to interoperate seamlessly with clients not able to
+     * implement the full compression tests by introspecting the call to verify
+     * the response's compression status.
+     *
+ * + * .grpc.testing.BoolValue response_compressed = 6; + */ + public function getResponseCompressed() + { + return $this->response_compressed; + } + + /** + * 
+     * Whether to request the server to compress the response. This field is
+     * "nullable" in order to interoperate seamlessly with clients not able to
+     * implement the full compression tests by introspecting the call to verify
+     * the response's compression status.
+     *
+ * + * .grpc.testing.BoolValue response_compressed = 6; + */ + public function setResponseCompressed(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\BoolValue::class); + $this->response_compressed = $var; + } + + /** + * 
+     * Whether server should return a given status
+     *
+ * + * .grpc.testing.EchoStatus response_status = 7; + */ + public function getResponseStatus() + { + return $this->response_status; + } + + /** + * 
+     * Whether server should return a given status
+     *
+ * + * .grpc.testing.EchoStatus response_status = 7; + */ + public function setResponseStatus(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\EchoStatus::class); + $this->response_status = $var; + } + + /** + * 
+     * Whether the server should expect this request to be compressed.
+     *
+ * + * .grpc.testing.BoolValue expect_compressed = 8; + */ + public function getExpectCompressed() + { + return $this->expect_compressed; + } + + /** + * 
+     * Whether the server should expect this request to be compressed.
+     *
+ * + * .grpc.testing.BoolValue expect_compressed = 8; + */ + public function setExpectCompressed(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\BoolValue::class); + $this->expect_compressed = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/SimpleResponse.php b/src/php/tests/qps/generated_code/Grpc/Testing/SimpleResponse.php new file mode 100644 index 00000000000..ccc628ec4c0 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/SimpleResponse.php @@ -0,0 +1,129 @@ + + * Unary response, as configured by the request. + * + * + * Protobuf type grpc.testing.SimpleResponse + */ +class SimpleResponse extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * Payload to increase message size.
+     *
+ * + * .grpc.testing.Payload payload = 1; + */ + private $payload = null; + /** + * 
+     * The user the request came from, for verifying authentication was
+     * successful when the client expected it.
+     *
+ * + * string username = 2; + */ + private $username = ''; + /** + * 
+     * OAuth scope.
+     *
+ * + * string oauth_scope = 3; + */ + private $oauth_scope = ''; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * 
+     * Payload to increase message size.
+     *
+ * + * .grpc.testing.Payload payload = 1; + */ + public function getPayload() + { + return $this->payload; + } + + /** + * 
+     * Payload to increase message size.
+     *
+ * + * .grpc.testing.Payload payload = 1; + */ + public function setPayload(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\Payload::class); + $this->payload = $var; + } + + /** + * 
+     * The user the request came from, for verifying authentication was
+     * successful when the client expected it.
+     *
+ * + * string username = 2; + */ + public function getUsername() + { + return $this->username; + } + + /** + * 
+     * The user the request came from, for verifying authentication was
+     * successful when the client expected it.
+     *
+ * + * string username = 2; + */ + public function setUsername($var) + { + GPBUtil::checkString($var, True); + $this->username = $var; + } + + /** + * 
+     * OAuth scope.
+     *
+ * + * string oauth_scope = 3; + */ + public function getOauthScope() + { + return $this->oauth_scope; + } + + /** + * 
+     * OAuth scope.
+     *
+ * + * string oauth_scope = 3; + */ + public function setOauthScope($var) + { + GPBUtil::checkString($var, True); + $this->oauth_scope = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/StreamingInputCallRequest.php b/src/php/tests/qps/generated_code/Grpc/Testing/StreamingInputCallRequest.php new file mode 100644 index 00000000000..d7bbc707799 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/StreamingInputCallRequest.php @@ -0,0 +1,102 @@ + + * Client-streaming request. + * + * + * Protobuf type grpc.testing.StreamingInputCallRequest + */ +class StreamingInputCallRequest extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * Optional input payload sent along with the request.
+     *
+ * + * .grpc.testing.Payload payload = 1; + */ + private $payload = null; + /** + * 
+     * Whether the server should expect this request to be compressed. This field
+     * is "nullable" in order to interoperate seamlessly with servers not able to
+     * implement the full compression tests by introspecting the call to verify
+     * the request's compression status.
+     *
+ * + * .grpc.testing.BoolValue expect_compressed = 2; + */ + private $expect_compressed = null; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * 
+     * Optional input payload sent along with the request.
+     *
+ * + * .grpc.testing.Payload payload = 1; + */ + public function getPayload() + { + return $this->payload; + } + + /** + * 
+     * Optional input payload sent along with the request.
+     *
+ * + * .grpc.testing.Payload payload = 1; + */ + public function setPayload(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\Payload::class); + $this->payload = $var; + } + + /** + * 
+     * Whether the server should expect this request to be compressed. This field
+     * is "nullable" in order to interoperate seamlessly with servers not able to
+     * implement the full compression tests by introspecting the call to verify
+     * the request's compression status.
+     *
+ * + * .grpc.testing.BoolValue expect_compressed = 2; + */ + public function getExpectCompressed() + { + return $this->expect_compressed; + } + + /** + * 
+     * Whether the server should expect this request to be compressed. This field
+     * is "nullable" in order to interoperate seamlessly with servers not able to
+     * implement the full compression tests by introspecting the call to verify
+     * the request's compression status.
+     *
+ * + * .grpc.testing.BoolValue expect_compressed = 2; + */ + public function setExpectCompressed(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\BoolValue::class); + $this->expect_compressed = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/StreamingInputCallResponse.php b/src/php/tests/qps/generated_code/Grpc/Testing/StreamingInputCallResponse.php new file mode 100644 index 00000000000..fdd1d0dbf8a --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/StreamingInputCallResponse.php @@ -0,0 +1,60 @@ + + * Client-streaming response. + * + * + * Protobuf type grpc.testing.StreamingInputCallResponse + */ +class StreamingInputCallResponse extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * Aggregated size of payloads received from the client.
+     *
+ * + * int32 aggregated_payload_size = 1; + */ + private $aggregated_payload_size = 0; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * 
+     * Aggregated size of payloads received from the client.
+     *
+ * + * int32 aggregated_payload_size = 1; + */ + public function getAggregatedPayloadSize() + { + return $this->aggregated_payload_size; + } + + /** + * 
+     * Aggregated size of payloads received from the client.
+     *
+ * + * int32 aggregated_payload_size = 1; + */ + public function setAggregatedPayloadSize($var) + { + GPBUtil::checkInt32($var); + $this->aggregated_payload_size = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/StreamingOutputCallRequest.php b/src/php/tests/qps/generated_code/Grpc/Testing/StreamingOutputCallRequest.php new file mode 100644 index 00000000000..2aab5fadad7 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/StreamingOutputCallRequest.php @@ -0,0 +1,171 @@ + + * Server-streaming request. + * + * + * Protobuf type grpc.testing.StreamingOutputCallRequest + */ +class StreamingOutputCallRequest extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * DEPRECATED, don't use. To be removed shortly.
+     * Desired payload type in the response from the server.
+     * If response_type is RANDOM, the payload from each response in the stream
+     * might be of different types. This is to simulate a mixed type of payload
+     * stream.
+     *
+ * + * .grpc.testing.PayloadType response_type = 1; + */ + private $response_type = 0; + /** + * 
+     * Configuration for each expected response message.
+     *
+ * + * repeated .grpc.testing.ResponseParameters response_parameters = 2; + */ + private $response_parameters; + /** + * 
+     * Optional input payload sent along with the request.
+     *
+ * + * .grpc.testing.Payload payload = 3; + */ + private $payload = null; + /** + * 
+     * Whether server should return a given status
+     *
+ * + * .grpc.testing.EchoStatus response_status = 7; + */ + private $response_status = null; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * 
+     * DEPRECATED, don't use. To be removed shortly.
+     * Desired payload type in the response from the server.
+     * If response_type is RANDOM, the payload from each response in the stream
+     * might be of different types. This is to simulate a mixed type of payload
+     * stream.
+     *
+ * + * .grpc.testing.PayloadType response_type = 1; + */ + public function getResponseType() + { + return $this->response_type; + } + + /** + * 
+     * DEPRECATED, don't use. To be removed shortly.
+     * Desired payload type in the response from the server.
+     * If response_type is RANDOM, the payload from each response in the stream
+     * might be of different types. This is to simulate a mixed type of payload
+     * stream.
+     *
+ * + * .grpc.testing.PayloadType response_type = 1; + */ + public function setResponseType($var) + { + GPBUtil::checkEnum($var, \Grpc\Testing\PayloadType::class); + $this->response_type = $var; + } + + /** + * 
+     * Configuration for each expected response message.
+     *
+ * + * repeated .grpc.testing.ResponseParameters response_parameters = 2; + */ + public function getResponseParameters() + { + return $this->response_parameters; + } + + /** + * 
+     * Configuration for each expected response message.
+     *
+ * + * repeated .grpc.testing.ResponseParameters response_parameters = 2; + */ + public function setResponseParameters(&$var) + { + GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::MESSAGE, \Grpc\Testing\ResponseParameters::class); + $this->response_parameters = $var; + } + + /** + * 
+     * Optional input payload sent along with the request.
+     *
+ * + * .grpc.testing.Payload payload = 3; + */ + public function getPayload() + { + return $this->payload; + } + + /** + * 
+     * Optional input payload sent along with the request.
+     *
+ * + * .grpc.testing.Payload payload = 3; + */ + public function setPayload(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\Payload::class); + $this->payload = $var; + } + + /** + * 
+     * Whether server should return a given status
+     *
+ * + * .grpc.testing.EchoStatus response_status = 7; + */ + public function getResponseStatus() + { + return $this->response_status; + } + + /** + * 
+     * Whether server should return a given status
+     *
+ * + * .grpc.testing.EchoStatus response_status = 7; + */ + public function setResponseStatus(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\EchoStatus::class); + $this->response_status = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/StreamingOutputCallResponse.php b/src/php/tests/qps/generated_code/Grpc/Testing/StreamingOutputCallResponse.php new file mode 100644 index 00000000000..c06c78c9d8f --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/StreamingOutputCallResponse.php @@ -0,0 +1,60 @@ + + * Server-streaming response, as configured by the request and parameters. + * + * + * Protobuf type grpc.testing.StreamingOutputCallResponse + */ +class StreamingOutputCallResponse extends \Google\Protobuf\Internal\Message +{ + /** + * 
+     * Payload to increase response size.
+     *
+ * + * .grpc.testing.Payload payload = 1; + */ + private $payload = null; + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct(); + } + + /** + * 
+     * Payload to increase response size.
+     *
+ * + * .grpc.testing.Payload payload = 1; + */ + public function getPayload() + { + return $this->payload; + } + + /** + * 
+     * Payload to increase response size.
+     *
+ * + * .grpc.testing.Payload payload = 1; + */ + public function setPayload(&$var) + { + GPBUtil::checkMessage($var, \Grpc\Testing\Payload::class); + $this->payload = $var; + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/Void.php b/src/php/tests/qps/generated_code/Grpc/Testing/Void.php new file mode 100644 index 00000000000..38c100845a0 --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/Void.php @@ -0,0 +1,23 @@ +grpc.testing.Void + */ +class Void extends \Google\Protobuf\Internal\Message +{ + + public function __construct() { + \GPBMetadata\Src\Proto\Grpc\Testing\Control::initOnce(); + parent::__construct(); + } + +} + diff --git a/src/php/tests/qps/generated_code/Grpc/Testing/WorkerServiceClient.php b/src/php/tests/qps/generated_code/Grpc/Testing/WorkerServiceClient.php new file mode 100644 index 00000000000..0a68e41269e --- /dev/null +++ b/src/php/tests/qps/generated_code/Grpc/Testing/WorkerServiceClient.php @@ -0,0 +1,111 @@ +_bidiRequest('/grpc.testing.WorkerService/RunServer', + ['\Grpc\Testing\ServerStatus','decode'], + $metadata, $options); + } + + /** + * Start client with specified workload. + * First request sent specifies the ClientConfig followed by ClientStatus + * response. After that, a "Mark" can be sent anytime to request the latest + * stats. Closing the stream will initiate shutdown of the test client + * and once the shutdown has finished, the OK status is sent to terminate + * this RPC. + * @param array $metadata metadata + * @param array $options call options + */ + public function RunClient($metadata = [], $options = []) { + return $this->_bidiRequest('/grpc.testing.WorkerService/RunClient', + ['\Grpc\Testing\ClientStatus','decode'], + $metadata, $options); + } + + /** + * Just return the core count - unary call + * @param \Grpc\Testing\CoreRequest $argument input argument + * @param array $metadata metadata + * @param array $options call options + */ + public function CoreCount(\Grpc\Testing\CoreRequest $argument, + $metadata = [], $options = []) { + return $this->_simpleRequest('/grpc.testing.WorkerService/CoreCount', + $argument, + ['\Grpc\Testing\CoreResponse', 'decode'], + $metadata, $options); + } + + /** + * Quit this worker + * @param \Grpc\Testing\Void $argument input argument + * @param array $metadata metadata + * @param array $options call options + */ + public function QuitWorker(\Grpc\Testing\Void $argument, + $metadata = [], $options = []) { + return $this->_simpleRequest('/grpc.testing.WorkerService/QuitWorker', + $argument, + ['\Grpc\Testing\Void', 'decode'], + $metadata, $options); + } + + } + +} diff --git a/src/proto/grpc/testing/proxy-service.proto b/src/proto/grpc/testing/proxy-service.proto new file mode 100644 index 00000000000..7b7de8d5494 --- /dev/null +++ b/src/proto/grpc/testing/proxy-service.proto @@ -0,0 +1,44 @@ +// Copyright 2017, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +import "src/proto/grpc/testing/control.proto"; + +package grpc.testing; + +message ProxyStat { + double latency = 1; +} + +service ProxyClientService { + rpc GetConfig(Void) returns (ClientConfig); + rpc ReportTime(stream ProxyStat) returns (Void); +} + diff --git a/src/ruby/qps/proxy-worker.rb b/src/ruby/qps/proxy-worker.rb new file mode 100755 index 00000000000..077920d1d3c --- /dev/null +++ b/src/ruby/qps/proxy-worker.rb @@ -0,0 +1,160 @@ +#!/usr/bin/env ruby + +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Proxy of worker service implementation for running a PHP client + +this_dir = File.expand_path(File.dirname(__FILE__)) +lib_dir = File.join(File.dirname(this_dir), 'lib') +$LOAD_PATH.unshift(lib_dir) unless $LOAD_PATH.include?(lib_dir) +$LOAD_PATH.unshift(this_dir) unless $LOAD_PATH.include?(this_dir) + +require 'grpc' +require 'optparse' +require 'histogram' +require 'etc' +require 'facter' +require 'qps-common' +require 'src/proto/grpc/testing/services_services_pb' +require 'src/proto/grpc/testing/proxy-service_services_pb' + +class ProxyBenchmarkClientServiceImpl < Grpc::Testing::ProxyClientService::Service + def initialize(port) + @mytarget = "localhost:" + port.to_s + end + def setup(config) + @config = config + @histres = config.histogram_params.resolution + @histmax = config.histogram_params.max_possible + @histogram = Histogram.new(@histres, @histmax) + @start_time = Time.now + # TODO(vjpai): Support multiple client channels by spawning off a PHP client per channel + command = "php " + File.expand_path(File.dirname(__FILE__)) + "/../../php/tests/qps/client.php " + @mytarget + puts "Starting command: " + command + @php_pid = spawn(command) + end + def stop + Process.kill("TERM", @php_pid) + Process.wait(@php_pid) + end + def get_config(_args, _call) + puts "Answering get_config" + @config + end + def report_time(call) + puts "Starting a time reporting stream" + call.each_remote_read do |lat| + @histogram.add((lat.latency)*1e9) + end + Grpc::Testing::Void.new + end + def mark(reset) + lat = Grpc::Testing::HistogramData.new( + bucket: @histogram.contents, + min_seen: @histogram.minimum, + max_seen: @histogram.maximum, + sum: @histogram.sum, + sum_of_squares: @histogram.sum_of_squares, + count: @histogram.count + ) + elapsed = Time.now-@start_time + if reset + @start_time = Time.now + @histogram = Histogram.new(@histres, @histmax) + end + Grpc::Testing::ClientStats.new(latencies: lat, time_elapsed: elapsed) + end +end + +class ProxyWorkerServiceImpl < Grpc::Testing::WorkerService::Service + def cpu_cores + Facter.value('processors')['count'] + end + # Leave run_server unimplemented since this proxies for a client only. + # If the driver tries to use this as a server, it will get an unimplemented + # status return value. + def run_client(reqs) + q = EnumeratorQueue.new(self) + Thread.new { + reqs.each do |req| + case req.argtype.to_s + when 'setup' + @bmc.setup(req.setup) + q.push(Grpc::Testing::ClientStatus.new(stats: @bmc.mark(false))) + when 'mark' + q.push(Grpc::Testing::ClientStatus.new(stats: + @bmc.mark(req.mark.reset))) + end + end + @bmc.stop + q.push(self) + } + q.each_item + end + def core_count(_args, _call) + Grpc::Testing::CoreResponse.new(cores: cpu_cores) + end + def quit_worker(_args, _call) + Thread.new { + sleep 3 + @server.stop + } + Grpc::Testing::Void.new + end + def initialize(s, bmc) + @server = s + @bmc = bmc + end +end + +def proxymain + options = { + 'driver_port' => 0 + } + OptionParser.new do |opts| + opts.banner = 'Usage: [--driver_port ]' + opts.on('--driver_port PORT', '') do |v| + options['driver_port'] = v + end + end.parse! + + # Configure any errors with client or server child threads to surface + Thread.abort_on_exception = true + + s = GRPC::RpcServer.new + port = s.add_http2_port("0.0.0.0:" + options['driver_port'].to_s, + :this_port_is_insecure) + bmc = ProxyBenchmarkClientServiceImpl.new(port) + s.handle(bmc) + s.handle(ProxyWorkerServiceImpl.new(s, bmc)) + s.run +end + +proxymain diff --git a/src/ruby/qps/src/proto/grpc/testing/proxy-service_pb.rb b/src/ruby/qps/src/proto/grpc/testing/proxy-service_pb.rb new file mode 100644 index 00000000000..d238198cca3 --- /dev/null +++ b/src/ruby/qps/src/proto/grpc/testing/proxy-service_pb.rb @@ -0,0 +1,17 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: src/proto/grpc/testing/proxy-service.proto + +require 'google/protobuf' + +require 'src/proto/grpc/testing/control_pb' +Google::Protobuf::DescriptorPool.generated_pool.build do + add_message "grpc.testing.ProxyStat" do + optional :latency, :double, 1 + end +end + +module Grpc + module Testing + ProxyStat = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.ProxyStat").msgclass + end +end diff --git a/src/ruby/qps/src/proto/grpc/testing/proxy-service_services_pb.rb b/src/ruby/qps/src/proto/grpc/testing/proxy-service_services_pb.rb new file mode 100644 index 00000000000..37ddbf5b030 --- /dev/null +++ b/src/ruby/qps/src/proto/grpc/testing/proxy-service_services_pb.rb @@ -0,0 +1,55 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# Source: src/proto/grpc/testing/proxy-service.proto for package 'grpc.testing' +# Original file comments: +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +require 'grpc' +require 'src/proto/grpc/testing/proxy-service_pb' + +module Grpc + module Testing + module ProxyClientService + class Service + + include GRPC::GenericService + + self.marshal_class_method = :encode + self.unmarshal_class_method = :decode + self.service_name = 'grpc.testing.ProxyClientService' + + rpc :GetConfig, Void, ClientConfig + rpc :ReportTime, stream(ProxyStat), Void + end + + Stub = Service.rpc_stub_class + end + end +end From dc87ef5836c1bc0f885941567beee51608056db0 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 10 Mar 2017 11:41:53 -0800 Subject: [PATCH 109/188] Use k-contant and add comment for encoding overhead --- src/objective-c/tests/InteropTestsLocalCleartext.m | 6 +++++- src/objective-c/tests/InteropTestsLocalSSL.m | 6 +++++- src/objective-c/tests/InteropTestsRemote.m | 6 +++++- .../InteropTestsRemoteWithCronet.m | 6 +++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/objective-c/tests/InteropTestsLocalCleartext.m b/src/objective-c/tests/InteropTestsLocalCleartext.m index b41210f50f8..49876608087 100644 --- a/src/objective-c/tests/InteropTestsLocalCleartext.m +++ b/src/objective-c/tests/InteropTestsLocalCleartext.m @@ -37,6 +37,10 @@ static NSString * const kLocalCleartextHost = @"localhost:5050"; +// The Protocol Buffers encoding overhead of local interop server. Acquired +// by experiment. Adjust this when server's proto file changes. +static int32_t kLocalInteropServerOverhead = 10; + /** Tests in InteropTests.m, sending the RPCs to a local cleartext server. */ @interface InteropTestsLocalCleartext : InteropTests @end @@ -48,7 +52,7 @@ static NSString * const kLocalCleartextHost = @"localhost:5050"; } - (int32_t)encodingOverhead { - return 10; // bytes + return kLocalInteropServerOverhead; // bytes } - (void)setUp { diff --git a/src/objective-c/tests/InteropTestsLocalSSL.m b/src/objective-c/tests/InteropTestsLocalSSL.m index 1479c5896c3..934d500abc0 100644 --- a/src/objective-c/tests/InteropTestsLocalSSL.m +++ b/src/objective-c/tests/InteropTestsLocalSSL.m @@ -37,6 +37,10 @@ static NSString * const kLocalSSLHost = @"localhost:5051"; +// The Protocol Buffers encoding overhead of local interop server. Acquired +// by experiment. Adjust this when server's proto file changes. +static int32_t kLocalInteropServerOverhead = 10; + /** Tests in InteropTests.m, sending the RPCs to a local SSL server. */ @interface InteropTestsLocalSSL : InteropTests @end @@ -48,7 +52,7 @@ static NSString * const kLocalSSLHost = @"localhost:5051"; } - (int32_t)encodingOverhead { - return 10; // bytes + return kLocalInteropServerOverhead; // bytes } - (void)setUp { diff --git a/src/objective-c/tests/InteropTestsRemote.m b/src/objective-c/tests/InteropTestsRemote.m index 70f84753bb6..9fb30aa43d0 100644 --- a/src/objective-c/tests/InteropTestsRemote.m +++ b/src/objective-c/tests/InteropTestsRemote.m @@ -37,6 +37,10 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; +// The Protocol Buffers encoding overhead of remote interop server. Acquired +// by experiment. Adjust this when server's proto file changes. +static int32_t kRemoteInteropServerOverhead = 12; + /** Tests in InteropTests.m, sending the RPCs to a remote SSL server. */ @interface InteropTestsRemote : InteropTests @end @@ -48,7 +52,7 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; } - (int32_t)encodingOverhead { - return 12; // bytes + return kRemoteInteropServerOverhead; // bytes } @end diff --git a/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m b/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m index 7bc303ac10d..005bac0a0d4 100644 --- a/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m +++ b/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m @@ -37,6 +37,10 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; +// The Protocol Buffers encoding overhead of remote interop server. Acquired +// by experiment. Adjust this when server's proto file changes. +static int32_t kRemoteInteropServerOverhead = 12; + /** Tests in InteropTests.m, sending the RPCs to a remote SSL server. */ @interface InteropTestsRemoteWithCronet : InteropTests @end @@ -48,7 +52,7 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; } - (int32_t)encodingOverhead { - return 12; // bytes + return kRemoteInteropServerOverhead; // bytes } @end From a582defdad755f76b55f9e7f4dc3614bb18452b7 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 9 Mar 2017 05:47:46 -0800 Subject: [PATCH 110/188] Temporarily disable resource quota QPS tests --- tools/run_tests/generated/tests.json | 488 ------------------ .../run_tests/performance/scenario_config.py | 21 +- 2 files changed, 12 insertions(+), 497 deletions(-) diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index f9e2a19d0ef..ec55fb73b97 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -39636,31 +39636,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 1024, - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -39711,31 +39686,6 @@ "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -39786,31 +39736,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_secure", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 1024, - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -39861,31 +39786,6 @@ "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -40065,31 +39965,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 1024, - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -40140,31 +40015,6 @@ "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -40215,31 +40065,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_insecure", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 1024, - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -40290,31 +40115,6 @@ "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "tsan", - "asan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -40571,42 +40371,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_low_thread_count", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 64, - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -40679,42 +40443,6 @@ "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure_low_thread_count", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -40787,42 +40515,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_secure_low_thread_count", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 64, - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -40895,42 +40587,6 @@ "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure_low_thread_count", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -41187,42 +40843,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_low_thread_count", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 64, - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -41295,42 +40915,6 @@ "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure_low_thread_count", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -41403,42 +40987,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_insecure_low_thread_count", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": 64, - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, { "args": [ "--scenarios_json", @@ -41511,42 +41059,6 @@ "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure_low_thread_count", "timeout_seconds": 360 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux" - ], - "cpu_cost": "capacity", - "defaults": "boringssl", - "exclude_configs": [ - "asan-noleaks", - "asan-trace-cmp", - "basicprof", - "counters", - "dbg", - "gcov", - "helgrind", - "memcheck", - "msan", - "mutrace", - "opt", - "stapprof", - "ubsan" - ], - "excluded_poll_engines": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux" - ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota_low_thread_count", - "timeout_seconds": 360 - }, { "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/00.bin" diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 830b8c0849d..1d91b61ba45 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -275,15 +275,18 @@ class CXXLanguage: secure=secure, categories=smoketest_categories+[SCALABLE]) - yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), - rpc_type=rpc_type.upper(), - client_type='%s_CLIENT' % synchronicity.upper(), - server_type='%s_SERVER' % synchronicity.upper(), - unconstrained_client=synchronicity, - secure=secure, - categories=smoketest_categories+[SCALABLE], - resource_quota_size=500*1024) + # TODO(vjpai): Re-enable this test. It has a lot of timeouts + # and hasn't yet been conclusively identified as a test failure + # or race in the library + # yield _ping_pong_scenario( + # 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), + # rpc_type=rpc_type.upper(), + # client_type='%s_CLIENT' % synchronicity.upper(), + # server_type='%s_SERVER' % synchronicity.upper(), + # unconstrained_client=synchronicity, + # secure=secure, + # categories=smoketest_categories+[SCALABLE], + # resource_quota_size=500*1024) for channels in geometric_progression(1, 20000, math.sqrt(10)): for outstanding in geometric_progression(1, 200000, math.sqrt(10)): From 34887809493347c9455027f9ae7fa6023d6e9bcc Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 10 Mar 2017 13:39:01 -0800 Subject: [PATCH 111/188] Handle floats in bm_json --- tools/profiling/microbenchmarks/bm_json.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index 604823712f9..c48a63b791f 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -103,11 +103,11 @@ _BM_SPECS = { def numericalize(s): if not s: return '' if s[-1] == 'k': - return int(s[:-1]) * 1024 + return float(s[:-1]) * 1024 if s[-1] == 'M': - return int(s[:-1]) * 1024 * 1024 + return float(s[:-1]) * 1024 * 1024 if 0 <= (ord(s[-1]) - ord('0')) <= 9: - return int(s) + return float(s) assert 'not a number: %s' % s def parse_name(name): From ff023ef7e3315e799740c6228783a4a886d8d763 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Fri, 10 Mar 2017 11:43:48 -0800 Subject: [PATCH 112/188] Fix bm_diff to handle 0s --- tools/profiling/microbenchmarks/bm_diff.py | 63 ++++++++++++++-------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 59cc2d0e475..14ea24f65e6 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -32,14 +32,45 @@ import sys import json import bm_json import tabulate +import argparse -with open(sys.argv[1]) as f: +def changed_ratio(n, o): + if float(o) <= .0001: o = 0 + if float(n) <= .0001: n = 0 + if o == 0 and n == 0: return 0 + if o == 0: return 100 + return (float(n)-float(o))/float(o) + +def min_change(pct): + return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 + +_INTERESTING = { + 'cpu_time': min_change(10), + 'real_time': min_change(10), + 'locks_per_iteration': min_change(5), + 'allocs_per_iteration': min_change(5), + 'writes_per_iteration': min_change(5), + 'atm_cas_per_iteration': min_change(1), + 'atm_add_per_iteration': min_change(5), +} + +argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks') +argp.add_argument('-t', '--track', + choices=sorted(_INTERESTING.keys()), + nargs='+', + default=sorted(_INTERESTING.keys()), + help='Which metrics to track') +argp.add_argument('files', metavar='bm_file.json', type=str, nargs=4, + help='files to diff. ') +args = argp.parse_args() + +with open(args.files[0]) as f: js_new_ctr = json.loads(f.read()) -with open(sys.argv[2]) as f: +with open(args.files[1]) as f: js_new_opt = json.loads(f.read()) -with open(sys.argv[3]) as f: +with open(args.files[2]) as f: js_old_ctr = json.loads(f.read()) -with open(sys.argv[4]) as f: +with open(args.files[3]) as f: js_old_opt = json.loads(f.read()) new = {} @@ -50,24 +81,9 @@ for row in bm_json.expand_json(js_new_ctr, js_new_opt): for row in bm_json.expand_json(js_old_ctr, js_old_opt): old[row['cpp_name']] = row -def changed_ratio(n, o): - return float(n-o)/float(o) - -def min_change(pct): - return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 - -_INTERESTING = ( - ('cpu_time', min_change(10)), - ('real_time', min_change(10)), - ('locks_per_iteration', min_change(5)), - ('allocs_per_iteration', min_change(5)), - ('writes_per_iteration', min_change(5)), - ('atm_cas_per_iteration', min_change(1)), - ('atm_add_per_iteration', min_change(5)), -) - changed = [] -for fld, chk in _INTERESTING: +for fld in args.track: + chk = _INTERESTING[fld] for bm in new.keys(): if bm not in old: continue n = new[bm] @@ -86,12 +102,13 @@ for bm in sorted(new.keys()): n = new[bm] o = old[bm] details = '' - for fld, chk in _INTERESTING: + for fld in args.track: + chk = _INTERESTING[fld] if fld not in n or fld not in o: continue if chk(n[fld], o[fld]): row.append(changed_ratio(n[fld], o[fld])) if details: details += ', ' - details += '%s:%r-->%r' % (fld, o[fld], n[fld]) + details += '%s:%r-->%r' % (fld, float(o[fld]), float(n[fld])) any_changed = True else: row.append('') From 6a4c1a37cc41d4f8114ea8116134d510af5f2dc6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 10 Mar 2017 14:18:00 -0800 Subject: [PATCH 113/188] Add benchmark to list --- tools/profiling/microbenchmarks/bm_json.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index c48a63b791f..4695f829f41 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -98,6 +98,10 @@ _BM_SPECS = { 'tpl': [], 'dyn': ['request_size'], }, + 'BM_PollEmptyPollset_SpeedOfLight': { + 'tpl': [], + 'dyn': ['request_size', 'request_count'], + } } def numericalize(s): From c5b90df9b2738076acf6a6972d833489f6470335 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 10 Mar 2017 16:11:08 -0800 Subject: [PATCH 114/188] debug --- src/core/lib/surface/call.c | 74 +++++++++--------------------- src/core/lib/transport/transport.c | 3 +- 2 files changed, 24 insertions(+), 53 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 01efcafceba..8b2f9fd3f53 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -143,9 +143,8 @@ struct grpc_call { grpc_channel *channel; grpc_call *parent; grpc_call *first_child; - gpr_atm has_children; gpr_timespec start_time; - /* protects first_child, setting has_children, and child next/prev links */ + /* protects first_child, and child next/prev links */ gpr_mu child_list_mu; /* client or server call */ @@ -315,7 +314,6 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, GPR_ASSERT(!args->parent_call->is_client); gpr_mu_lock(&args->parent_call->child_list_mu); - gpr_atm_rel_store(&args->parent_call->has_children, 1); if (args->propagation_mask & GRPC_PROPAGATE_DEADLINE) { send_deadline = gpr_time_min( @@ -566,45 +564,19 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c, return GRPC_CALL_OK; } -typedef struct termination_closure { - grpc_closure closure; - grpc_call *call; - grpc_transport_stream_op op; -} termination_closure; - -static void done_termination(grpc_exec_ctx *exec_ctx, void *tcp, - grpc_error *error) { - termination_closure *tc = tcp; - GRPC_CALL_INTERNAL_UNREF(exec_ctx, tc->call, "termination"); - gpr_free(tc); -} - -static void send_termination(grpc_exec_ctx *exec_ctx, void *tcp, +static void done_termination(grpc_exec_ctx *exec_ctx, void *call, grpc_error *error) { - termination_closure *tc = tcp; - memset(&tc->op, 0, sizeof(tc->op)); - tc->op.cancel_error = GRPC_ERROR_REF(error); - /* reuse closure to catch completion */ - tc->op.on_complete = grpc_closure_init(&tc->closure, done_termination, tc, - grpc_schedule_on_exec_ctx); - execute_op(exec_ctx, tc->call, &tc->op); -} - -static void terminate_with_error(grpc_exec_ctx *exec_ctx, grpc_call *c, - grpc_error *error) { - termination_closure *tc = gpr_malloc(sizeof(*tc)); - memset(tc, 0, sizeof(*tc)); - tc->call = c; - GRPC_CALL_INTERNAL_REF(tc->call, "termination"); - grpc_closure_sched(exec_ctx, grpc_closure_init(&tc->closure, send_termination, - tc, grpc_schedule_on_exec_ctx), - error); + GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "termination"); } static void cancel_with_error(grpc_exec_ctx *exec_ctx, grpc_call *c, status_source source, grpc_error *error) { + GRPC_CALL_INTERNAL_REF(c, "termination"); set_status_from_error(exec_ctx, c, source, GRPC_ERROR_REF(error)); - terminate_with_error(exec_ctx, c, error); + grpc_transport_stream_op *op = grpc_make_transport_stream_op( + grpc_closure_create(done_termination, c, grpc_schedule_on_exec_ctx)); + op->cancel_error = error; + execute_op(exec_ctx, c, op); } static grpc_error *error_from_status(grpc_status_code status, @@ -1100,23 +1072,21 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, gpr_atm_rel_store(&call->received_final_op_atm, 1); /* propagate cancellation to any interested children */ - if (gpr_atm_acq_load(&call->has_children)) { - gpr_mu_lock(&call->child_list_mu); - child_call = call->first_child; - if (child_call != NULL) { - do { - next_child_call = child_call->sibling_next; - if (child_call->cancellation_is_inherited) { - GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel"); - cancel_with_error(exec_ctx, call, STATUS_FROM_API_OVERRIDE, - GRPC_ERROR_CANCELLED); - GRPC_CALL_INTERNAL_UNREF(exec_ctx, child_call, "propagate_cancel"); - } - child_call = next_child_call; - } while (child_call != call->first_child); - } - gpr_mu_unlock(&call->child_list_mu); + gpr_mu_lock(&call->child_list_mu); + child_call = call->first_child; + if (child_call != NULL) { + do { + next_child_call = child_call->sibling_next; + if (child_call->cancellation_is_inherited) { + GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel"); + cancel_with_error(exec_ctx, call, STATUS_FROM_API_OVERRIDE, + GRPC_ERROR_CANCELLED); + GRPC_CALL_INTERNAL_UNREF(exec_ctx, child_call, "propagate_cancel"); + } + child_call = next_child_call; + } while (child_call != call->first_child); } + gpr_mu_unlock(&call->child_list_mu); if (call->is_client) { get_final_status(call, set_status_value_directly, diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index 165950e288e..3024f2ae781 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -254,8 +254,9 @@ typedef struct { static void destroy_made_transport_stream_op(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { made_transport_stream_op *op = arg; - grpc_closure_sched(exec_ctx, op->inner_on_complete, GRPC_ERROR_REF(error)); + grpc_closure *c = op->inner_on_complete; gpr_free(op); + grpc_closure_run(exec_ctx, c, GRPC_ERROR_REF(error)); } grpc_transport_stream_op *grpc_make_transport_stream_op( From bbfb25bf708688748fd9bf37c03ea7cfc5bcc410 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 10 Mar 2017 17:24:45 -0800 Subject: [PATCH 115/188] Sanitize before bigquery upload --- tools/profiling/microbenchmarks/bm2bq.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index ae59332f1b3..ffb11f57d8f 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -73,6 +73,14 @@ columns = [ ('framing_bytes_per_iteration', 'float'), ] +SANITIZE = { + 'integer': int, + 'float': float, + 'boolean': bool, + 'string': str, + 'timestamp': str, +} + if sys.argv[1] == '--schema': print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns) sys.exit(0) @@ -89,7 +97,10 @@ else: writer = csv.DictWriter(sys.stdout, [c for c,t in columns]) for row in bm_json.expand_json(js, js2): - if 'label' in row: - del row['label'] - del row['cpp_name'] - writer.writerow(row) + sane_row = {} + for name, sql_type in columns: + if name in row: + if row[name] == '': continue + sane_row[name] = SANITIZE[sql_type](row[name]) + writer.writerow(sane_row) + From a08b11f33a7a7046d8d329b5c331fa93b2d479e6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 10 Mar 2017 20:34:08 -0800 Subject: [PATCH 116/188] Dont track allocations if we are not going to use them --- test/cpp/microbenchmarks/helpers.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h index 2829a46e5a5..f44b7cf83a7 100644 --- a/test/cpp/microbenchmarks/helpers.h +++ b/test/cpp/microbenchmarks/helpers.h @@ -55,7 +55,9 @@ class Library { private: Library() { +#ifdef GPR_LOW_LEVEL_COUNTERS grpc_memory_counters_init(); +#endif init_lib_.init(); rq_ = grpc_resource_quota_create("bm"); } From 75a41b4532fa82f02698a7fa01c361330a67029c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 10 Mar 2017 20:44:25 -0800 Subject: [PATCH 117/188] Hide memory counters from atomic counters --- test/core/util/memory_counters.c | 48 ++++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/core/util/memory_counters.c b/test/core/util/memory_counters.c index 7c8b620f34f..c27065f2605 100644 --- a/test/core/util/memory_counters.c +++ b/test/core/util/memory_counters.c @@ -46,17 +46,23 @@ static void *guard_malloc(size_t size); static void *guard_realloc(void *vptr, size_t size); static void guard_free(void *vptr); +#ifdef GPR_LOW_LEVEL_COUNTERS +/* hide these from the microbenchmark atomic stats */ +#define NO_BARRIER_FETCH_ADD(x, sz) \ + __atomic_fetch_add((x), (sz), __ATOMIC_RELAXED) +#define NO_BARRIER_LOAD(x) __atomic_load_n((x), __ATOMIC_RELAXED) +#else +#define NO_BARRIER_FETCH_ADD(x, sz) gpr_atm_no_barrier_fetch_add(x, sz) +#define NO_BARRIER_LOAD(x) gpr_atm_no_barrier_load(x) +#endif + static void *guard_malloc(size_t size) { size_t *ptr; if (!size) return NULL; - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_absolute, - (gpr_atm)size); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, - (gpr_atm)size); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_absolute, - (gpr_atm)1); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_relative, - (gpr_atm)1); + NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_absolute, (gpr_atm)size); + NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_relative, (gpr_atm)size); + NO_BARRIER_FETCH_ADD(&g_memory_counters.total_allocs_absolute, (gpr_atm)1); + NO_BARRIER_FETCH_ADD(&g_memory_counters.total_allocs_relative, (gpr_atm)1); ptr = g_old_allocs.malloc_fn(size + sizeof(size)); *ptr++ = size; return ptr; @@ -72,14 +78,10 @@ static void *guard_realloc(void *vptr, size_t size) { return NULL; } --ptr; - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_absolute, - (gpr_atm)size); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, - -(gpr_atm)*ptr); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, - (gpr_atm)size); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_absolute, - (gpr_atm)1); + NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_absolute, (gpr_atm)size); + NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_relative, -(gpr_atm)*ptr); + NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_relative, (gpr_atm)size); + NO_BARRIER_FETCH_ADD(&g_memory_counters.total_allocs_absolute, (gpr_atm)1); ptr = g_old_allocs.realloc_fn(ptr, size + sizeof(size)); *ptr++ = size; return ptr; @@ -89,10 +91,8 @@ static void guard_free(void *vptr) { size_t *ptr = vptr; if (!vptr) return; --ptr; - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative, - -(gpr_atm)*ptr); - gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_relative, - -(gpr_atm)1); + NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_relative, -(gpr_atm)*ptr); + NO_BARRIER_FETCH_ADD(&g_memory_counters.total_allocs_relative, -(gpr_atm)1); g_old_allocs.free_fn(ptr); } @@ -112,12 +112,12 @@ void grpc_memory_counters_destroy() { struct grpc_memory_counters grpc_memory_counters_snapshot() { struct grpc_memory_counters counters; counters.total_size_relative = - gpr_atm_no_barrier_load(&g_memory_counters.total_size_relative); + NO_BARRIER_LOAD(&g_memory_counters.total_size_relative); counters.total_size_absolute = - gpr_atm_no_barrier_load(&g_memory_counters.total_size_absolute); + NO_BARRIER_LOAD(&g_memory_counters.total_size_absolute); counters.total_allocs_relative = - gpr_atm_no_barrier_load(&g_memory_counters.total_allocs_relative); + NO_BARRIER_LOAD(&g_memory_counters.total_allocs_relative); counters.total_allocs_absolute = - gpr_atm_no_barrier_load(&g_memory_counters.total_allocs_absolute); + NO_BARRIER_LOAD(&g_memory_counters.total_allocs_absolute); return counters; } From fe6bc6999c22f42d5b5b015fccec754f2c00d147 Mon Sep 17 00:00:00 2001 From: Siddharth Shukla Date: Wed, 15 Feb 2017 20:18:20 +0100 Subject: [PATCH 118/188] Skip split_definitions_test if PyPy Skip SameSeparateTest, SameCommonTest, SplitCommonTest, and SplitSeparateTest if the interpreter is PyPy. --- .../tests/protoc_plugin/_split_definitions_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py index db938e65459..6f9269dd402 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py @@ -42,6 +42,7 @@ import sys import tempfile import threading import unittest +import platform import grpc from grpc_tools import protoc @@ -150,6 +151,8 @@ class CommonTestMixin(object): self.assertEqual(expected_response, response) +@unittest.skipIf(platform.python_implementation() == "PyPy", + "Skip test if run with PyPy") class SameSeparateTest(unittest.TestCase, SeparateTestMixin): def setUp(self): @@ -191,6 +194,8 @@ class SameSeparateTest(unittest.TestCase, SeparateTestMixin): shutil.rmtree(self.directory) +@unittest.skipIf(platform.python_implementation() == "PyPy", + "Skip test if run with PyPy") class SameCommonTest(unittest.TestCase, CommonTestMixin): def setUp(self): @@ -228,6 +233,8 @@ class SameCommonTest(unittest.TestCase, CommonTestMixin): shutil.rmtree(self.directory) +@unittest.skipIf(platform.python_implementation() == "PyPy", + "Skip test if run with PyPy") class SplitCommonTest(unittest.TestCase, CommonTestMixin): def setUp(self): @@ -277,6 +284,8 @@ class SplitCommonTest(unittest.TestCase, CommonTestMixin): shutil.rmtree(self.directory) +@unittest.skipIf(platform.python_implementation() == "PyPy", + "Skip test if run with PyPy") class SplitSeparateTest(unittest.TestCase, SeparateTestMixin): def setUp(self): From d194f59939de8e9956e0d040266e3da269879a01 Mon Sep 17 00:00:00 2001 From: Siddharth Shukla Date: Sat, 11 Mar 2017 19:12:43 +0100 Subject: [PATCH 119/188] Make testing scripts python3.x compatible Update run_tests/*.py to use six based isomorphisms and print function from __future__ module. --- .../run_tests/performance/bq_upload_result.py | 8 ++- .../python_utils/filter_pull_request_tests.py | 5 +- tools/run_tests/python_utils/report_utils.py | 3 +- tools/run_tests/run_build_statistics.py | 4 +- tools/run_tests/run_interop_tests.py | 11 ++-- tools/run_tests/run_performance_tests.py | 9 +-- tools/run_tests/run_stress_tests.py | 14 ++--- tools/run_tests/run_tests.py | 3 +- tools/run_tests/run_tests_matrix.py | 2 + .../sanity/check_sources_and_headers.py | 2 + .../run_tests/sanity/check_test_filtering.py | 2 +- tools/run_tests/sanity/check_version.py | 16 +++-- .../run_tests/sanity/core_banned_functions.py | 4 +- tools/run_tests/start_port_server.py | 4 +- tools/run_tests/stress_test/run_on_gke.py | 58 ++++++++++--------- 15 files changed, 87 insertions(+), 58 deletions(-) diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index 89d2a9b320f..e2e1393509e 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -30,6 +30,8 @@ # Uploads performance benchmark result file to bigquery. +from __future__ import print_function + import argparse import calendar import json @@ -70,7 +72,7 @@ def _upload_netperf_latency_csv_to_bigquery(dataset_id, table_id, result_file): _create_results_table(bq, dataset_id, table_id) if not _insert_result(bq, dataset_id, table_id, scenario_result, flatten=False): - print 'Error uploading result to bigquery.' + print('Error uploading result to bigquery.') sys.exit(1) @@ -82,7 +84,7 @@ def _upload_scenario_result_to_bigquery(dataset_id, table_id, result_file): _create_results_table(bq, dataset_id, table_id) if not _insert_result(bq, dataset_id, table_id, scenario_result): - print 'Error uploading result to bigquery.' + print('Error uploading result to bigquery.') sys.exit(1) @@ -179,4 +181,4 @@ if args.file_format == 'netperf_latency_csv': _upload_netperf_latency_csv_to_bigquery(dataset_id, table_id, args.file_to_upload) else: _upload_scenario_result_to_bigquery(dataset_id, table_id, args.file_to_upload) -print 'Successfully uploaded %s to BigQuery.\n' % args.file_to_upload +print('Successfully uploaded %s to BigQuery.\n' % args.file_to_upload) diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index 3734f025d5f..041e3491627 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -30,7 +30,10 @@ """Filter out tests based on file differences compared to merge target branch""" +from __future__ import print_function + import re +import six from subprocess import check_output @@ -125,7 +128,7 @@ _WHITELIST_DICT = { } # Add all triggers to their respective test suites -for trigger, test_suites in _WHITELIST_DICT.iteritems(): +for trigger, test_suites in six.iteritems(_WHITELIST_DICT): for test_suite in test_suites: test_suite.add_trigger(trigger) diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py index 9dad60408f2..131772f55fc 100644 --- a/tools/run_tests/python_utils/report_utils.py +++ b/tools/run_tests/python_utils/report_utils.py @@ -40,6 +40,7 @@ except (ImportError): import os import string import xml.etree.cElementTree as ET +import six def _filter_msg(msg, output_format): @@ -63,7 +64,7 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc', root = ET.Element('testsuites') testsuite = ET.SubElement(root, 'testsuite', id='1', package=suite_package, name=suite_name) - for shortname, results in resultset.iteritems(): + for shortname, results in six.iteritems(resultset): for result in results: xml_test = ET.SubElement(testsuite, 'testcase', name=shortname) if result.elapsed_time: diff --git a/tools/run_tests/run_build_statistics.py b/tools/run_tests/run_build_statistics.py index 654cf95a38c..e33e01e7034 100755 --- a/tools/run_tests/run_build_statistics.py +++ b/tools/run_tests/run_build_statistics.py @@ -30,6 +30,8 @@ """Tool to get build statistics from Jenkins and upload to BigQuery.""" +from __future__ import print_function + import argparse import jenkinsapi from jenkinsapi.custom_exceptions import JenkinsAPIException @@ -243,6 +245,6 @@ for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds: rows = [big_query_utils.make_row(build_number, build_result)] if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, build_name, rows): - print '====> Error uploading result to bigquery.' + print('====> Error uploading result to bigquery.') sys.exit(1) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 161fa81956b..ace142ed8ad 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -44,6 +44,7 @@ import sys import tempfile import time import uuid +import six import python_utils.dockerjob as dockerjob import python_utils.jobset as jobset @@ -885,7 +886,7 @@ if not args.use_docker and servers: languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( - _LANGUAGES.iterkeys() if x == 'all' else [x] + six.iterkeys(_LANGUAGES) if x == 'all' else [x] for x in args.language)) languages_http2_badserver_interop = set() @@ -925,7 +926,7 @@ if args.use_docker: else: jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True) - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) @@ -1053,7 +1054,7 @@ try: if not jobs: print('No jobs to run.') - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) @@ -1093,9 +1094,9 @@ finally: if not job.is_running(): print('Server "%s" has exited prematurely.' % server) - dockerjob.finish_jobs([j for j in server_jobs.itervalues()]) + dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): if not args.manual_run: print('Removing docker image %s' % image) dockerjob.remove_image(image) diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index ee4102c5913..c79f47a3175 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -46,6 +46,7 @@ import tempfile import time import traceback import uuid +import six import performance.scenario_config as scenario_config import python_utils.jobset as jobset @@ -502,8 +503,8 @@ args = argp.parse_args() languages = set(scenario_config.LANGUAGES[l] for l in itertools.chain.from_iterable( - scenario_config.LANGUAGES.iterkeys() if x == 'all' else [x] - for x in args.language)) + six.iterkeys(scenario_config.LANGUAGES) if x == 'all' + else [x] for x in args.language)) # Put together set of remote hosts where to run and build @@ -572,8 +573,8 @@ for scenario in scenarios: jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)) scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures - merged_resultset = dict(itertools.chain(merged_resultset.iteritems(), - resultset.iteritems())) + merged_resultset = dict(itertools.chain(six.iteritems(merged_resultset), + six.iteritems(resultset))) finally: # Consider qps workers that need to be killed as failures qps_workers_killed += finish_qps_workers(scenario.workers) diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py index a94a615b886..3a2661a9aa6 100755 --- a/tools/run_tests/run_stress_tests.py +++ b/tools/run_tests/run_stress_tests.py @@ -43,6 +43,7 @@ import sys import tempfile import time import uuid +import six import python_utils.dockerjob as dockerjob import python_utils.jobset as jobset @@ -239,9 +240,8 @@ servers = set( for s in itertools.chain.from_iterable(_SERVERS if x == 'all' else [x] for x in args.server)) -languages = set(_LANGUAGES[l] - for l in itertools.chain.from_iterable(_LANGUAGES.iterkeys( - ) if x == 'all' else [x] for x in args.language)) +languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( + six.iterkeys(_LANGUAGES) if x == 'all' else [x] for x in args.language)) docker_images = {} # languages for which to build docker images @@ -267,7 +267,7 @@ if build_jobs: jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True) - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) @@ -306,7 +306,7 @@ try: if not jobs: print('No jobs to run.') - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) @@ -324,8 +324,8 @@ finally: if not job.is_running(): print('Server "%s" has exited prematurely.' % server) - dockerjob.finish_jobs([j for j in server_jobs.itervalues()]) + dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): print('Removing docker image %s' % image) dockerjob.remove_image(image) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 43b8f8184e2..cba91a9a632 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -54,6 +54,7 @@ import traceback import time from six.moves import urllib import uuid +import six import python_utils.jobset as jobset import python_utils.report_utils as report_utils @@ -771,7 +772,7 @@ class CSharpLanguage(object): runtime_cmd = ['mono'] specs = [] - for assembly in tests_by_assembly.iterkeys(): + for assembly in six.iterkeys(tests_by_assembly): assembly_file = 'src/csharp/%s/%s/%s%s' % (assembly, assembly_subdir, assembly, diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 47422451f8d..191228e4db7 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -30,6 +30,8 @@ """Run test matrix.""" +from __future__ import print_function + import argparse import multiprocessing import os diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py index a86db02b80b..0bb92fda8b5 100755 --- a/tools/run_tests/sanity/check_sources_and_headers.py +++ b/tools/run_tests/sanity/check_sources_and_headers.py @@ -28,6 +28,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function + import json import os import re diff --git a/tools/run_tests/sanity/check_test_filtering.py b/tools/run_tests/sanity/check_test_filtering.py index 290a6e2ddf6..aceee6d4d72 100755 --- a/tools/run_tests/sanity/check_test_filtering.py +++ b/tools/run_tests/sanity/check_test_filtering.py @@ -62,7 +62,7 @@ class TestFilteringTest(unittest.TestCase): def _get_changed_files(foo): return changed_files filter_pull_request_tests._get_changed_files = _get_changed_files - print + print() filtered_jobs = filter_pull_request_tests.filter_tests(all_jobs, "test") # Make sure sanity tests aren't being filtered out diff --git a/tools/run_tests/sanity/check_version.py b/tools/run_tests/sanity/check_version.py index e62f3908185..d8162fb3c57 100755 --- a/tools/run_tests/sanity/check_version.py +++ b/tools/run_tests/sanity/check_version.py @@ -29,6 +29,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function + import sys import yaml import os @@ -48,13 +50,13 @@ try: 'git rev-parse --abbrev-ref HEAD', shell=True) except: - print 'WARNING: not a git repository' + print('WARNING: not a git repository') branch_name = None if branch_name is not None: m = re.match(r'^release-([0-9]+)_([0-9]+)$', branch_name) if m: - print 'RELEASE branch' + print('RELEASE branch') # version number should align with the branched version check_version = lambda version: ( version.major == int(m.group(1)) and @@ -78,7 +80,7 @@ settings = build_yaml['settings'] top_version = Version(settings['version']) if not check_version(top_version): errors += 1 - print warning % ('version', top_version) + print(warning % ('version', top_version)) for tag, value in settings.iteritems(): if re.match(r'^[a-z]+_version$', tag): @@ -86,12 +88,14 @@ for tag, value in settings.iteritems(): if tag != 'core_version': if value.major != top_version.major: errors += 1 - print 'major version mismatch on %s: %d vs %d' % (tag, value.major, top_version.major) + print('major version mismatch on %s: %d vs %d' % (tag, value.major, + top_version.major)) if value.minor != top_version.minor: errors += 1 - print 'minor version mismatch on %s: %d vs %d' % (tag, value.minor, top_version.minor) + print('minor version mismatch on %s: %d vs %d' % (tag, value.minor, + top_version.minor)) if not check_version(value): errors += 1 - print warning % (tag, value) + print(warning % (tag, value)) sys.exit(errors) diff --git a/tools/run_tests/sanity/core_banned_functions.py b/tools/run_tests/sanity/core_banned_functions.py index 6d9be2e5d4a..c1069ccfb84 100755 --- a/tools/run_tests/sanity/core_banned_functions.py +++ b/tools/run_tests/sanity/core_banned_functions.py @@ -29,6 +29,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function + import os import sys @@ -60,7 +62,7 @@ for root, dirs, files in os.walk('src/core'): for banned, exceptions in BANNED_EXCEPT.items(): if path in exceptions: continue if banned in text: - print 'Illegal use of "%s" in %s' % (banned, path) + print('Illegal use of "%s" in %s' % (banned, path)) errors += 1 assert errors == 0 diff --git a/tools/run_tests/start_port_server.py b/tools/run_tests/start_port_server.py index e33ac12bd3b..bfd72222b66 100755 --- a/tools/run_tests/start_port_server.py +++ b/tools/run_tests/start_port_server.py @@ -39,8 +39,10 @@ The path to this file is called out in test/core/util/port.c, and printed as an error message to users. """ +from __future__ import print_function + import python_utils.start_port_server as start_port_server start_port_server.start_port_server() -print "Port server started successfully" +print("Port server started successfully") diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py index e2be76e245a..a2a5f42a2c6 100755 --- a/tools/run_tests/stress_test/run_on_gke.py +++ b/tools/run_tests/stress_test/run_on_gke.py @@ -27,6 +27,9 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import print_function + import argparse import datetime import json @@ -124,23 +127,24 @@ class DockerImage: return 'gcr.io/%s/%s' % (project_id, image_name) def build_image(self): - print 'Building docker image: %s (tag: %s)' % (self.image_name, - self.tag_name) + print('Building docker image: %s (tag: %s)' % (self.image_name, + self.tag_name)) os.environ['INTEROP_IMAGE'] = self.image_name os.environ['INTEROP_IMAGE_REPOSITORY_TAG'] = self.tag_name os.environ['BASE_NAME'] = self.dockerfile_dir os.environ['BUILD_TYPE'] = self.build_type - print 'DEBUG: path: ', self.build_script_path + print('DEBUG: path: ', self.build_script_path) if subprocess.call(args=[self.build_script_path]) != 0: - print 'Error in building the Docker image' + print('Error in building the Docker image') return False return True def push_to_gke_registry(self): cmd = ['gcloud', 'docker', 'push', self.tag_name] - print 'Pushing %s to the GKE registry..' % self.tag_name + print('Pushing %s to the GKE registry..' % self.tag_name) if subprocess.call(args=cmd) != 0: - print 'Error in pushing the image %s to the GKE registry' % self.tag_name + print('Error in pushing the image %s to the GKE registry' % + self.tag_name) return False return True @@ -199,11 +203,11 @@ class Gke: cmd = ['kubectl', 'proxy', '--port=%d' % port] self.p = subprocess.Popen(args=cmd) time.sleep(2) - print '\nStarted kubernetes proxy on port: %d' % port + print('\nStarted kubernetes proxy on port: %d' % port) def __del__(self): if self.p is not None: - print 'Shutting down Kubernetes proxy..' + print('Shutting down Kubernetes proxy..') self.p.kill() def __init__(self, project_id, run_id, dataset_id, summary_table_id, @@ -253,7 +257,7 @@ class Gke: for pod_name in server_pod_spec.pod_names(): server_env['POD_NAME'] = pod_name - print 'Creating server: %s' % pod_name + print('Creating server: %s' % pod_name) is_success = kubernetes_api.create_pod_and_service( 'localhost', self.kubernetes_port, @@ -267,11 +271,11 @@ class Gke: True # Headless = True for server to that GKE creates a DNS record for pod_name ) if not is_success: - print 'Error in launching server: %s' % pod_name + print('Error in launching server: %s' % pod_name) break if is_success: - print 'Successfully created server(s)' + print('Successfully created server(s)') return is_success @@ -301,7 +305,7 @@ class Gke: for pod_name in client_pod_spec.pod_names(): client_env['POD_NAME'] = pod_name - print 'Creating client: %s' % pod_name + print('Creating client: %s' % pod_name) is_success = kubernetes_api.create_pod_and_service( 'localhost', self.kubernetes_port, @@ -316,18 +320,18 @@ class Gke: ) if not is_success: - print 'Error in launching client %s' % pod_name + print('Error in launching client %s' % pod_name) break if is_success: - print 'Successfully created all client(s)' + print('Successfully created all client(s)') return is_success def _delete_pods(self, pod_name_list): is_success = True for pod_name in pod_name_list: - print 'Deleting %s' % pod_name + print('Deleting %s' % pod_name) is_success = kubernetes_api.delete_pod_and_service( 'localhost', self.kubernetes_port, @@ -335,11 +339,11 @@ class Gke: pod_name) if not is_success: - print 'Error in deleting pod %s' % pod_name + print('Error in deleting pod %s' % pod_name) break if is_success: - print 'Successfully deleted all pods' + print('Successfully deleted all pods') return is_success @@ -353,7 +357,7 @@ class Gke: class Config: def __init__(self, config_filename, gcp_project_id): - print 'Loading configuration...' + print('Loading configuration...') config_dict = self._load_config(config_filename) self.global_settings = self._parse_global_settings(config_dict, @@ -367,7 +371,7 @@ class Config: self.client_pod_specs_dict = self._parse_client_pod_specs( config_dict, self.docker_images_dict, self.client_templates_dict, self.server_pod_specs_dict) - print 'Loaded Configuaration.' + print('Loaded Configuaration.') def _parse_global_settings(self, config_dict, gcp_project_id): global_settings_dict = config_dict['globalSettings'] @@ -540,8 +544,8 @@ def run_tests(config): # run id. This is useful in debugging when looking at records in Biq query) run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') dataset_id = '%s_%s' % (config.global_settings.dataset_id_prefix, run_id) - print 'Run id:', run_id - print 'Dataset id:', dataset_id + print('Run id:', run_id) + print('Dataset id:', dataset_id) bq_helper = BigQueryHelper(run_id, '', '', config.global_settings.gcp_project_id, dataset_id, @@ -557,7 +561,7 @@ def run_tests(config): is_success = True try: - print 'Launching servers..' + print('Launching servers..') for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): if not gke.launch_servers(server_pod_spec): is_success = False # is_success is checked in the 'finally' block @@ -579,11 +583,12 @@ def run_tests(config): start_time = datetime.datetime.now() end_time = start_time + datetime.timedelta( seconds=config.global_settings.test_duration_secs) - print 'Running the test until %s' % end_time.isoformat() + print('Running the test until %s' % end_time.isoformat()) while True: if datetime.datetime.now() > end_time: - print 'Test was run for %d seconds' % config.global_settings.test_duration_secs + print('Test was run for %d seconds' % + config.global_settings.test_duration_secs) break # Check if either stress server or clients have failed (btw, the bq_helper @@ -591,11 +596,12 @@ def run_tests(config): # have a failure status) if bq_helper.check_if_any_tests_failed(): is_success = False - print 'Some tests failed.' + print('Some tests failed.') break # Don't 'return' here. We still want to call bq_helper to print qps/summary tables # Tests running fine. Wait until next poll time to check the status - print 'Sleeping for %d seconds..' % config.global_settings.test_poll_interval_secs + print('Sleeping for %d seconds..' % + config.global_settings.test_poll_interval_secs) time.sleep(config.global_settings.test_poll_interval_secs) # Print BiqQuery tables From 21e7dfc71269e9dcb4b9b5423e3c99c2bb0d16d7 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Sun, 12 Mar 2017 18:24:09 +0000 Subject: [PATCH 120/188] Ignore the number of failed skipped tests jobset.run was changed on 16 October 2015 in e4c3561074ca27fc4bdf1a6bf to return the number of failures and the result set. On 20 October 2016 in 5c691c634de144b50fc1f2e166303219721c19d0 jobset.run was changed to allow jobs to be skipped, but the introduced code path only returned the result set, giving jobset.run inconsistent return value semantics. On 24 October 2016 7e9bd6ca9a2e167dcb6133c78c05cf27584ee062 introduced a use of jobset.run's "skip_jobs" return value semantics, and on 8 March 2017 d159af31fcf300cfe2d7b9b41d789cd664ee4f1b resolved jobset.run's inconsistent return value semantics but failed to correct run_tests_matrix's use of the inconsistent "skip_jobs" return value semantics. --- tools/run_tests/run_tests_matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 47422451f8d..96075269204 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -415,8 +415,8 @@ if __name__ == "__main__": maxjobs=args.jobs) # Merge skipped tests into results to show skipped tests on report.xml if skipped_jobs: - skipped_results = jobset.run(skipped_jobs, - skip_jobs=True) + ignored_num_skipped_failures, skipped_results = jobset.run( + skipped_jobs, skip_jobs=True) resultset.update(skipped_results) report_utils.render_junit_xml_report(resultset, 'report_%s' % _REPORT_SUFFIX, suite_name='aggregate_tests') From 8e64d901fd7efd39868a889159329f41f74e3f9e Mon Sep 17 00:00:00 2001 From: Siddharth Shukla Date: Sun, 12 Mar 2017 19:50:18 +0100 Subject: [PATCH 121/188] Modify shebang on python scripts Change shebangs to `#!/usr/bin/env python` instead of `#!/usr/bin/env python2.7`. --- tools/run_tests/artifacts/artifact_targets.py | 2 +- tools/run_tests/artifacts/distribtest_targets.py | 2 +- tools/run_tests/artifacts/package_targets.py | 2 +- tools/run_tests/performance/bq_upload_result.py | 2 +- tools/run_tests/python_utils/antagonist.py | 2 +- tools/run_tests/python_utils/filter_pull_request_tests.py | 2 +- tools/run_tests/python_utils/port_server.py | 2 +- tools/run_tests/run_build_statistics.py | 2 +- tools/run_tests/run_interop_tests.py | 2 +- tools/run_tests/run_microbenchmark.py | 2 +- tools/run_tests/run_performance_tests.py | 2 +- tools/run_tests/run_stress_tests.py | 2 +- tools/run_tests/run_tests_matrix.py | 2 +- tools/run_tests/sanity/check_sources_and_headers.py | 2 +- tools/run_tests/sanity/check_test_filtering.py | 2 +- tools/run_tests/sanity/check_version.py | 2 +- tools/run_tests/sanity/core_banned_functions.py | 2 +- tools/run_tests/stress_test/print_summary.py | 2 +- tools/run_tests/stress_test/run_on_gke.py | 2 +- tools/run_tests/task_runner.py | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index e0658f4678d..04702baccae 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index a7535b38521..90bbde83cf4 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index d490f571c37..2547f2073cf 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index e2e1393509e..3bac1199a7b 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/python_utils/antagonist.py b/tools/run_tests/python_utils/antagonist.py index 857addfb386..111839ccf93 100755 --- a/tools/run_tests/python_utils/antagonist.py +++ b/tools/run_tests/python_utils/antagonist.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index 041e3491627..e0133762959 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/python_utils/port_server.py b/tools/run_tests/python_utils/port_server.py index e9b3f7ff795..dbd32efc0e8 100755 --- a/tools/run_tests/python_utils/port_server.py +++ b/tools/run_tests/python_utils/port_server.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/run_build_statistics.py b/tools/run_tests/run_build_statistics.py index e33e01e7034..dd11a45e789 100755 --- a/tools/run_tests/run_build_statistics.py +++ b/tools/run_tests/run_build_statistics.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index ace142ed8ad..29d5a9aaa00 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 94711109680..57b2636e569 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2017, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index c79f47a3175..35d20be5b75 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py index 3a2661a9aa6..4eea02118e5 100755 --- a/tools/run_tests/run_stress_tests.py +++ b/tools/run_tests/run_stress_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 191228e4db7..b9a9bd52f64 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py index 0bb92fda8b5..f2e0bfeb3d4 100755 --- a/tools/run_tests/sanity/check_sources_and_headers.py +++ b/tools/run_tests/sanity/check_sources_and_headers.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/sanity/check_test_filtering.py b/tools/run_tests/sanity/check_test_filtering.py index aceee6d4d72..ba03f112092 100755 --- a/tools/run_tests/sanity/check_test_filtering.py +++ b/tools/run_tests/sanity/check_test_filtering.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. diff --git a/tools/run_tests/sanity/check_version.py b/tools/run_tests/sanity/check_version.py index d8162fb3c57..d247260dbbb 100755 --- a/tools/run_tests/sanity/check_version.py +++ b/tools/run_tests/sanity/check_version.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. diff --git a/tools/run_tests/sanity/core_banned_functions.py b/tools/run_tests/sanity/core_banned_functions.py index c1069ccfb84..c3c3cbec767 100755 --- a/tools/run_tests/sanity/core_banned_functions.py +++ b/tools/run_tests/sanity/core_banned_functions.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. diff --git a/tools/run_tests/stress_test/print_summary.py b/tools/run_tests/stress_test/print_summary.py index cb1a33961e4..6f4ada2f4fd 100755 --- a/tools/run_tests/stress_test/print_summary.py +++ b/tools/run_tests/stress_test/print_summary.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py index a2a5f42a2c6..b190ebde7a4 100755 --- a/tools/run_tests/stress_test/run_on_gke.py +++ b/tools/run_tests/stress_test/run_on_gke.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015-2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py index fdc46682223..0ec7efbbee6 100755 --- a/tools/run_tests/task_runner.py +++ b/tools/run_tests/task_runner.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # From e30f41d70e2defe9148a70dcbf2c58f438f5c76d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sun, 12 Mar 2017 14:45:41 -0700 Subject: [PATCH 122/188] Fix sanity --- tools/doxygen/Doxyfile.c++ | 1 + tools/doxygen/Doxyfile.c++.internal | 1 + tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 1 + 4 files changed, 4 insertions(+) diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 60a1261235f..b5021ede157 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -788,6 +788,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 180d568809b..758ba2402ae 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -788,6 +788,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 478bdb2d042..cd3f2af44c8 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -787,6 +787,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index a7d6aaeb9ef..131a0134519 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -787,6 +787,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ From 9de5ffebbdee52b594bacf4f8f53660bbaf37aeb Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Sun, 12 Mar 2017 18:32:18 -0700 Subject: [PATCH 123/188] Run generate_projects.sh --- tools/doxygen/Doxyfile.c++ | 1 + tools/doxygen/Doxyfile.c++.internal | 1 + tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 1 + 4 files changed, 4 insertions(+) diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 60a1261235f..b5021ede157 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -788,6 +788,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 180d568809b..758ba2402ae 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -788,6 +788,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 478bdb2d042..cd3f2af44c8 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -787,6 +787,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index a7d6aaeb9ef..131a0134519 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -787,6 +787,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ From e6e7fd0c9963b10f46484cf1bc426667269ff94b Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Sun, 12 Mar 2017 18:35:34 -0700 Subject: [PATCH 124/188] Run generate_projects.sh --- tools/doxygen/Doxyfile.c++ | 1 + tools/doxygen/Doxyfile.c++.internal | 1 + tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 1 + 4 files changed, 4 insertions(+) diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 60a1261235f..b5021ede157 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -788,6 +788,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 180d568809b..758ba2402ae 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -788,6 +788,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 478bdb2d042..cd3f2af44c8 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -787,6 +787,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index a7d6aaeb9ef..131a0134519 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -787,6 +787,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/status_ordering.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ doc/wait-for-ready.md \ From 9202b3fdfdaf890752d7472692296490ec55c750 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sun, 12 Mar 2017 22:30:38 -0700 Subject: [PATCH 125/188] Arena allocator for grpc --- CMakeLists.txt | 69 +++++++ Makefile | 85 ++++++++ binding.gyp | 1 + build.yaml | 30 +++ config.m4 | 1 + gRPC-Core.podspec | 3 + grpc.gemspec | 2 + package.xml | 2 + src/core/lib/support/arena.c | 90 ++++++++ src/core/lib/support/arena.h | 54 +++++ src/python/grpcio/grpc_core_dependencies.py | 1 + test/core/support/arena_test.c | 91 +++++++++ test/cpp/microbenchmarks/bm_arena.cc | 69 +++++++ tools/doxygen/Doxyfile.core.internal | 2 + .../generated/sources_and_headers.json | 39 ++++ tools/run_tests/generated/tests.json | 44 ++++ vsprojects/buildtests_c.sln | 25 +++ vsprojects/vcxproj/gpr/gpr.vcxproj | 3 + vsprojects/vcxproj/gpr/gpr.vcxproj.filters | 6 + .../test/arena_test/arena_test.vcxproj | 193 ++++++++++++++++++ .../arena_test/arena_test.vcxproj.filters | 21 ++ 21 files changed, 831 insertions(+) create mode 100644 src/core/lib/support/arena.c create mode 100644 src/core/lib/support/arena.h create mode 100644 test/core/support/arena_test.c create mode 100644 test/cpp/microbenchmarks/bm_arena.cc create mode 100644 vsprojects/vcxproj/test/arena_test/arena_test.vcxproj create mode 100644 vsprojects/vcxproj/test/arena_test/arena_test.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index d0a65b44932..faa7d528e8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -332,6 +332,7 @@ add_dependencies(buildtests_c alarm_test) add_dependencies(buildtests_c algorithm_test) add_dependencies(buildtests_c alloc_test) add_dependencies(buildtests_c alpn_test) +add_dependencies(buildtests_c arena_test) add_dependencies(buildtests_c bad_server_response_test) add_dependencies(buildtests_c bdp_estimator_test) add_dependencies(buildtests_c bin_decoder_test) @@ -574,6 +575,9 @@ add_dependencies(buildtests_cxx alarm_cpp_test) add_dependencies(buildtests_cxx async_end2end_test) add_dependencies(buildtests_cxx auth_property_iterator_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_arena) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_call_create) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -690,6 +694,7 @@ add_library(gpr src/core/lib/profiling/basic_timers.c src/core/lib/profiling/stap_timers.c src/core/lib/support/alloc.c + src/core/lib/support/arena.c src/core/lib/support/avl.c src/core/lib/support/backoff.c src/core/lib/support/cmdline.c @@ -4112,6 +4117,31 @@ target_link_libraries(alpn_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +add_executable(arena_test + test/core/support/arena_test.c +) + + +target_include_directories(arena_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include +) + +target_link_libraries(arena_test + ${_gRPC_ALLTARGETS_LIBRARIES} + gpr_test_util + gpr +) + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_executable(bad_server_response_test test/core/end2end/bad_server_response_test.c ) @@ -7626,6 +7656,45 @@ endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_executable(bm_arena + test/cpp/microbenchmarks/bm_arena.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_arena + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_arena + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark + benchmark + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + add_executable(bm_call_create test/cpp/microbenchmarks/bm_call_create.cc third_party/googletest/src/gtest-all.cc diff --git a/Makefile b/Makefile index a9242dddbdd..c6203768adc 100644 --- a/Makefile +++ b/Makefile @@ -906,6 +906,7 @@ algorithm_test: $(BINDIR)/$(CONFIG)/algorithm_test alloc_test: $(BINDIR)/$(CONFIG)/alloc_test alpn_test: $(BINDIR)/$(CONFIG)/alpn_test api_fuzzer: $(BINDIR)/$(CONFIG)/api_fuzzer +arena_test: $(BINDIR)/$(CONFIG)/arena_test bad_server_response_test: $(BINDIR)/$(CONFIG)/bad_server_response_test bdp_estimator_test: $(BINDIR)/$(CONFIG)/bdp_estimator_test bin_decoder_test: $(BINDIR)/$(CONFIG)/bin_decoder_test @@ -1047,6 +1048,7 @@ wakeup_fd_cv_test: $(BINDIR)/$(CONFIG)/wakeup_fd_cv_test alarm_cpp_test: $(BINDIR)/$(CONFIG)/alarm_cpp_test async_end2end_test: $(BINDIR)/$(CONFIG)/async_end2end_test auth_property_iterator_test: $(BINDIR)/$(CONFIG)/auth_property_iterator_test +bm_arena: $(BINDIR)/$(CONFIG)/bm_arena bm_call_create: $(BINDIR)/$(CONFIG)/bm_call_create bm_chttp2_hpack: $(BINDIR)/$(CONFIG)/bm_chttp2_hpack bm_closure: $(BINDIR)/$(CONFIG)/bm_closure @@ -1285,6 +1287,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/algorithm_test \ $(BINDIR)/$(CONFIG)/alloc_test \ $(BINDIR)/$(CONFIG)/alpn_test \ + $(BINDIR)/$(CONFIG)/arena_test \ $(BINDIR)/$(CONFIG)/bad_server_response_test \ $(BINDIR)/$(CONFIG)/bdp_estimator_test \ $(BINDIR)/$(CONFIG)/bin_decoder_test \ @@ -1468,6 +1471,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/alarm_cpp_test \ $(BINDIR)/$(CONFIG)/async_end2end_test \ $(BINDIR)/$(CONFIG)/auth_property_iterator_test \ + $(BINDIR)/$(CONFIG)/bm_arena \ $(BINDIR)/$(CONFIG)/bm_call_create \ $(BINDIR)/$(CONFIG)/bm_chttp2_hpack \ $(BINDIR)/$(CONFIG)/bm_closure \ @@ -1583,6 +1587,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/alarm_cpp_test \ $(BINDIR)/$(CONFIG)/async_end2end_test \ $(BINDIR)/$(CONFIG)/auth_property_iterator_test \ + $(BINDIR)/$(CONFIG)/bm_arena \ $(BINDIR)/$(CONFIG)/bm_call_create \ $(BINDIR)/$(CONFIG)/bm_chttp2_hpack \ $(BINDIR)/$(CONFIG)/bm_closure \ @@ -1663,6 +1668,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/alloc_test || ( echo test alloc_test failed ; exit 1 ) $(E) "[RUN] Testing alpn_test" $(Q) $(BINDIR)/$(CONFIG)/alpn_test || ( echo test alpn_test failed ; exit 1 ) + $(E) "[RUN] Testing arena_test" + $(Q) $(BINDIR)/$(CONFIG)/arena_test || ( echo test arena_test failed ; exit 1 ) $(E) "[RUN] Testing bad_server_response_test" $(Q) $(BINDIR)/$(CONFIG)/bad_server_response_test || ( echo test bad_server_response_test failed ; exit 1 ) $(E) "[RUN] Testing bdp_estimator_test" @@ -1923,6 +1930,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/async_end2end_test || ( echo test async_end2end_test failed ; exit 1 ) $(E) "[RUN] Testing auth_property_iterator_test" $(Q) $(BINDIR)/$(CONFIG)/auth_property_iterator_test || ( echo test auth_property_iterator_test failed ; exit 1 ) + $(E) "[RUN] Testing bm_arena" + $(Q) $(BINDIR)/$(CONFIG)/bm_arena || ( echo test bm_arena failed ; exit 1 ) $(E) "[RUN] Testing bm_call_create" $(Q) $(BINDIR)/$(CONFIG)/bm_call_create || ( echo test bm_call_create failed ; exit 1 ) $(E) "[RUN] Testing bm_chttp2_hpack" @@ -2594,6 +2603,7 @@ LIBGPR_SRC = \ src/core/lib/profiling/basic_timers.c \ src/core/lib/profiling/stap_timers.c \ src/core/lib/support/alloc.c \ + src/core/lib/support/arena.c \ src/core/lib/support/avl.c \ src/core/lib/support/backoff.c \ src/core/lib/support/cmdline.c \ @@ -8094,6 +8104,38 @@ endif endif +ARENA_TEST_SRC = \ + test/core/support/arena_test.c \ + +ARENA_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(ARENA_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/arena_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/arena_test: $(ARENA_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(ARENA_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/arena_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/support/arena_test.o: $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_arena_test: $(ARENA_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(ARENA_TEST_OBJS:.o=.dep) +endif +endif + + BAD_SERVER_RESPONSE_TEST_SRC = \ test/core/end2end/bad_server_response_test.c \ @@ -12639,6 +12681,49 @@ endif endif +BM_ARENA_SRC = \ + test/cpp/microbenchmarks/bm_arena.cc \ + +BM_ARENA_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_ARENA_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_arena: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_arena: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_arena: $(PROTOBUF_DEP) $(BM_ARENA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_ARENA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_arena + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_arena.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_arena: $(BM_ARENA_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_ARENA_OBJS:.o=.dep) +endif +endif + + BM_CALL_CREATE_SRC = \ test/cpp/microbenchmarks/bm_call_create.cc \ diff --git a/binding.gyp b/binding.gyp index c521a27c300..59b6b600e90 100644 --- a/binding.gyp +++ b/binding.gyp @@ -544,6 +544,7 @@ 'src/core/lib/profiling/basic_timers.c', 'src/core/lib/profiling/stap_timers.c', 'src/core/lib/support/alloc.c', + 'src/core/lib/support/arena.c', 'src/core/lib/support/avl.c', 'src/core/lib/support/backoff.c', 'src/core/lib/support/cmdline.c', diff --git a/build.yaml b/build.yaml index a32a8a06d34..e799928c047 100644 --- a/build.yaml +++ b/build.yaml @@ -85,6 +85,7 @@ filegroups: - include/grpc/support/useful.h headers: - src/core/lib/profiling/timers.h + - src/core/lib/support/arena.h - src/core/lib/support/backoff.h - src/core/lib/support/block_annotate.h - src/core/lib/support/env.h @@ -101,6 +102,7 @@ filegroups: - src/core/lib/profiling/basic_timers.c - src/core/lib/profiling/stap_timers.c - src/core/lib/support/alloc.c + - src/core/lib/support/arena.c - src/core/lib/support/avl.c - src/core/lib/support/backoff.c - src/core/lib/support/cmdline.c @@ -1483,6 +1485,14 @@ targets: - test/core/end2end/fuzzers/api_fuzzer_corpus dict: test/core/end2end/fuzzers/api_fuzzer.dictionary maxlen: 2048 +- name: arena_test + build: test + language: c + src: + - test/core/support/arena_test.c + deps: + - gpr_test_util + - gpr - name: bad_server_response_test build: test language: c @@ -3052,6 +3062,26 @@ targets: - grpc - gpr_test_util - gpr +- name: bm_arena + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_arena.cc + deps: + - grpc_benchmark + - benchmark + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + platforms: + - mac + - linux + - posix - name: bm_call_create build: test language: c++ diff --git a/config.m4 b/config.m4 index 90536e503ed..f43a0316380 100644 --- a/config.m4 +++ b/config.m4 @@ -39,6 +39,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/profiling/basic_timers.c \ src/core/lib/profiling/stap_timers.c \ src/core/lib/support/alloc.c \ + src/core/lib/support/arena.c \ src/core/lib/support/avl.c \ src/core/lib/support/backoff.c \ src/core/lib/support/cmdline.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 027babcda46..8a5dcbb99cb 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -196,6 +196,7 @@ Pod::Spec.new do |s| # To save you from scrolling, this is the last part of the podspec. ss.source_files = 'src/core/lib/profiling/timers.h', + 'src/core/lib/support/arena.h', 'src/core/lib/support/backoff.h', 'src/core/lib/support/block_annotate.h', 'src/core/lib/support/env.h', @@ -211,6 +212,7 @@ Pod::Spec.new do |s| 'src/core/lib/profiling/basic_timers.c', 'src/core/lib/profiling/stap_timers.c', 'src/core/lib/support/alloc.c', + 'src/core/lib/support/arena.c', 'src/core/lib/support/avl.c', 'src/core/lib/support/backoff.c', 'src/core/lib/support/cmdline.c', @@ -675,6 +677,7 @@ Pod::Spec.new do |s| 'src/core/plugin_registry/grpc_plugin_registry.c' ss.private_header_files = 'src/core/lib/profiling/timers.h', + 'src/core/lib/support/arena.h', 'src/core/lib/support/backoff.h', 'src/core/lib/support/block_annotate.h', 'src/core/lib/support/env.h', diff --git a/grpc.gemspec b/grpc.gemspec index 8d5b7b2ab1c..825c23d7a87 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -82,6 +82,7 @@ Gem::Specification.new do |s| s.files += %w( include/grpc/impl/codegen/sync_posix.h ) s.files += %w( include/grpc/impl/codegen/sync_windows.h ) s.files += %w( src/core/lib/profiling/timers.h ) + s.files += %w( src/core/lib/support/arena.h ) s.files += %w( src/core/lib/support/backoff.h ) s.files += %w( src/core/lib/support/block_annotate.h ) s.files += %w( src/core/lib/support/env.h ) @@ -97,6 +98,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/profiling/basic_timers.c ) s.files += %w( src/core/lib/profiling/stap_timers.c ) s.files += %w( src/core/lib/support/alloc.c ) + s.files += %w( src/core/lib/support/arena.c ) s.files += %w( src/core/lib/support/avl.c ) s.files += %w( src/core/lib/support/backoff.c ) s.files += %w( src/core/lib/support/cmdline.c ) diff --git a/package.xml b/package.xml index 4167bef26e3..67669286c37 100644 --- a/package.xml +++ b/package.xml @@ -91,6 +91,7 @@ + @@ -106,6 +107,7 @@ + diff --git a/src/core/lib/support/arena.c b/src/core/lib/support/arena.c new file mode 100644 index 00000000000..a5b0be4d48e --- /dev/null +++ b/src/core/lib/support/arena.c @@ -0,0 +1,90 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/support/arena.h" +#include +#include +#include + +typedef struct zone { + size_t size_begin; + size_t size_end; + gpr_atm next_atm; +} zone; + +struct gpr_arena { + gpr_atm size_so_far; + zone initial_zone; +}; + +gpr_arena *gpr_arena_create(size_t initial_size) { + gpr_arena *a = gpr_zalloc(sizeof(gpr_arena) + initial_size); + a->initial_zone.size_end = initial_size; + return a; +} + +size_t gpr_arena_destroy(gpr_arena *arena) { + gpr_atm size = gpr_atm_no_barrier_load(&arena->size_so_far); + zone *z = (zone *)gpr_atm_no_barrier_load(&arena->initial_zone.next_atm); + gpr_free(arena); + while (z) { + zone *next_z = (zone *)gpr_atm_no_barrier_load(&z->next_atm); + gpr_free(z); + z = next_z; + } + return (size_t)size; +} + +void *gpr_arena_alloc(gpr_arena *arena, size_t size) { + size_t start = + (size_t)gpr_atm_no_barrier_fetch_add(&arena->size_so_far, size); + zone *z = &arena->initial_zone; + while (start > z->size_begin) { + zone *next_z = (zone *)gpr_atm_acq_load(&z->next_atm); + while (next_z == NULL) { + size_t next_z_size = GPR_MAX(2 * start, size); + next_z = gpr_zalloc(sizeof(zone) + next_z_size); + next_z->size_begin = z->size_end; + next_z->size_end = z->size_end + next_z_size; + if (!gpr_atm_rel_cas(&z->next_atm, (gpr_atm)NULL, (gpr_atm)next_z)) { + gpr_free(next_z); + next_z = NULL; + } + } + z = next_z; + } + if (start + size > z->size_end) { + return gpr_arena_alloc(arena, size); + } + return ((char *)(z + 1)) + start - z->size_begin; +} diff --git a/src/core/lib/support/arena.h b/src/core/lib/support/arena.h new file mode 100644 index 00000000000..c28033ffc35 --- /dev/null +++ b/src/core/lib/support/arena.h @@ -0,0 +1,54 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +// \file Arena based allocator +// Allows very fast allocation of memory, but that memory cannot be freed until +// the arena as a whole is freed +// Tracks the total memory allocated against it, so that future arenas can +// pre-allocate the right amount of memory + +#ifndef GRPC_CORE_LIB_SUPPORT_ARENA_H +#define GRPC_CORE_LIB_SUPPORT_ARENA_H + +#include + +typedef struct gpr_arena gpr_arena; + +// Create an arena, with \a initial_size bytes in the first allocated buffer +gpr_arena *gpr_arena_create(size_t initial_size); +// Allocate \a size bytes from the arena +void *gpr_arena_alloc(gpr_arena *arena, size_t size); +// Destroy an arena, returning the total number of bytes allocated +size_t gpr_arena_destroy(gpr_arena *arena); + +#endif /* GRPC_CORE_LIB_SUPPORT_ARENA_H */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index a9f20e6d2a8..52d3ffd2c43 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -33,6 +33,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/profiling/basic_timers.c', 'src/core/lib/profiling/stap_timers.c', 'src/core/lib/support/alloc.c', + 'src/core/lib/support/arena.c', 'src/core/lib/support/avl.c', 'src/core/lib/support/backoff.c', 'src/core/lib/support/cmdline.c', diff --git a/test/core/support/arena_test.c b/test/core/support/arena_test.c new file mode 100644 index 00000000000..bbf6d48db35 --- /dev/null +++ b/test/core/support/arena_test.c @@ -0,0 +1,91 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/support/arena.h" + +#include +#include +#include +#include +#include + +#include "src/core/lib/support/string.h" +#include "test/core/util/test_config.h" + +static void test_noop(void) { gpr_arena_destroy(gpr_arena_create(1)); } + +static void test(const char *name, size_t init_size, const size_t *allocs, + size_t nallocs) { + gpr_strvec v; + char *s; + gpr_strvec_init(&v); + gpr_asprintf(&s, "test '%s': %" PRIdPTR " <- {", name, init_size); + gpr_strvec_add(&v, s); + for (size_t i = 0; i < nallocs; i++) { + gpr_asprintf(&s, "%" PRIdPTR ",", allocs[i]); + gpr_strvec_add(&v, s); + } + gpr_strvec_add(&v, gpr_strdup("}")); + s = gpr_strvec_flatten(&v, NULL); + gpr_strvec_destroy(&v); + gpr_log(GPR_INFO, "%s", s); + gpr_free(s); + + gpr_arena *a = gpr_arena_create(init_size); + void **ps = gpr_zalloc(sizeof(*ps) * nallocs); + for (size_t i = 0; i < nallocs; i++) { + ps[i] = gpr_arena_alloc(a, allocs[i]); + for (size_t j = 0; j < i; j++) { + GPR_ASSERT(ps[i] != ps[j]); + } + } + gpr_arena_destroy(a); +} + +#define TEST(name, init_size, ...) \ + static const size_t allocs_##name[] = {__VA_ARGS__}; \ + test(#name, init_size, allocs_##name, GPR_ARRAY_SIZE(allocs_##name)) + +int main(int argc, char *argv[]) { + grpc_test_init(argc, argv); + + test_noop(); + TEST(0_1, 0, 1); + TEST(1_1, 1, 1); + TEST(1_2, 1, 2); + TEST(1_3, 1, 3); + TEST(1_inc, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + TEST(6_123, 6, 1, 2, 3); + + return 0; +} diff --git a/test/cpp/microbenchmarks/bm_arena.cc b/test/cpp/microbenchmarks/bm_arena.cc new file mode 100644 index 00000000000..33e073f7495 --- /dev/null +++ b/test/cpp/microbenchmarks/bm_arena.cc @@ -0,0 +1,69 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Benchmark arenas */ + +extern "C" { +#include "src/core/lib/support/arena.h" +} +#include "test/cpp/microbenchmarks/helpers.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +static void BM_Arena_NoOp(benchmark::State& state) { + while (state.KeepRunning()) { + gpr_arena_destroy(gpr_arena_create(state.range(0))); + } +} +BENCHMARK(BM_Arena_NoOp)->Range(1, 1024 * 1024); + +static void BM_Arena_ManyAlloc(benchmark::State& state) { + gpr_arena* a = gpr_arena_create(state.range(0)); + while (state.KeepRunning()) { + gpr_arena_alloc(a, state.range(1)); + } + gpr_arena_destroy(a); +} +BENCHMARK(BM_Arena_ManyAlloc)->Ranges({{1, 1024 * 1024}, {1, 32 * 1024}}); + +static void BM_Arena_Batch(benchmark::State& state) { + while (state.KeepRunning()) { + gpr_arena* a = gpr_arena_create(state.range(0)); + for (int i = 0; i < state.range(1); i++) { + gpr_arena_alloc(a, state.range(2)); + } + gpr_arena_destroy(a); + } +} +BENCHMARK(BM_Arena_Batch)->Ranges({{1, 64 * 1024}, {1, 64}, {1, 1024}}); + +BENCHMARK_MAIN(); diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 131a0134519..e4f56f23747 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1228,6 +1228,8 @@ src/core/lib/slice/slice_internal.h \ src/core/lib/slice/slice_string_helpers.c \ src/core/lib/slice/slice_string_helpers.h \ src/core/lib/support/alloc.c \ +src/core/lib/support/arena.c \ +src/core/lib/support/arena.h \ src/core/lib/support/avl.c \ src/core/lib/support/backoff.c \ src/core/lib/support/backoff.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 767c2b2e368..1dc86068850 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -84,6 +84,21 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "arena_test", + "src": [ + "test/core/support/arena_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", @@ -2429,6 +2444,27 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_arena", + "src": [ + "test/cpp/microbenchmarks/bm_arena.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "benchmark", @@ -7243,6 +7279,7 @@ "include/grpc/support/tls_pthread.h", "include/grpc/support/useful.h", "src/core/lib/profiling/timers.h", + "src/core/lib/support/arena.h", "src/core/lib/support/backoff.h", "src/core/lib/support/block_annotate.h", "src/core/lib/support/env.h", @@ -7290,6 +7327,8 @@ "src/core/lib/profiling/stap_timers.c", "src/core/lib/profiling/timers.h", "src/core/lib/support/alloc.c", + "src/core/lib/support/arena.c", + "src/core/lib/support/arena.h", "src/core/lib/support/avl.c", "src/core/lib/support/backoff.c", "src/core/lib/support/backoff.h", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index f9e2a19d0ef..1eebc9a1d50 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -89,6 +89,28 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "arena_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ @@ -2605,6 +2627,28 @@ "windows" ] }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_arena", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "--benchmark_min_time=0" diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index daafd3f3504..c5bbaed60c4 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -45,6 +45,15 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "alpn_test", "vcxproj\test\a {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "arena_test", "vcxproj\test\arena_test\arena_test.vcxproj", "{D85AC722-A88F-4280-F62E-672F571787FF}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bad_client_test", "vcxproj\test/bad_client\bad_client_test\bad_client_test.vcxproj", "{BA67B418-B699-E41A-9CC4-0279C49481A5}" ProjectSection(myProperties) = preProject lib = "True" @@ -1708,6 +1717,22 @@ Global {5BAAE7EA-A972-DD80-F190-29B9E3110BB3}.Release-DLL|Win32.Build.0 = Release|Win32 {5BAAE7EA-A972-DD80-F190-29B9E3110BB3}.Release-DLL|x64.ActiveCfg = Release|x64 {5BAAE7EA-A972-DD80-F190-29B9E3110BB3}.Release-DLL|x64.Build.0 = Release|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug|Win32.ActiveCfg = Debug|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug|x64.ActiveCfg = Debug|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release|Win32.ActiveCfg = Release|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release|x64.ActiveCfg = Release|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug|Win32.Build.0 = Debug|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug|x64.Build.0 = Debug|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release|Win32.Build.0 = Release|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release|x64.Build.0 = Release|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug-DLL|x64.Build.0 = Debug|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release-DLL|Win32.Build.0 = Release|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release-DLL|x64.ActiveCfg = Release|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release-DLL|x64.Build.0 = Release|x64 {BA67B418-B699-E41A-9CC4-0279C49481A5}.Debug|Win32.ActiveCfg = Debug|Win32 {BA67B418-B699-E41A-9CC4-0279C49481A5}.Debug|x64.ActiveCfg = Debug|x64 {BA67B418-B699-E41A-9CC4-0279C49481A5}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/gpr/gpr.vcxproj b/vsprojects/vcxproj/gpr/gpr.vcxproj index 44c21ddeb31..1b37ace69ab 100644 --- a/vsprojects/vcxproj/gpr/gpr.vcxproj +++ b/vsprojects/vcxproj/gpr/gpr.vcxproj @@ -188,6 +188,7 @@ + @@ -208,6 +209,8 @@ + + diff --git a/vsprojects/vcxproj/gpr/gpr.vcxproj.filters b/vsprojects/vcxproj/gpr/gpr.vcxproj.filters index a5924a624a8..fafd54cdf93 100644 --- a/vsprojects/vcxproj/gpr/gpr.vcxproj.filters +++ b/vsprojects/vcxproj/gpr/gpr.vcxproj.filters @@ -10,6 +10,9 @@ src\core\lib\support + + src\core\lib\support + src\core\lib\support @@ -254,6 +257,9 @@ src\core\lib\profiling + + src\core\lib\support + src\core\lib\support diff --git a/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj b/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj new file mode 100644 index 00000000000..5ae2f8e483c --- /dev/null +++ b/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj @@ -0,0 +1,193 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {D85AC722-A88F-4280-F62E-672F571787FF} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + arena_test + static + Debug + static + Debug + + + arena_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj.filters b/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj.filters new file mode 100644 index 00000000000..c470f175272 --- /dev/null +++ b/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\core\support + + + + + + {130788b2-eacc-90df-a4f6-f5102a7d3370} + + + {5c3e1753-6fdb-9476-f98c-a3a394fac54a} + + + {1d3d2cc8-4e69-8b2e-6ceb-6569fcb19a86} + + + + From f134206509de8417f3f568e8b599290c1b955e1f Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Sun, 12 Mar 2017 22:39:11 -0700 Subject: [PATCH 126/188] Fix android build --- src/core/lib/iomgr/port.h | 4 ++++ src/core/lib/iomgr/tcp_server_posix.c | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h index f1897bb91f2..94a454c0b7c 100644 --- a/src/core/lib/iomgr/port.h +++ b/src/core/lib/iomgr/port.h @@ -39,6 +39,7 @@ #if defined(GRPC_UV) // Do nothing #elif defined(GPR_MANYLINUX1) +#define GRPC_HAVE_IFADDRS 1 #define GRPC_HAVE_IPV6_RECVPKTINFO 1 #define GRPC_HAVE_IP_PKTINFO 1 #define GRPC_HAVE_MSG_NOSIGNAL 1 @@ -65,6 +66,7 @@ #define GRPC_POSIX_WAKEUP_FD 1 #define GRPC_TIMER_USE_GENERIC 1 #elif defined(GPR_LINUX) +#define GRPC_HAVE_IFADDRS 1 #define GRPC_HAVE_IPV6_RECVPKTINFO 1 #define GRPC_HAVE_IP_PKTINFO 1 #define GRPC_HAVE_MSG_NOSIGNAL 1 @@ -90,6 +92,7 @@ #define GRPC_POSIX_SOCKETUTILS #endif #elif defined(GPR_APPLE) +#define GRPC_HAVE_IFADDRS 1 #define GRPC_HAVE_SO_NOSIGPIPE 1 #define GRPC_HAVE_UNIX_SOCKET 1 #define GRPC_MSG_IOVLEN_TYPE int @@ -100,6 +103,7 @@ #define GRPC_POSIX_WAKEUP_FD 1 #define GRPC_TIMER_USE_GENERIC 1 #elif defined(GPR_FREEBSD) +#define GRPC_HAVE_IFADDRS 1 #define GRPC_HAVE_IPV6_RECVPKTINFO 1 #define GRPC_HAVE_SO_NOSIGPIPE 1 #define GRPC_HAVE_UNIX_SOCKET 1 diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index 5f286a6723c..7cc44fabe7e 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -42,9 +42,12 @@ #include "src/core/lib/iomgr/tcp_server.h" +#ifdef GRPC_HAVE_IFADDRS +#include +#endif /* GRPC_HAVE_IFADDRS */ + #include #include -#include #include #include #include @@ -598,6 +601,7 @@ static grpc_error *add_all_local_addrs_to_server(grpc_tcp_server *s, unsigned port_index, int requested_port, int *out_port) { +#ifdef GRPC_HAVE_IFADDRS struct ifaddrs *ifa = NULL; struct ifaddrs *ifa_it; unsigned fd_index = 0; @@ -685,6 +689,9 @@ static grpc_error *add_all_local_addrs_to_server(grpc_tcp_server *s, *out_port = sp->port; return GRPC_ERROR_NONE; } +#else /* GRPC_HAVE_IFADDRS */ + return GRPC_ERROR_NONE; +#endif /* GRPC_HAVE_IFADDRS */ } /* Treat :: or 0.0.0.0 as a family-agnostic wildcard. */ From 0dd81003b5a9601d4bc79afdd269a7e93b53c5a9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Mar 2017 06:57:29 -0700 Subject: [PATCH 127/188] Concurrent test --- src/core/lib/support/arena.c | 5 ++++ test/core/support/arena_test.c | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/core/lib/support/arena.c b/src/core/lib/support/arena.c index a5b0be4d48e..faceb1a1eb3 100644 --- a/src/core/lib/support/arena.c +++ b/src/core/lib/support/arena.c @@ -36,6 +36,9 @@ #include #include +#define ROUND_UP_TO_ALIGNMENT_SIZE(x) \ + (((x) + GPR_MAX_ALIGNMENT - 1u) & ~(GPR_MAX_ALIGNMENT - 1u)) + typedef struct zone { size_t size_begin; size_t size_end; @@ -48,6 +51,7 @@ struct gpr_arena { }; gpr_arena *gpr_arena_create(size_t initial_size) { + initial_size = ROUND_UP_TO_ALIGNMENT_SIZE(initial_size); gpr_arena *a = gpr_zalloc(sizeof(gpr_arena) + initial_size); a->initial_zone.size_end = initial_size; return a; @@ -66,6 +70,7 @@ size_t gpr_arena_destroy(gpr_arena *arena) { } void *gpr_arena_alloc(gpr_arena *arena, size_t size) { + size = ROUND_UP_TO_ALIGNMENT_SIZE(size); size_t start = (size_t)gpr_atm_no_barrier_fetch_add(&arena->size_so_far, size); zone *z = &arena->initial_zone; diff --git a/test/core/support/arena_test.c b/test/core/support/arena_test.c index bbf6d48db35..3e21867bcff 100644 --- a/test/core/support/arena_test.c +++ b/test/core/support/arena_test.c @@ -36,8 +36,11 @@ #include #include #include +#include +#include #include #include +#include #include "src/core/lib/support/string.h" #include "test/core/util/test_config.h" @@ -65,9 +68,12 @@ static void test(const char *name, size_t init_size, const size_t *allocs, void **ps = gpr_zalloc(sizeof(*ps) * nallocs); for (size_t i = 0; i < nallocs; i++) { ps[i] = gpr_arena_alloc(a, allocs[i]); + // ensure no duplicate results for (size_t j = 0; j < i; j++) { GPR_ASSERT(ps[i] != ps[j]); } + // ensure writable + memset(ps[i], 1, allocs[i]); } gpr_arena_destroy(a); } @@ -76,6 +82,42 @@ static void test(const char *name, size_t init_size, const size_t *allocs, static const size_t allocs_##name[] = {__VA_ARGS__}; \ test(#name, init_size, allocs_##name, GPR_ARRAY_SIZE(allocs_##name)) +#define CONCURRENT_TEST_ITERATIONS 100000 +#define CONCURRENT_TEST_THREADS 100 + +typedef struct { + gpr_event ev_start; + gpr_arena *arena; +} concurrent_test_args; + +static void concurrent_test_body(void *arg) { + concurrent_test_args *a = arg; + gpr_event_wait(&a->ev_start, gpr_inf_future(GPR_CLOCK_REALTIME)); + for (size_t i = 0; i < CONCURRENT_TEST_ITERATIONS; i++) { + *(char *)gpr_arena_alloc(a->arena, 1) = (char)i; + } +} + +static void concurrent_test(void) { + concurrent_test_args args; + gpr_event_init(&args.ev_start); + args.arena = gpr_arena_create(1024); + + gpr_thd_id thds[CONCURRENT_TEST_THREADS]; + + for (int i = 0; i < CONCURRENT_TEST_THREADS; i++) { + gpr_thd_options opt = gpr_thd_options_default(); + gpr_thd_options_is_joinable(&opt); + gpr_thd_new(&thds[i], concurrent_test_body, &args, &opt); + } + + gpr_event_set(&args.ev_start, (void *)1); + + for (int i = 0; i < CONCURRENT_TEST_THREADS; i++) { + gpr_thd_join(thds[i]); + } +} + int main(int argc, char *argv[]) { grpc_test_init(argc, argv); @@ -86,6 +128,7 @@ int main(int argc, char *argv[]) { TEST(1_3, 1, 3); TEST(1_inc, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); TEST(6_123, 6, 1, 2, 3); + concurrent_test(); return 0; } From 37723c9ee07c05633b763cfd5cfe0935a20a9258 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Mar 2017 07:45:44 -0700 Subject: [PATCH 128/188] Fix race condition --- src/core/lib/support/arena.c | 11 +++++++---- test/core/support/arena_test.c | 7 ++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/core/lib/support/arena.c b/src/core/lib/support/arena.c index faceb1a1eb3..b5c32b20416 100644 --- a/src/core/lib/support/arena.c +++ b/src/core/lib/support/arena.c @@ -34,6 +34,7 @@ #include "src/core/lib/support/arena.h" #include #include +#include #include #define ROUND_UP_TO_ALIGNMENT_SIZE(x) \ @@ -74,16 +75,16 @@ void *gpr_arena_alloc(gpr_arena *arena, size_t size) { size_t start = (size_t)gpr_atm_no_barrier_fetch_add(&arena->size_so_far, size); zone *z = &arena->initial_zone; - while (start > z->size_begin) { + while (start > z->size_end) { zone *next_z = (zone *)gpr_atm_acq_load(&z->next_atm); - while (next_z == NULL) { - size_t next_z_size = GPR_MAX(2 * start, size); + if (next_z == NULL) { + size_t next_z_size = GPR_MAX((size_t)gpr_atm_no_barrier_load(&arena->size_so_far), size); next_z = gpr_zalloc(sizeof(zone) + next_z_size); next_z->size_begin = z->size_end; next_z->size_end = z->size_end + next_z_size; if (!gpr_atm_rel_cas(&z->next_atm, (gpr_atm)NULL, (gpr_atm)next_z)) { gpr_free(next_z); - next_z = NULL; + next_z = (zone*)gpr_atm_acq_load(&z->next_atm); } } z = next_z; @@ -91,5 +92,7 @@ void *gpr_arena_alloc(gpr_arena *arena, size_t size) { if (start + size > z->size_end) { return gpr_arena_alloc(arena, size); } + GPR_ASSERT(start >= z->size_begin); + GPR_ASSERT(start + size <= z->size_end); return ((char *)(z + 1)) + start - z->size_begin; } diff --git a/test/core/support/arena_test.c b/test/core/support/arena_test.c index 3e21867bcff..35b2bbd1b10 100644 --- a/test/core/support/arena_test.c +++ b/test/core/support/arena_test.c @@ -76,6 +76,7 @@ static void test(const char *name, size_t init_size, const size_t *allocs, memset(ps[i], 1, allocs[i]); } gpr_arena_destroy(a); + gpr_free(ps); } #define TEST(name, init_size, ...) \ @@ -99,6 +100,8 @@ static void concurrent_test_body(void *arg) { } static void concurrent_test(void) { + gpr_log(GPR_DEBUG, "concurrent_test"); + concurrent_test_args args; gpr_event_init(&args.ev_start); args.arena = gpr_arena_create(1024); @@ -107,7 +110,7 @@ static void concurrent_test(void) { for (int i = 0; i < CONCURRENT_TEST_THREADS; i++) { gpr_thd_options opt = gpr_thd_options_default(); - gpr_thd_options_is_joinable(&opt); + gpr_thd_options_set_joinable(&opt); gpr_thd_new(&thds[i], concurrent_test_body, &args, &opt); } @@ -116,6 +119,8 @@ static void concurrent_test(void) { for (int i = 0; i < CONCURRENT_TEST_THREADS; i++) { gpr_thd_join(thds[i]); } + + gpr_arena_destroy(args.arena); } int main(int argc, char *argv[]) { From 09acb108b7620eff840fb0428f7eb159a4f27cf4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Mar 2017 07:45:59 -0700 Subject: [PATCH 129/188] clang-format --- src/core/lib/support/arena.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/lib/support/arena.c b/src/core/lib/support/arena.c index b5c32b20416..6610acd3a07 100644 --- a/src/core/lib/support/arena.c +++ b/src/core/lib/support/arena.c @@ -78,13 +78,14 @@ void *gpr_arena_alloc(gpr_arena *arena, size_t size) { while (start > z->size_end) { zone *next_z = (zone *)gpr_atm_acq_load(&z->next_atm); if (next_z == NULL) { - size_t next_z_size = GPR_MAX((size_t)gpr_atm_no_barrier_load(&arena->size_so_far), size); + size_t next_z_size = + GPR_MAX((size_t)gpr_atm_no_barrier_load(&arena->size_so_far), size); next_z = gpr_zalloc(sizeof(zone) + next_z_size); next_z->size_begin = z->size_end; next_z->size_end = z->size_end + next_z_size; if (!gpr_atm_rel_cas(&z->next_atm, (gpr_atm)NULL, (gpr_atm)next_z)) { gpr_free(next_z); - next_z = (zone*)gpr_atm_acq_load(&z->next_atm); + next_z = (zone *)gpr_atm_acq_load(&z->next_atm); } } z = next_z; From 04b06b7e325a8020c705ad9f4c3511f776d4b7b6 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 13 Mar 2017 09:53:31 -0700 Subject: [PATCH 130/188] Add comments --- .../ext/transport/cronet/transport/cronet_transport.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 1759fe7ad2d..fabfaf8a270 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -152,12 +152,16 @@ struct write_state { struct op_state { bool state_op_done[OP_NUM_OPS]; bool state_callback_received[OP_NUM_OPS]; + /* A non-zero gRPC status code has been seen */ bool fail_state; + /* Transport is discarding all buffered messages */ bool flush_read; bool flush_cronet_when_ready; bool pending_write_for_trailer; bool pending_send_message; + /* User requested RECV_TRAILING_METADATA */ bool pending_recv_trailing_metadata; + /* Cronet has not issued a callback of a bidirectional read */ bool pending_read_from_cronet; grpc_error *cancel_error; /* data structure for storing data coming from server */ @@ -260,6 +264,13 @@ static void null_and_maybe_free_read_buffer(stream_obj *s) { } static void maybe_flush_read(stream_obj *s) { + /* To enter flush read state (discarding all the buffered messages in + * transport layer), two conditions must be satisfied: 1) non-zero grpc status + * has been received, and 2) an op requesting the status code + * (RECV_TRAILING_METADATA) is issued by the user. (See + * doc/status_ordering.md) */ + /* Whenever the evaluation of any of the two condition is changed, we check + * whether we should enter the flush read state. */ if (s->state.pending_recv_trailing_metadata && s->state.fail_state) { if (!s->state.flush_read) { CRONET_LOG(GPR_DEBUG, "%p: Flush read", s); From e7a1702fe99ef598efd957c1f7546904298b9dba Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Mar 2017 10:20:38 -0700 Subject: [PATCH 131/188] Fixes --- src/core/lib/channel/channel_stack.c | 7 ++++--- src/core/lib/channel/channel_stack.h | 10 +++++----- src/core/lib/channel/compress_filter.c | 2 +- src/core/lib/channel/connected_channel.c | 4 ++-- src/core/lib/channel/deadline_filter.c | 2 +- src/core/lib/channel/http_client_filter.c | 2 +- src/core/lib/channel/http_server_filter.c | 2 +- src/core/lib/channel/message_size_filter.c | 2 +- src/core/lib/surface/call.c | 22 ++++++++++++++++++---- src/core/lib/surface/lame_client.c | 4 ++-- src/core/lib/surface/server.c | 2 +- src/core/lib/transport/transport.h | 3 ++- src/core/lib/transport/transport_impl.h | 3 ++- 13 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c index 3fb2a60ac71..187f2c8b343 100644 --- a/src/core/lib/channel/channel_stack.c +++ b/src/core/lib/channel/channel_stack.c @@ -241,15 +241,16 @@ void grpc_call_stack_ignore_set_pollset_or_pollset_set( void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack, const grpc_call_final_info *final_info, - void *and_free_memory) { + grpc_closure *then_schedule_closure) { grpc_call_element *elems = CALL_ELEMS_FROM_STACK(stack); size_t count = stack->count; size_t i; /* destroy per-filter data */ for (i = 0; i < count; i++) { - elems[i].filter->destroy_call_elem(exec_ctx, &elems[i], final_info, - i == count - 1 ? and_free_memory : NULL); + elems[i].filter->destroy_call_elem( + exec_ctx, &elems[i], final_info, + i == count - 1 ? then_schedule_closure : NULL); } } diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index 6d3340bcbf8..1b02f02dd61 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -139,12 +139,12 @@ typedef struct { /* Destroy per call data. The filter does not need to do any chaining. The bottom filter of a stack will be passed a non-NULL pointer to - \a and_free_memory that should be passed to gpr_free when destruction - is complete. \a final_info contains data about the completed call, mainly - for reporting purposes. */ + \a then_schedule_closure that should be passed to grpc_closure_sched when + destruction is complete. \a final_info contains data about the completed + call, mainly for reporting purposes. */ void (*destroy_call_elem)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *and_free_memory); + grpc_closure *then_schedule_closure); /* sizeof(per channel data) */ size_t sizeof_channel_data; @@ -271,7 +271,7 @@ void grpc_call_stack_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx, /* Destroy a call stack */ void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack, const grpc_call_final_info *final_info, - void *and_free_memory); + grpc_closure *then_schedule_closure); /* Ignore set pollset{_set} - used by filters if they don't care about pollsets * at all. Does nothing. */ diff --git a/src/core/lib/channel/compress_filter.c b/src/core/lib/channel/compress_filter.c index aa41014a217..02dc479f3a2 100644 --- a/src/core/lib/channel/compress_filter.c +++ b/src/core/lib/channel/compress_filter.c @@ -292,7 +292,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *ignored) { + grpc_closure *ignored) { /* grab pointers to our data from the call element */ call_data *calld = elem->call_data; grpc_slice_buffer_destroy_internal(exec_ctx, &calld->slices); diff --git a/src/core/lib/channel/connected_channel.c b/src/core/lib/channel/connected_channel.c index 29796f7ca7e..64cf95b5a58 100644 --- a/src/core/lib/channel/connected_channel.c +++ b/src/core/lib/channel/connected_channel.c @@ -105,12 +105,12 @@ static void set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx, /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *and_free_memory) { + grpc_closure *then_schedule_closure) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; grpc_transport_destroy_stream(exec_ctx, chand->transport, TRANSPORT_STREAM_FROM_CALL_DATA(calld), - and_free_memory); + then_schedule_closure); } /* Constructor for channel_data */ diff --git a/src/core/lib/channel/deadline_filter.c b/src/core/lib/channel/deadline_filter.c index 5a12d62f1d1..34114bbebfd 100644 --- a/src/core/lib/channel/deadline_filter.c +++ b/src/core/lib/channel/deadline_filter.c @@ -256,7 +256,7 @@ static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx, // Destructor for call_data. Used for both client and server filters. static void destroy_call_elem(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, const grpc_call_final_info* final_info, - void* and_free_memory) { + grpc_closure* ignored) { grpc_deadline_state_destroy(exec_ctx, elem); } diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c index c031533dd86..f9d0d689acd 100644 --- a/src/core/lib/channel/http_client_filter.c +++ b/src/core/lib/channel/http_client_filter.c @@ -412,7 +412,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *ignored) { + grpc_closure *ignored) { call_data *calld = elem->call_data; grpc_slice_buffer_destroy_internal(exec_ctx, &calld->slices); } diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c index fb70de8e96c..bebd3af335b 100644 --- a/src/core/lib/channel/http_server_filter.c +++ b/src/core/lib/channel/http_server_filter.c @@ -358,7 +358,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *ignored) { + grpc_closure *ignored) { call_data *calld = elem->call_data; grpc_slice_buffer_destroy_internal(exec_ctx, &calld->read_slice_buffer); } diff --git a/src/core/lib/channel/message_size_filter.c b/src/core/lib/channel/message_size_filter.c index b424c0d2acb..5ba13fe2514 100644 --- a/src/core/lib/channel/message_size_filter.c +++ b/src/core/lib/channel/message_size_filter.c @@ -200,7 +200,7 @@ static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx, // Destructor for call_data. static void destroy_call_elem(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, const grpc_call_final_info* final_info, - void* ignored) {} + grpc_closure* ignored) {} // Constructor for channel_data. static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx, diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index c2547c5147a..b57dd8bd8ae 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -51,6 +51,7 @@ #include "src/core/lib/profiling/timers.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" +#include "src/core/lib/support/arena.h" #include "src/core/lib/support/string.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/call.h" @@ -138,6 +139,7 @@ typedef struct batch_control { } batch_control; struct grpc_call { + gpr_arena *arena; grpc_completion_queue *cq; grpc_polling_entity pollent; grpc_channel *channel; @@ -212,6 +214,8 @@ struct grpc_call { grpc_closure receiving_initial_metadata_ready; uint32_t test_only_last_message_flags; + grpc_closure release_call; + union { struct { grpc_status_code *status; @@ -273,7 +277,10 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, grpc_channel_get_channel_stack(args->channel); grpc_call *call; GPR_TIMER_BEGIN("grpc_call_create", 0); - call = gpr_zalloc(sizeof(grpc_call) + channel_stack->call_stack_size); + gpr_arena *arena = gpr_arena_create(8192); + call = gpr_arena_alloc(arena, + sizeof(grpc_call) + channel_stack->call_stack_size); + call->arena = arena; *out_call = call; gpr_mu_init(&call->mu); call->channel = args->channel; @@ -421,6 +428,13 @@ void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c REF_ARG) { GRPC_CALL_STACK_UNREF(exec_ctx, CALL_STACK_FROM_CALL(c), REF_REASON); } +static void release_call(grpc_exec_ctx *exec_ctx, void *call, + grpc_error *error) { + grpc_call *c = call; + gpr_arena_destroy(c->arena); + GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, c->channel, "call"); +} + static void set_status_value_directly(grpc_status_code status, void *dest); static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, grpc_error *error) { @@ -447,7 +461,6 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, if (c->cq) { GRPC_CQ_INTERNAL_UNREF(c->cq, "bind"); } - grpc_channel *channel = c->channel; get_final_status(call, set_status_value_directly, &c->final_info.final_status, NULL); @@ -459,8 +472,9 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, unpack_received_status(gpr_atm_no_barrier_load(&c->status[i])).error); } - grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), &c->final_info, c); - GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "call"); + grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), &c->final_info, + grpc_closure_init(&c->release_call, release_call, c, + grpc_schedule_on_exec_ctx)); GPR_TIMER_END("destroy_call", 0); } diff --git a/src/core/lib/surface/lame_client.c b/src/core/lib/surface/lame_client.c index 49bc4c114b3..9ddb88bd114 100644 --- a/src/core/lib/surface/lame_client.c +++ b/src/core/lib/surface/lame_client.c @@ -130,8 +130,8 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *and_free_memory) { - gpr_free(and_free_memory); + grpc_closure *then_schedule_closure) { + grpc_closure_sched(exec_ctx, then_schedule_closure, GRPC_ERROR_NONE); } static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index b3605795536..1186a4af636 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -898,7 +898,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *ignored) { + grpc_closure *ignored) { channel_data *chand = elem->channel_data; call_data *calld = elem->call_data; diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index cc1c277b355..11fd5d1697c 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -246,7 +246,8 @@ void grpc_transport_set_pops(grpc_exec_ctx *exec_ctx, grpc_transport *transport, caller, but any child memory must be cleaned up) */ void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *transport, - grpc_stream *stream, void *and_free_memory); + grpc_stream *stream, + grpc_closure *then_schedule_closure); void grpc_transport_stream_op_finish_with_failure(grpc_exec_ctx *exec_ctx, grpc_transport_stream_op *op, diff --git a/src/core/lib/transport/transport_impl.h b/src/core/lib/transport/transport_impl.h index 8553148c35f..9b6095f6666 100644 --- a/src/core/lib/transport/transport_impl.h +++ b/src/core/lib/transport/transport_impl.h @@ -67,7 +67,8 @@ typedef struct grpc_transport_vtable { /* implementation of grpc_transport_destroy_stream */ void (*destroy_stream)(grpc_exec_ctx *exec_ctx, grpc_transport *self, - grpc_stream *stream, void *and_free_memory); + grpc_stream *stream, + grpc_closure *then_schedule_closure); /* implementation of grpc_transport_destroy */ void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_transport *self); From 5398a9696398917090619f95530df7f4acaadba6 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 13 Mar 2017 10:27:18 -0700 Subject: [PATCH 132/188] Fix copy and unref race cond bug --- src/core/lib/iomgr/error.c | 8 +++++--- src/core/lib/iomgr/error_internal.h | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index 7cdbe30198d..f0e294b6091 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -382,10 +382,12 @@ static grpc_error *copy_error_and_unref(grpc_error *in) { #ifdef GRPC_ERROR_REFCOUNT_DEBUG gpr_log(GPR_DEBUG, "%p create copying %p", out, in); #endif - memcpy(out, in, sizeof(*in) + in->arena_size * sizeof(intptr_t)); - out->arena_capacity = new_arena_capacity; - gpr_atm_no_barrier_store(&out->error_string, 0); gpr_ref_init(&out->refs, 1); + gpr_atm_no_barrier_store(&out->error_string, 0); + size_t skip = sizeof(gpr_refcount) + sizeof(gpr_atm); + memcpy((void *)out + skip, (void *)in + skip, + sizeof(*in) + (in->arena_size * sizeof(intptr_t)) - skip); + out->arena_capacity = new_arena_capacity; ref_strs(out); ref_errs(out); GRPC_ERROR_UNREF(in); diff --git a/src/core/lib/iomgr/error_internal.h b/src/core/lib/iomgr/error_internal.h index fb4814e41f2..0718d31f232 100644 --- a/src/core/lib/iomgr/error_internal.h +++ b/src/core/lib/iomgr/error_internal.h @@ -48,12 +48,12 @@ struct grpc_linked_error { struct grpc_error { gpr_refcount refs; + gpr_atm error_string; uint8_t ints[GRPC_ERROR_INT_MAX]; uint8_t strs[GRPC_ERROR_STR_MAX]; uint8_t times[GRPC_ERROR_TIME_MAX]; uint8_t first_err; uint8_t last_err; - gpr_atm error_string; uint8_t arena_size; uint8_t arena_capacity; intptr_t arena[0]; From d426caca811823f525334ec5952995e0b887f04f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Mar 2017 12:30:45 -0700 Subject: [PATCH 133/188] Use an arena for call & subchannel_call allocation --- src/core/ext/census/grpc_filter.c | 4 +-- src/core/ext/client_channel/client_channel.c | 26 +++++++++++---- src/core/ext/client_channel/subchannel.c | 33 ++++++++++++++----- src/core/ext/client_channel/subchannel.h | 18 ++++++++-- .../load_reporting/load_reporting_filter.c | 2 +- .../chttp2/transport/chttp2_transport.c | 7 ++-- .../ext/transport/chttp2/transport/internal.h | 2 +- src/core/lib/channel/channel_stack.c | 29 ++++++---------- src/core/lib/channel/channel_stack.h | 13 ++++---- .../security/transport/client_auth_filter.c | 2 +- .../security/transport/server_auth_filter.c | 2 +- src/core/lib/surface/call.c | 15 ++++++--- src/core/lib/transport/transport.c | 5 +-- test/core/channel/channel_stack_test.c | 16 ++++++--- .../end2end/tests/filter_call_init_fails.c | 2 +- test/core/end2end/tests/filter_causes_close.c | 2 +- test/core/end2end/tests/filter_latency.c | 4 +-- 17 files changed, 116 insertions(+), 66 deletions(-) diff --git a/src/core/ext/census/grpc_filter.c b/src/core/ext/census/grpc_filter.c index b80d831557f..fc29dbd4545 100644 --- a/src/core/ext/census/grpc_filter.c +++ b/src/core/ext/census/grpc_filter.c @@ -138,7 +138,7 @@ static grpc_error *client_init_call_elem(grpc_exec_ctx *exec_ctx, static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *ignored) { + grpc_closure *ignored) { call_data *d = elem->call_data; GPR_ASSERT(d != NULL); /* TODO(hongyu): record rpc client stats and census_rpc_end_op here */ @@ -160,7 +160,7 @@ static grpc_error *server_init_call_elem(grpc_exec_ctx *exec_ctx, static void server_destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *ignored) { + grpc_closure *ignored) { call_data *d = elem->call_data; GPR_ASSERT(d != NULL); /* TODO(hongyu): record rpc server stats and census_tracing_end_op here */ diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index bf64f84772c..f6550292a0a 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -660,6 +660,7 @@ typedef struct client_channel_call_data { /** either 0 for no call, 1 for cancelled, or a pointer to a grpc_subchannel_call */ gpr_atm subchannel_call; + gpr_arena *arena; subchannel_creation_phase creation_phase; grpc_connected_subchannel *connected_subchannel; @@ -754,9 +755,14 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, } else { /* Create call on subchannel. */ grpc_subchannel_call *subchannel_call = NULL; + grpc_connected_subchannel_call_args call_args = { + .pollent = calld->pollent, + .path = calld->path, + .start_time = calld->call_start_time, + .deadline = calld->deadline, + .arena = calld->arena}; grpc_error *new_error = grpc_connected_subchannel_create_call( - exec_ctx, calld->connected_subchannel, calld->pollent, calld->path, - calld->call_start_time, calld->deadline, &subchannel_call); + exec_ctx, calld->connected_subchannel, &call_args, &subchannel_call); if (new_error != GRPC_ERROR_NONE) { new_error = grpc_error_add_child(new_error, error); subchannel_call = CANCELLED_CALL; @@ -982,9 +988,14 @@ static void start_transport_stream_op_locked_inner(grpc_exec_ctx *exec_ctx, if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && calld->connected_subchannel != NULL) { grpc_subchannel_call *subchannel_call = NULL; + grpc_connected_subchannel_call_args call_args = { + .pollent = calld->pollent, + .path = calld->path, + .start_time = calld->call_start_time, + .deadline = calld->deadline, + .arena = calld->arena}; grpc_error *error = grpc_connected_subchannel_create_call( - exec_ctx, calld->connected_subchannel, calld->pollent, calld->path, - calld->call_start_time, calld->deadline, &subchannel_call); + exec_ctx, calld->connected_subchannel, &call_args, &subchannel_call); if (error != GRPC_ERROR_NONE) { subchannel_call = CANCELLED_CALL; fail_locked(exec_ctx, calld, GRPC_ERROR_REF(error)); @@ -1161,6 +1172,7 @@ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; calld->owning_call = args->call_stack; calld->pollent = NULL; + calld->arena = args->arena; GRPC_CALL_STACK_REF(calld->owning_call, "initial_read_service_config"); grpc_closure_sched( exec_ctx, @@ -1175,7 +1187,7 @@ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, static void cc_destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *and_free_memory) { + grpc_closure *then_schedule_closure) { call_data *calld = elem->call_data; grpc_deadline_state_destroy(exec_ctx, elem); grpc_slice_unref_internal(exec_ctx, calld->path); @@ -1185,6 +1197,8 @@ static void cc_destroy_call_elem(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(calld->cancel_error); grpc_subchannel_call *call = GET_CALL(calld); if (call != NULL && call != CANCELLED_CALL) { + grpc_subchannel_call_set_cleanup_closure(call, then_schedule_closure); + then_schedule_closure = NULL; GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, call, "client_channel_destroy_call"); } GPR_ASSERT(calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING); @@ -1194,7 +1208,7 @@ static void cc_destroy_call_elem(grpc_exec_ctx *exec_ctx, "picked"); } gpr_free(calld->waiting_ops); - gpr_free(and_free_memory); + grpc_closure_sched(exec_ctx, then_schedule_closure, GRPC_ERROR_NONE); } static void cc_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx, diff --git a/src/core/ext/client_channel/subchannel.c b/src/core/ext/client_channel/subchannel.c index 5df0a9060d0..e4b669c5828 100644 --- a/src/core/ext/client_channel/subchannel.c +++ b/src/core/ext/client_channel/subchannel.c @@ -148,6 +148,7 @@ struct grpc_subchannel { struct grpc_subchannel_call { grpc_connected_subchannel *connection; + grpc_closure *schedule_closure_after_destroy; }; #define SUBCHANNEL_CALL_TO_CALL_STACK(call) ((grpc_call_stack *)((call) + 1)) @@ -719,13 +720,22 @@ static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg, static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call, grpc_error *error) { grpc_subchannel_call *c = call; + GPR_ASSERT(c->schedule_closure_after_destroy != NULL); GPR_TIMER_BEGIN("grpc_subchannel_call_unref.destroy", 0); grpc_connected_subchannel *connection = c->connection; - grpc_call_stack_destroy(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c), NULL, c); + grpc_call_stack_destroy(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c), NULL, + c->schedule_closure_after_destroy); GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, connection, "subchannel_call"); GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0); } +void grpc_subchannel_call_set_cleanup_closure(grpc_subchannel_call *call, + grpc_closure *closure) { + GPR_ASSERT(call->schedule_closure_after_destroy == NULL); + GPR_ASSERT(closure != NULL); + call->schedule_closure_after_destroy = closure; +} + void grpc_subchannel_call_ref( grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CALL_STACK_REF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON); @@ -761,15 +771,22 @@ grpc_connected_subchannel *grpc_subchannel_get_connected_subchannel( grpc_error *grpc_connected_subchannel_create_call( grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *con, - grpc_polling_entity *pollent, grpc_slice path, gpr_timespec start_time, - gpr_timespec deadline, grpc_subchannel_call **call) { + const grpc_connected_subchannel_call_args *args, + grpc_subchannel_call **call) { grpc_channel_stack *chanstk = CHANNEL_STACK_FROM_CONNECTION(con); - *call = gpr_zalloc(sizeof(grpc_subchannel_call) + chanstk->call_stack_size); + *call = gpr_arena_alloc( + args->arena, sizeof(grpc_subchannel_call) + chanstk->call_stack_size); grpc_call_stack *callstk = SUBCHANNEL_CALL_TO_CALL_STACK(*call); (*call)->connection = con; // Ref is added below. - grpc_error *error = - grpc_call_stack_init(exec_ctx, chanstk, 1, subchannel_call_destroy, *call, - NULL, NULL, path, start_time, deadline, callstk); + grpc_call_element_args call_args = {.call_stack = callstk, + .server_transport_data = NULL, + .context = NULL, + .path = args->path, + .start_time = args->start_time, + .deadline = args->deadline, + .arena = args->arena}; + grpc_error *error = grpc_call_stack_init( + exec_ctx, chanstk, 1, subchannel_call_destroy, *call, &call_args); if (error != GRPC_ERROR_NONE) { const char *error_string = grpc_error_string(error); gpr_log(GPR_ERROR, "error: %s", error_string); @@ -778,7 +795,7 @@ grpc_error *grpc_connected_subchannel_create_call( return error; } GRPC_CONNECTED_SUBCHANNEL_REF(con, "subchannel_call"); - grpc_call_stack_set_pollset_or_pollset_set(exec_ctx, callstk, pollent); + grpc_call_stack_set_pollset_or_pollset_set(exec_ctx, callstk, args->pollent); return GRPC_ERROR_NONE; } diff --git a/src/core/ext/client_channel/subchannel.h b/src/core/ext/client_channel/subchannel.h index 6a70a764676..3e64a2507cf 100644 --- a/src/core/ext/client_channel/subchannel.h +++ b/src/core/ext/client_channel/subchannel.h @@ -37,6 +37,7 @@ #include "src/core/ext/client_channel/connector.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/iomgr/polling_entity.h" +#include "src/core/lib/support/arena.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata.h" @@ -112,10 +113,18 @@ void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx, GRPC_SUBCHANNEL_REF_EXTRA_ARGS); /** construct a subchannel call */ +typedef struct { + grpc_polling_entity *pollent; + grpc_slice path; + gpr_timespec start_time; + gpr_timespec deadline; + gpr_arena *arena; +} grpc_connected_subchannel_call_args; + grpc_error *grpc_connected_subchannel_create_call( grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *connected_subchannel, - grpc_polling_entity *pollent, grpc_slice path, gpr_timespec start_time, - gpr_timespec deadline, grpc_subchannel_call **subchannel_call); + const grpc_connected_subchannel_call_args *args, + grpc_subchannel_call **subchannel_call); /** process a transport level op */ void grpc_connected_subchannel_process_transport_op( @@ -154,6 +163,11 @@ void grpc_subchannel_call_process_op(grpc_exec_ctx *exec_ctx, char *grpc_subchannel_call_get_peer(grpc_exec_ctx *exec_ctx, grpc_subchannel_call *subchannel_call); +/** Must be called once per call. Sets the 'then_schedule_closure' argument for + call stack destruction. */ +void grpc_subchannel_call_set_cleanup_closure( + grpc_subchannel_call *subchannel_call, grpc_closure *closure); + grpc_call_stack *grpc_subchannel_call_get_call_stack( grpc_subchannel_call *subchannel_call); diff --git a/src/core/ext/load_reporting/load_reporting_filter.c b/src/core/ext/load_reporting/load_reporting_filter.c index c2750634a50..4ed832671d9 100644 --- a/src/core/ext/load_reporting/load_reporting_filter.c +++ b/src/core/ext/load_reporting/load_reporting_filter.c @@ -123,7 +123,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *ignored) { + grpc_closure *ignored) { call_data *calld = elem->call_data; /* TODO(dgq): do something with the data diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index da4c7dc7b23..8024dd47020 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -665,16 +665,17 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, GPR_TIMER_END("destroy_stream", 0); - gpr_free(s->destroy_stream_arg); + grpc_closure_sched(exec_ctx, s->destroy_stream_arg, GRPC_ERROR_NONE); } static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, void *and_free_memory) { + grpc_stream *gs, + grpc_closure *then_schedule_closure) { GPR_TIMER_BEGIN("destroy_stream", 0); grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; grpc_chttp2_stream *s = (grpc_chttp2_stream *)gs; - s->destroy_stream_arg = and_free_memory; + s->destroy_stream_arg = then_schedule_closure; grpc_closure_sched( exec_ctx, grpc_closure_init(&s->destroy_stream, destroy_stream_locked, s, grpc_combiner_scheduler(t->combiner, false)), diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index d26812ad6be..3c56c215991 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -425,7 +425,7 @@ struct grpc_chttp2_stream { grpc_stream_refcount *refcount; grpc_closure destroy_stream; - void *destroy_stream_arg; + grpc_closure *destroy_stream_arg; grpc_chttp2_stream_link links[STREAM_LIST_COUNT]; uint8_t included[STREAM_LIST_COUNT]; diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c index 187f2c8b343..6d53b0576e7 100644 --- a/src/core/lib/channel/channel_stack.c +++ b/src/core/lib/channel/channel_stack.c @@ -166,41 +166,32 @@ void grpc_channel_stack_destroy(grpc_exec_ctx *exec_ctx, } } -grpc_error *grpc_call_stack_init( - grpc_exec_ctx *exec_ctx, grpc_channel_stack *channel_stack, - int initial_refs, grpc_iomgr_cb_func destroy, void *destroy_arg, - grpc_call_context_element *context, const void *transport_server_data, - grpc_slice path, gpr_timespec start_time, gpr_timespec deadline, - grpc_call_stack *call_stack) { +grpc_error *grpc_call_stack_init(grpc_exec_ctx *exec_ctx, + grpc_channel_stack *channel_stack, + int initial_refs, grpc_iomgr_cb_func destroy, + void *destroy_arg, + const grpc_call_element_args *elem_args) { grpc_channel_element *channel_elems = CHANNEL_ELEMS_FROM_STACK(channel_stack); size_t count = channel_stack->count; grpc_call_element *call_elems; char *user_data; size_t i; - call_stack->count = count; - GRPC_STREAM_REF_INIT(&call_stack->refcount, initial_refs, destroy, + elem_args->call_stack->count = count; + GRPC_STREAM_REF_INIT(&elem_args->call_stack->refcount, initial_refs, destroy, destroy_arg, "CALL_STACK"); - call_elems = CALL_ELEMS_FROM_STACK(call_stack); + call_elems = CALL_ELEMS_FROM_STACK(elem_args->call_stack); user_data = ((char *)call_elems) + ROUND_UP_TO_ALIGNMENT_SIZE(count * sizeof(grpc_call_element)); /* init per-filter data */ grpc_error *first_error = GRPC_ERROR_NONE; - const grpc_call_element_args args = { - .start_time = start_time, - .call_stack = call_stack, - .server_transport_data = transport_server_data, - .context = context, - .path = path, - .deadline = deadline, - }; for (i = 0; i < count; i++) { call_elems[i].filter = channel_elems[i].filter; call_elems[i].channel_data = channel_elems[i].channel_data; call_elems[i].call_data = user_data; - grpc_error *error = - call_elems[i].filter->init_call_elem(exec_ctx, &call_elems[i], &args); + grpc_error *error = call_elems[i].filter->init_call_elem( + exec_ctx, &call_elems[i], elem_args); if (error != GRPC_ERROR_NONE) { if (first_error == GRPC_ERROR_NONE) { first_error = error; diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index 1b02f02dd61..80e3603e8d8 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -56,6 +56,7 @@ #include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/polling_entity.h" +#include "src/core/lib/support/arena.h" #include "src/core/lib/transport/transport.h" #ifdef __cplusplus @@ -84,6 +85,7 @@ typedef struct { grpc_slice path; gpr_timespec start_time; gpr_timespec deadline; + gpr_arena *arena; } grpc_call_element_args; typedef struct { @@ -236,12 +238,11 @@ void grpc_channel_stack_destroy(grpc_exec_ctx *exec_ctx, /* Initialize a call stack given a channel stack. transport_server_data is expected to be NULL on a client, or an opaque transport owned pointer on the server. */ -grpc_error *grpc_call_stack_init( - grpc_exec_ctx *exec_ctx, grpc_channel_stack *channel_stack, - int initial_refs, grpc_iomgr_cb_func destroy, void *destroy_arg, - grpc_call_context_element *context, const void *transport_server_data, - grpc_slice path, gpr_timespec start_time, gpr_timespec deadline, - grpc_call_stack *call_stack); +grpc_error *grpc_call_stack_init(grpc_exec_ctx *exec_ctx, + grpc_channel_stack *channel_stack, + int initial_refs, grpc_iomgr_cb_func destroy, + void *destroy_arg, + const grpc_call_element_args *elem_args); /* Set a pollset or a pollset_set for a call stack: must occur before the first * op is started */ void grpc_call_stack_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c index a23082a8667..8dea1d98fff 100644 --- a/src/core/lib/security/transport/client_auth_filter.c +++ b/src/core/lib/security/transport/client_auth_filter.c @@ -318,7 +318,7 @@ static void set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx, /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *ignored) { + grpc_closure *ignored) { call_data *calld = elem->call_data; grpc_call_credentials_unref(exec_ctx, calld->creds); if (calld->have_host) { diff --git a/src/core/lib/security/transport/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c index 14619d97caf..01cb4731773 100644 --- a/src/core/lib/security/transport/server_auth_filter.c +++ b/src/core/lib/security/transport/server_auth_filter.c @@ -227,7 +227,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *ignored) {} + grpc_closure *ignored) {} /* Constructor for channel_data */ static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index b57dd8bd8ae..84c401db35a 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -367,11 +367,16 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, GRPC_CHANNEL_INTERNAL_REF(args->channel, "call"); /* initial refcount dropped by grpc_call_destroy */ + grpc_call_element_args call_args = { + .call_stack = CALL_STACK_FROM_CALL(call), + .server_transport_data = args->server_transport_data, + .context = call->context, + .path = path, + .start_time = call->start_time, + .deadline = send_deadline, + .arena = call->arena}; add_init_error(&error, grpc_call_stack_init(exec_ctx, channel_stack, 1, - destroy_call, call, call->context, - args->server_transport_data, path, - call->start_time, send_deadline, - CALL_STACK_FROM_CALL(call))); + destroy_call, call, &call_args)); if (error != GRPC_ERROR_NONE) { cancel_with_error(exec_ctx, call, STATUS_FROM_SURFACE, GRPC_ERROR_REF(error)); @@ -431,8 +436,8 @@ void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c REF_ARG) { static void release_call(grpc_exec_ctx *exec_ctx, void *call, grpc_error *error) { grpc_call *c = call; - gpr_arena_destroy(c->arena); GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, c->channel, "call"); + gpr_arena_destroy(c->arena); } static void set_status_value_directly(grpc_status_code status, void *dest); diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index 165950e288e..7635fc730d2 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -197,9 +197,10 @@ void grpc_transport_set_pops(grpc_exec_ctx *exec_ctx, grpc_transport *transport, void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *transport, - grpc_stream *stream, void *and_free_memory) { + grpc_stream *stream, + grpc_closure *then_schedule_closure) { transport->vtable->destroy_stream(exec_ctx, transport, stream, - and_free_memory); + then_schedule_closure); } char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx, diff --git a/test/core/channel/channel_stack_test.c b/test/core/channel/channel_stack_test.c index 76bb57346c0..0c424c1ec48 100644 --- a/test/core/channel/channel_stack_test.c +++ b/test/core/channel/channel_stack_test.c @@ -68,7 +68,7 @@ static void channel_destroy_func(grpc_exec_ctx *exec_ctx, static void call_destroy_func(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *ignored) { + grpc_closure *ignored) { ++*(int *)(elem->channel_data); } @@ -139,10 +139,16 @@ static void test_create_channel_stack(void) { GPR_ASSERT(*channel_data == 0); call_stack = gpr_malloc(channel_stack->call_stack_size); - grpc_error *error = - grpc_call_stack_init(&exec_ctx, channel_stack, 1, free_call, call_stack, - NULL, NULL, path, gpr_now(GPR_CLOCK_MONOTONIC), - gpr_inf_future(GPR_CLOCK_MONOTONIC), call_stack); + grpc_call_element_args args = { + .call_stack = call_stack, + .server_transport_data = NULL, + .context = NULL, + .path = path, + .start_time = gpr_now(GPR_CLOCK_MONOTONIC), + .deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC), + .arena = NULL}; + grpc_error *error = grpc_call_stack_init(&exec_ctx, channel_stack, 1, + free_call, call_stack, &args); GPR_ASSERT(error == GRPC_ERROR_NONE); GPR_ASSERT(call_stack->count == 1); call_elem = grpc_call_stack_element(call_stack, 0); diff --git a/test/core/end2end/tests/filter_call_init_fails.c b/test/core/end2end/tests/filter_call_init_fails.c index d2d6e82d574..65216cf19dc 100644 --- a/test/core/end2end/tests/filter_call_init_fails.c +++ b/test/core/end2end/tests/filter_call_init_fails.c @@ -213,7 +213,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *and_free_memory) {} + grpc_closure *ignored) {} static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, diff --git a/test/core/end2end/tests/filter_causes_close.c b/test/core/end2end/tests/filter_causes_close.c index 25e606556da..c968f30b3b8 100644 --- a/test/core/end2end/tests/filter_causes_close.c +++ b/test/core/end2end/tests/filter_causes_close.c @@ -236,7 +236,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *and_free_memory) {} + grpc_closure *ignored) {} static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, diff --git a/test/core/end2end/tests/filter_latency.c b/test/core/end2end/tests/filter_latency.c index d05e9e79a11..2428c92a42b 100644 --- a/test/core/end2end/tests/filter_latency.c +++ b/test/core/end2end/tests/filter_latency.c @@ -267,7 +267,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *and_free_memory) { + grpc_closure *ignored) { gpr_mu_lock(&g_mu); g_client_latency = final_info->stats.latency; gpr_mu_unlock(&g_mu); @@ -276,7 +276,7 @@ static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx, static void server_destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *and_free_memory) { + grpc_closure *ignored) { gpr_mu_lock(&g_mu); g_server_latency = final_info->stats.latency; gpr_mu_unlock(&g_mu); From 7d2c39827688026ca54b6a8a4d36c36d79aa282a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Mar 2017 12:58:29 -0700 Subject: [PATCH 134/188] Additionally use arena for incoming metadata allocation --- .../chttp2/transport/chttp2_transport.c | 26 ++++---- .../chttp2/transport/incoming_metadata.c | 65 +++++++------------ .../chttp2/transport/incoming_metadata.h | 18 +++-- .../ext/transport/chttp2/transport/parsing.c | 18 ++++- src/core/lib/channel/connected_channel.c | 2 +- src/core/lib/transport/transport.c | 4 +- src/core/lib/transport/transport.h | 3 +- src/core/lib/transport/transport_impl.h | 2 +- 8 files changed, 67 insertions(+), 71 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 8024dd47020..114b14e8845 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -575,7 +575,7 @@ void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s) { static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_stream_refcount *refcount, - const void *server_data) { + const void *server_data, gpr_arena *arena) { GPR_TIMER_BEGIN("init_stream", 0); grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; grpc_chttp2_stream *s = (grpc_chttp2_stream *)gs; @@ -588,8 +588,8 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, gpr_ref_init(&s->active_streams, 1); GRPC_CHTTP2_STREAM_REF(s, "chttp2"); - grpc_chttp2_incoming_metadata_buffer_init(&s->metadata_buffer[0]); - grpc_chttp2_incoming_metadata_buffer_init(&s->metadata_buffer[1]); + grpc_chttp2_incoming_metadata_buffer_init(&s->metadata_buffer[0], arena); + grpc_chttp2_incoming_metadata_buffer_init(&s->metadata_buffer[1], arena); grpc_chttp2_data_parser_init(&s->data_parser); grpc_slice_buffer_init(&s->flow_controlled_buffer); s->deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); @@ -1630,15 +1630,19 @@ void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, s->recv_trailing_metadata_finished != NULL) { char status_string[GPR_LTOA_MIN_BUFSIZE]; gpr_ltoa(status, status_string); - grpc_chttp2_incoming_metadata_buffer_replace_or_add( - exec_ctx, &s->metadata_buffer[1], - grpc_mdelem_from_slices(exec_ctx, GRPC_MDSTR_GRPC_STATUS, - grpc_slice_from_copied_string(status_string))); + GRPC_LOG_IF_ERROR("add_status", + grpc_chttp2_incoming_metadata_buffer_replace_or_add( + exec_ctx, &s->metadata_buffer[1], + grpc_mdelem_from_slices( + exec_ctx, GRPC_MDSTR_GRPC_STATUS, + grpc_slice_from_copied_string(status_string)))); if (msg != NULL) { - grpc_chttp2_incoming_metadata_buffer_replace_or_add( - exec_ctx, &s->metadata_buffer[1], - grpc_mdelem_from_slices(exec_ctx, GRPC_MDSTR_GRPC_MESSAGE, - grpc_slice_from_copied_string(msg))); + GRPC_LOG_IF_ERROR( + "add_status_message", + grpc_chttp2_incoming_metadata_buffer_replace_or_add( + exec_ctx, &s->metadata_buffer[1], + grpc_mdelem_from_slices(exec_ctx, GRPC_MDSTR_GRPC_MESSAGE, + grpc_slice_from_copied_string(msg)))); } s->published_metadata[1] = GRPC_METADATA_SYNTHESIZED_FROM_FAKE; grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s); diff --git a/src/core/ext/transport/chttp2/transport/incoming_metadata.c b/src/core/ext/transport/chttp2/transport/incoming_metadata.c index c91b019aa05..ab900fdff26 100644 --- a/src/core/ext/transport/chttp2/transport/incoming_metadata.c +++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.c @@ -41,69 +41,48 @@ #include void grpc_chttp2_incoming_metadata_buffer_init( - grpc_chttp2_incoming_metadata_buffer *buffer) { - buffer->deadline = gpr_inf_future(GPR_CLOCK_REALTIME); + grpc_chttp2_incoming_metadata_buffer *buffer, gpr_arena *arena) { + grpc_metadata_batch_init(&buffer->batch); + buffer->arena = arena; + buffer->batch.deadline = gpr_inf_future(GPR_CLOCK_REALTIME); } void grpc_chttp2_incoming_metadata_buffer_destroy( grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_metadata_buffer *buffer) { - size_t i; - if (!buffer->published) { - for (i = 0; i < buffer->count; i++) { - GRPC_MDELEM_UNREF(exec_ctx, buffer->elems[i].md); - } - } - gpr_free(buffer->elems); + grpc_metadata_batch_destroy(exec_ctx, &buffer->batch); } -void grpc_chttp2_incoming_metadata_buffer_add( - grpc_chttp2_incoming_metadata_buffer *buffer, grpc_mdelem elem) { - GPR_ASSERT(!buffer->published); - if (buffer->capacity == buffer->count) { - buffer->capacity = GPR_MAX(8, 2 * buffer->capacity); - buffer->elems = - gpr_realloc(buffer->elems, sizeof(*buffer->elems) * buffer->capacity); - } - buffer->elems[buffer->count++].md = elem; +grpc_error *grpc_chttp2_incoming_metadata_buffer_add( + grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_metadata_buffer *buffer, + grpc_mdelem elem) { buffer->size += GRPC_MDELEM_LENGTH(elem); + return grpc_metadata_batch_add_tail( + exec_ctx, &buffer->batch, + gpr_arena_alloc(buffer->arena, sizeof(grpc_linked_mdelem)), elem); } -void grpc_chttp2_incoming_metadata_buffer_replace_or_add( +grpc_error *grpc_chttp2_incoming_metadata_buffer_replace_or_add( grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_metadata_buffer *buffer, grpc_mdelem elem) { - for (size_t i = 0; i < buffer->count; i++) { - if (grpc_slice_eq(GRPC_MDKEY(buffer->elems[i].md), GRPC_MDKEY(elem))) { - GRPC_MDELEM_UNREF(exec_ctx, buffer->elems[i].md); - buffer->elems[i].md = elem; - return; + for (grpc_linked_mdelem *l = buffer->batch.list.head; l != NULL; + l = l->next) { + if (grpc_slice_eq(GRPC_MDKEY(l->md), GRPC_MDKEY(elem))) { + GRPC_MDELEM_UNREF(exec_ctx, l->md); + l->md = elem; + return GRPC_ERROR_NONE; } } - grpc_chttp2_incoming_metadata_buffer_add(buffer, elem); + return grpc_chttp2_incoming_metadata_buffer_add(exec_ctx, buffer, elem); } void grpc_chttp2_incoming_metadata_buffer_set_deadline( grpc_chttp2_incoming_metadata_buffer *buffer, gpr_timespec deadline) { - GPR_ASSERT(!buffer->published); - buffer->deadline = deadline; + buffer->batch.deadline = deadline; } void grpc_chttp2_incoming_metadata_buffer_publish( grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_metadata_buffer *buffer, grpc_metadata_batch *batch) { - GPR_ASSERT(!buffer->published); - buffer->published = 1; - if (buffer->count > 0) { - size_t i; - for (i = 0; i < buffer->count; i++) { - /* TODO(ctiller): do something better here */ - if (!GRPC_LOG_IF_ERROR("grpc_chttp2_incoming_metadata_buffer_publish", - grpc_metadata_batch_link_tail( - exec_ctx, batch, &buffer->elems[i]))) { - GRPC_MDELEM_UNREF(exec_ctx, buffer->elems[i].md); - } - } - } else { - batch->list.head = batch->list.tail = NULL; - } - batch->deadline = buffer->deadline; + *batch = buffer->batch; + grpc_metadata_batch_init(&buffer->batch); } diff --git a/src/core/ext/transport/chttp2/transport/incoming_metadata.h b/src/core/ext/transport/chttp2/transport/incoming_metadata.h index 1eac6fc1504..288c917e65a 100644 --- a/src/core/ext/transport/chttp2/transport/incoming_metadata.h +++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.h @@ -37,28 +37,26 @@ #include "src/core/lib/transport/transport.h" typedef struct { - grpc_linked_mdelem *elems; - size_t count; - size_t capacity; - gpr_timespec deadline; - int published; + gpr_arena *arena; + grpc_metadata_batch batch; size_t size; // total size of metadata } grpc_chttp2_incoming_metadata_buffer; /** assumes everything initially zeroed */ void grpc_chttp2_incoming_metadata_buffer_init( - grpc_chttp2_incoming_metadata_buffer *buffer); + grpc_chttp2_incoming_metadata_buffer *buffer, gpr_arena *arena); void grpc_chttp2_incoming_metadata_buffer_destroy( grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_metadata_buffer *buffer); void grpc_chttp2_incoming_metadata_buffer_publish( grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_metadata_buffer *buffer, grpc_metadata_batch *batch); -void grpc_chttp2_incoming_metadata_buffer_add( - grpc_chttp2_incoming_metadata_buffer *buffer, grpc_mdelem elem); -void grpc_chttp2_incoming_metadata_buffer_replace_or_add( +grpc_error *grpc_chttp2_incoming_metadata_buffer_add( grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_metadata_buffer *buffer, - grpc_mdelem elem); + grpc_mdelem elem) GRPC_MUST_USE_RESULT; +grpc_error *grpc_chttp2_incoming_metadata_buffer_replace_or_add( + grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_metadata_buffer *buffer, + grpc_mdelem elem) GRPC_MUST_USE_RESULT; void grpc_chttp2_incoming_metadata_buffer_set_deadline( grpc_chttp2_incoming_metadata_buffer *buffer, gpr_timespec deadline); diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index e7f2597f89c..7efc8c63c91 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -548,7 +548,14 @@ static void on_initial_header(grpc_exec_ctx *exec_ctx, void *tp, s->seen_error = true; GRPC_MDELEM_UNREF(exec_ctx, md); } else { - grpc_chttp2_incoming_metadata_buffer_add(&s->metadata_buffer[0], md); + grpc_error *error = grpc_chttp2_incoming_metadata_buffer_add( + exec_ctx, &s->metadata_buffer[0], md); + if (error != GRPC_ERROR_NONE) { + grpc_chttp2_cancel_stream(exec_ctx, t, s, error); + grpc_chttp2_parsing_become_skip_parser(exec_ctx, t); + s->seen_error = true; + GRPC_MDELEM_UNREF(exec_ctx, md); + } } } @@ -598,7 +605,14 @@ static void on_trailing_header(grpc_exec_ctx *exec_ctx, void *tp, s->seen_error = true; GRPC_MDELEM_UNREF(exec_ctx, md); } else { - grpc_chttp2_incoming_metadata_buffer_add(&s->metadata_buffer[1], md); + grpc_error *error = grpc_chttp2_incoming_metadata_buffer_add( + exec_ctx, &s->metadata_buffer[1], md); + if (error != GRPC_ERROR_NONE) { + grpc_chttp2_cancel_stream(exec_ctx, t, s, error); + grpc_chttp2_parsing_become_skip_parser(exec_ctx, t); + s->seen_error = true; + GRPC_MDELEM_UNREF(exec_ctx, md); + } } GPR_TIMER_END("on_trailing_header", 0); diff --git a/src/core/lib/channel/connected_channel.c b/src/core/lib/channel/connected_channel.c index 64cf95b5a58..42ef7b78063 100644 --- a/src/core/lib/channel/connected_channel.c +++ b/src/core/lib/channel/connected_channel.c @@ -88,7 +88,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, channel_data *chand = elem->channel_data; int r = grpc_transport_init_stream( exec_ctx, chand->transport, TRANSPORT_STREAM_FROM_CALL_DATA(calld), - &args->call_stack->refcount, args->server_transport_data); + &args->call_stack->refcount, args->server_transport_data, args->arena); return r == 0 ? GRPC_ERROR_NONE : GRPC_ERROR_CREATE("transport stream initialization failed"); } diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index 7635fc730d2..c1a364cbbf2 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -162,9 +162,9 @@ void grpc_transport_destroy(grpc_exec_ctx *exec_ctx, int grpc_transport_init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *transport, grpc_stream *stream, grpc_stream_refcount *refcount, - const void *server_data) { + const void *server_data, gpr_arena *arena) { return transport->vtable->init_stream(exec_ctx, transport, stream, refcount, - server_data); + server_data, arena); } void grpc_transport_perform_stream_op(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 11fd5d1697c..950b18aeda2 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -41,6 +41,7 @@ #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" +#include "src/core/lib/support/arena.h" #include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" @@ -229,7 +230,7 @@ size_t grpc_transport_stream_size(grpc_transport *transport); int grpc_transport_init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *transport, grpc_stream *stream, grpc_stream_refcount *refcount, - const void *server_data); + const void *server_data, gpr_arena *arena); void grpc_transport_set_pops(grpc_exec_ctx *exec_ctx, grpc_transport *transport, grpc_stream *stream, grpc_polling_entity *pollent); diff --git a/src/core/lib/transport/transport_impl.h b/src/core/lib/transport/transport_impl.h index 9b6095f6666..6f688bf8d28 100644 --- a/src/core/lib/transport/transport_impl.h +++ b/src/core/lib/transport/transport_impl.h @@ -47,7 +47,7 @@ typedef struct grpc_transport_vtable { /* implementation of grpc_transport_init_stream */ int (*init_stream)(grpc_exec_ctx *exec_ctx, grpc_transport *self, grpc_stream *stream, grpc_stream_refcount *refcount, - const void *server_data); + const void *server_data, gpr_arena *arena); /* implementation of grpc_transport_set_pollset */ void (*set_pollset)(grpc_exec_ctx *exec_ctx, grpc_transport *self, From 11c17d49a619de3b3617a78adb90a634db516bcb Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Mar 2017 13:36:34 -0700 Subject: [PATCH 135/188] Move functions around to avoid forward declarations --- src/core/ext/client_channel/client_channel.c | 84 ++++++++++---------- 1 file changed, 40 insertions(+), 44 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index a19c94f85aa..ba3f712bc01 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -677,9 +677,6 @@ typedef struct client_channel_call_data { grpc_linked_mdelem lb_token_mdelem; } call_data; -static void apply_final_configuration_locked(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem); - grpc_subchannel_call *grpc_client_channel_get_subchannel_call( grpc_call_element *call_elem) { grpc_subchannel_call *scc = GET_CALL((call_data *)call_elem->call_data); @@ -730,6 +727,46 @@ static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, call_data *calld) { gpr_free(ops); } +// Sets calld->method_params. +// If the method params specify a timeout, populates +// *per_method_deadline and returns true. +static bool set_call_method_params_from_service_config_locked( + grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + gpr_timespec *per_method_deadline) { + channel_data *chand = elem->channel_data; + call_data *calld = elem->call_data; + if (chand->method_params_table != NULL) { + calld->method_params = grpc_method_config_table_get( + exec_ctx, chand->method_params_table, calld->path); + if (calld->method_params != NULL) { + method_parameters_ref(calld->method_params); + if (gpr_time_cmp(calld->method_params->timeout, + gpr_time_0(GPR_TIMESPAN)) != 0) { + *per_method_deadline = + gpr_time_add(calld->call_start_time, calld->method_params->timeout); + return true; + } + } + } + return false; +} +static void apply_final_configuration_locked(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem) { + /* apply service-config level configuration to the call (now that we're + * certain it exists) */ + call_data *calld = elem->call_data; + gpr_timespec per_method_deadline; + if (set_call_method_params_from_service_config_locked(exec_ctx, elem, + &per_method_deadline)) { + // If the deadline from the service config is shorter than the one + // from the client API, reset the deadline timer. + if (gpr_time_cmp(per_method_deadline, calld->deadline) < 0) { + calld->deadline = per_method_deadline; + grpc_deadline_state_reset(exec_ctx, elem, calld->deadline); + } + } +} + static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_call_element *elem = arg; @@ -1066,47 +1103,6 @@ static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, GPR_TIMER_END("cc_start_transport_stream_op", 0); } -// Sets calld->method_params. -// If the method params specify a timeout, populates -// *per_method_deadline and returns true. -static bool set_call_method_params_from_service_config_locked( - grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - gpr_timespec *per_method_deadline) { - channel_data *chand = elem->channel_data; - call_data *calld = elem->call_data; - if (chand->method_params_table != NULL) { - calld->method_params = grpc_method_config_table_get( - exec_ctx, chand->method_params_table, calld->path); - if (calld->method_params != NULL) { - method_parameters_ref(calld->method_params); - if (gpr_time_cmp(calld->method_params->timeout, - gpr_time_0(GPR_TIMESPAN)) != 0) { - *per_method_deadline = - gpr_time_add(calld->call_start_time, calld->method_params->timeout); - return true; - } - } - } - return false; -} - -static void apply_final_configuration_locked(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem) { - /* apply service-config level configuration to the call (now that we're - * certain it exists) */ - call_data *calld = elem->call_data; - gpr_timespec per_method_deadline; - if (set_call_method_params_from_service_config_locked(exec_ctx, elem, - &per_method_deadline)) { - // If the deadline from the service config is shorter than the one - // from the client API, reset the deadline timer. - if (gpr_time_cmp(per_method_deadline, calld->deadline) < 0) { - calld->deadline = per_method_deadline; - grpc_deadline_state_reset(exec_ctx, elem, calld->deadline); - } - } -} - /* Constructor for call_data */ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, From ea4a4f1f792741cc051e7cec130e8ffde747db76 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Mar 2017 13:36:52 -0700 Subject: [PATCH 136/188] Fix formatting --- src/core/ext/client_channel/client_channel.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index ba3f712bc01..d1ce186fec0 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -750,6 +750,7 @@ static bool set_call_method_params_from_service_config_locked( } return false; } + static void apply_final_configuration_locked(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { /* apply service-config level configuration to the call (now that we're From ca294650ba9613044c89635b91d6d5ed603d14c5 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 13 Mar 2017 14:50:34 -0700 Subject: [PATCH 137/188] Change macros --- src/core/lib/iomgr/tcp_server_posix.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index 7cc44fabe7e..61ffb02a716 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -601,7 +601,9 @@ static grpc_error *add_all_local_addrs_to_server(grpc_tcp_server *s, unsigned port_index, int requested_port, int *out_port) { -#ifdef GRPC_HAVE_IFADDRS +#ifndef GRPC_HAVE_IFADDRS + return GRPC_ERROR_CREATE("no ifaddrs available"); +#else /* ifndef GRPC_HAVE_IFADDRS */ struct ifaddrs *ifa = NULL; struct ifaddrs *ifa_it; unsigned fd_index = 0; @@ -689,9 +691,7 @@ static grpc_error *add_all_local_addrs_to_server(grpc_tcp_server *s, *out_port = sp->port; return GRPC_ERROR_NONE; } -#else /* GRPC_HAVE_IFADDRS */ - return GRPC_ERROR_NONE; -#endif /* GRPC_HAVE_IFADDRS */ +#endif /* ifndef GRPC_HAVE_IFADDRS */ } /* Treat :: or 0.0.0.0 as a family-agnostic wildcard. */ @@ -708,10 +708,14 @@ static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s, grpc_error *v6_err = GRPC_ERROR_NONE; grpc_error *v4_err = GRPC_ERROR_NONE; *out_port = -1; + +#ifdef GRPC_HAVE_IFADDRS if (s->expand_wildcard_addrs) { return add_all_local_addrs_to_server(s, port_index, requested_port, out_port); } +#endif /* GRPC_HAVE_IFADDRS */ + grpc_sockaddr_make_wildcards(requested_port, &wild4, &wild6); /* Try listening on IPv6 first. */ if ((v6_err = add_addr_to_server(s, &wild6, port_index, fd_index, &dsmode, From 547cbb229e3630d9b500fd576c2d6a7ad463ba10 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 13 Mar 2017 11:33:05 -0700 Subject: [PATCH 138/188] Code health and comments --- src/core/lib/iomgr/error.c | 38 ++++++++++++++++------------- src/core/lib/iomgr/error_internal.h | 15 ++++++++++-- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index f0e294b6091..e031945f9fd 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -140,14 +140,15 @@ grpc_error *grpc_error_ref(grpc_error *err, const char *file, int line, const char *func) { if (grpc_error_is_special(err)) return err; gpr_log(GPR_DEBUG, "%p: %" PRIdPTR " -> %" PRIdPTR " [%s:%d %s]", err, - err->refs.count, err->refs.count + 1, file, line, func); - gpr_ref(&err->refs); + err->atomics.refs.count, err->atomics.refs.count + 1, file, line, + func); + gpr_ref(&err->atomics.refs); return err; } #else grpc_error *grpc_error_ref(grpc_error *err) { if (grpc_error_is_special(err)) return err; - gpr_ref(&err->refs); + gpr_ref(&err->atomics.refs); return err; } #endif @@ -182,7 +183,7 @@ static void error_destroy(grpc_error *err) { GPR_ASSERT(!grpc_error_is_special(err)); unref_errs(err); unref_strs(err); - gpr_free((void *)gpr_atm_acq_load(&err->error_string)); + gpr_free((void *)gpr_atm_acq_load(&err->atomics.error_string)); gpr_free(err); } @@ -191,15 +192,16 @@ void grpc_error_unref(grpc_error *err, const char *file, int line, const char *func) { if (grpc_error_is_special(err)) return; gpr_log(GPR_DEBUG, "%p: %" PRIdPTR " -> %" PRIdPTR " [%s:%d %s]", err, - err->refs.count, err->refs.count - 1, file, line, func); - if (gpr_unref(&err->refs)) { + err->atomics.refs.count, err->atomics.refs.count - 1, file, line, + func); + if (gpr_unref(&err->atomics.refs)) { error_destroy(err); } } #else void grpc_error_unref(grpc_error *err) { if (grpc_error_is_special(err)) return; - if (gpr_unref(&err->refs)) { + if (gpr_unref(&err->atomics.refs)) { error_destroy(err); } } @@ -328,8 +330,8 @@ grpc_error *grpc_error_create(const char *file, int line, const char *desc, internal_set_time(&err, GRPC_ERROR_TIME_CREATED, gpr_now(GPR_CLOCK_REALTIME)); - gpr_atm_no_barrier_store(&err->error_string, 0); - gpr_ref_init(&err->refs, 1); + gpr_atm_no_barrier_store(&err->atomics.error_string, 0); + gpr_ref_init(&err->atomics.refs, 1); GPR_TIMER_END("grpc_error_create", 0); return err; } @@ -369,7 +371,7 @@ static grpc_error *copy_error_and_unref(grpc_error *in) { grpc_slice_from_static_string("cancelled")); internal_set_int(&out, GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_CANCELLED); } - } else if (gpr_ref_is_unique(&in->refs)) { + } else if (gpr_ref_is_unique(&in->atomics.refs)) { out = in; } else { uint8_t new_arena_capacity = in->arena_capacity; @@ -382,10 +384,12 @@ static grpc_error *copy_error_and_unref(grpc_error *in) { #ifdef GRPC_ERROR_REFCOUNT_DEBUG gpr_log(GPR_DEBUG, "%p create copying %p", out, in); #endif - gpr_ref_init(&out->refs, 1); - gpr_atm_no_barrier_store(&out->error_string, 0); - size_t skip = sizeof(gpr_refcount) + sizeof(gpr_atm); - memcpy((void *)out + skip, (void *)in + skip, + // manually set the atomics of the error. + gpr_atm_no_barrier_store(&out->atomics.error_string, 0); + gpr_ref_init(&out->atomics.refs, 1); + // bulk memcpy of the rest of the struct. + size_t skip = sizeof(&out->atomics); + memcpy((void *)((uintptr_t)out + skip), (void *)((uintptr_t)in + skip), sizeof(*in) + (in->arena_size * sizeof(intptr_t)) - skip); out->arena_capacity = new_arena_capacity; ref_strs(out); @@ -694,7 +698,7 @@ const char *grpc_error_string(grpc_error *err) { if (err == GRPC_ERROR_OOM) return oom_error_string; if (err == GRPC_ERROR_CANCELLED) return cancelled_error_string; - void *p = (void *)gpr_atm_acq_load(&err->error_string); + void *p = (void *)gpr_atm_acq_load(&err->atomics.error_string); if (p != NULL) { GPR_TIMER_END("grpc_error_string", 0); return p; @@ -714,9 +718,9 @@ const char *grpc_error_string(grpc_error *err) { char *out = finish_kvs(&kvs); - if (!gpr_atm_rel_cas(&err->error_string, 0, (gpr_atm)out)) { + if (!gpr_atm_rel_cas(&err->atomics.error_string, 0, (gpr_atm)out)) { gpr_free(out); - out = (char *)gpr_atm_no_barrier_load(&err->error_string); + out = (char *)gpr_atm_no_barrier_load(&err->atomics.error_string); } GPR_TIMER_END("grpc_error_string", 0); diff --git a/src/core/lib/iomgr/error_internal.h b/src/core/lib/iomgr/error_internal.h index 0718d31f232..7f204df1b2d 100644 --- a/src/core/lib/iomgr/error_internal.h +++ b/src/core/lib/iomgr/error_internal.h @@ -46,14 +46,25 @@ struct grpc_linked_error { uint8_t next; }; +// c core representation of an error. See error.h for high level description of +// this object. struct grpc_error { - gpr_refcount refs; - gpr_atm error_string; + // All atomics in grpc_error must be stored in this nested struct. The rest of + // the object is memcpy-ed in bulk in copy_and_unref. + struct atomics { + gpr_refcount refs; + gpr_atm error_string; + } atomics; + // These arrays index into dynamic arena at the bottom of the struct. + // UINT8_MAX is used as a sentinel value. uint8_t ints[GRPC_ERROR_INT_MAX]; uint8_t strs[GRPC_ERROR_STR_MAX]; uint8_t times[GRPC_ERROR_TIME_MAX]; + // The child errors are stored in the arena, but are effectively a linked list + // structure, since they are contained withing grpc_linked_error objects. uint8_t first_err; uint8_t last_err; + // The arena is dynamically reallocated with a grow factor of 1.5. uint8_t arena_size; uint8_t arena_capacity; intptr_t arena[0]; From b18c8ba0b6bbc2317dd92353288f2b1c0ff353d5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 13 Mar 2017 15:51:37 -0700 Subject: [PATCH 139/188] Fix cancellation --- src/core/lib/surface/call.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 8b2f9fd3f53..47f36be5fda 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -1070,8 +1070,8 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; recv_trailing_filter(exec_ctx, call, md); - gpr_atm_rel_store(&call->received_final_op_atm, 1); /* propagate cancellation to any interested children */ + gpr_atm_rel_store(&call->received_final_op_atm, 1); gpr_mu_lock(&call->child_list_mu); child_call = call->first_child; if (child_call != NULL) { @@ -1079,7 +1079,7 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, next_child_call = child_call->sibling_next; if (child_call->cancellation_is_inherited) { GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel"); - cancel_with_error(exec_ctx, call, STATUS_FROM_API_OVERRIDE, + cancel_with_error(exec_ctx, child_call, STATUS_FROM_API_OVERRIDE, GRPC_ERROR_CANCELLED); GRPC_CALL_INTERNAL_UNREF(exec_ctx, child_call, "propagate_cancel"); } From 53bd23591cc70e889dd8833e4e138dd8cd5c3b04 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 13 Mar 2017 16:12:28 -0700 Subject: [PATCH 140/188] Atomically load ref debugging output numbers --- src/core/lib/iomgr/error.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index e031945f9fd..1009194a312 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -140,7 +140,7 @@ grpc_error *grpc_error_ref(grpc_error *err, const char *file, int line, const char *func) { if (grpc_error_is_special(err)) return err; gpr_log(GPR_DEBUG, "%p: %" PRIdPTR " -> %" PRIdPTR " [%s:%d %s]", err, - err->atomics.refs.count, err->atomics.refs.count + 1, file, line, + gpr_atm_no_barrier_load(&err->atomics.refs.count), gpr_atm_no_barrier_load(&err->atomics.refs.count) + 1, file, line, func); gpr_ref(&err->atomics.refs); return err; @@ -192,7 +192,7 @@ void grpc_error_unref(grpc_error *err, const char *file, int line, const char *func) { if (grpc_error_is_special(err)) return; gpr_log(GPR_DEBUG, "%p: %" PRIdPTR " -> %" PRIdPTR " [%s:%d %s]", err, - err->atomics.refs.count, err->atomics.refs.count - 1, file, line, + gpr_atm_no_barrier_load(&err->atomics.refs.count), gpr_atm_no_barrier_load(&err->atomics.refs.count) - 1, file, line, func); if (gpr_unref(&err->atomics.refs)) { error_destroy(err); @@ -384,13 +384,13 @@ static grpc_error *copy_error_and_unref(grpc_error *in) { #ifdef GRPC_ERROR_REFCOUNT_DEBUG gpr_log(GPR_DEBUG, "%p create copying %p", out, in); #endif - // manually set the atomics of the error. - gpr_atm_no_barrier_store(&out->atomics.error_string, 0); - gpr_ref_init(&out->atomics.refs, 1); // bulk memcpy of the rest of the struct. size_t skip = sizeof(&out->atomics); memcpy((void *)((uintptr_t)out + skip), (void *)((uintptr_t)in + skip), sizeof(*in) + (in->arena_size * sizeof(intptr_t)) - skip); + // manually set the atomics and the new capacity + gpr_atm_no_barrier_store(&out->atomics.error_string, 0); + gpr_ref_init(&out->atomics.refs, 1); out->arena_capacity = new_arena_capacity; ref_strs(out); ref_errs(out); From 47155ed254e7d798cd42797eea1533d5264c565c Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 13 Mar 2017 17:18:07 -0700 Subject: [PATCH 141/188] Split ifaddr related functions out into a separate file --- BUILD | 3 + CMakeLists.txt | 10 + Makefile | 10 + binding.gyp | 2 + build.yaml | 3 + config.m4 | 2 + gRPC-Core.podspec | 4 + grpc.gemspec | 3 + package.xml | 3 + src/core/lib/iomgr/tcp_server_posix.c | 192 ++---------------- src/core/lib/iomgr/tcp_server_utils_posix.h | 89 ++++++++ .../iomgr/tcp_server_utils_posix_ifaddrs.c | 178 ++++++++++++++++ .../iomgr/tcp_server_utils_posix_noifaddrs.c | 49 +++++ src/python/grpcio/grpc_core_dependencies.py | 2 + tools/doxygen/Doxyfile.core.internal | 3 + .../generated/sources_and_headers.json | 4 + vsprojects/vcxproj/grpc/grpc.vcxproj | 5 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 9 + .../grpc_test_util/grpc_test_util.vcxproj | 5 + .../grpc_test_util.vcxproj.filters | 9 + .../grpc_unsecure/grpc_unsecure.vcxproj | 5 + .../grpc_unsecure.vcxproj.filters | 9 + 22 files changed, 423 insertions(+), 176 deletions(-) create mode 100644 src/core/lib/iomgr/tcp_server_utils_posix.h create mode 100644 src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c create mode 100644 src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c diff --git a/BUILD b/BUILD index ca0a1c56076..dc6431484e3 100644 --- a/BUILD +++ b/BUILD @@ -481,6 +481,8 @@ grpc_cc_library( "src/core/lib/iomgr/tcp_client_windows.c", "src/core/lib/iomgr/tcp_posix.c", "src/core/lib/iomgr/tcp_server_posix.c", + "src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c", + "src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c", "src/core/lib/iomgr/tcp_server_uv.c", "src/core/lib/iomgr/tcp_server_windows.c", "src/core/lib/iomgr/tcp_uv.c", @@ -599,6 +601,7 @@ grpc_cc_library( "src/core/lib/iomgr/tcp_client_posix.h", "src/core/lib/iomgr/tcp_posix.h", "src/core/lib/iomgr/tcp_server.h", + "src/core/lib/iomgr/tcp_server_utils_posix.h", "src/core/lib/iomgr/tcp_uv.h", "src/core/lib/iomgr/tcp_windows.h", "src/core/lib/iomgr/time_averaged_stats.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index d0a65b44932..cfc4555834a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -914,6 +914,8 @@ add_library(grpc src/core/lib/iomgr/tcp_client_windows.c src/core/lib/iomgr/tcp_posix.c src/core/lib/iomgr/tcp_server_posix.c + src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c + src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c src/core/lib/iomgr/tcp_server_uv.c src/core/lib/iomgr/tcp_server_windows.c src/core/lib/iomgr/tcp_uv.c @@ -1223,6 +1225,8 @@ add_library(grpc_cronet src/core/lib/iomgr/tcp_client_windows.c src/core/lib/iomgr/tcp_posix.c src/core/lib/iomgr/tcp_server_posix.c + src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c + src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c src/core/lib/iomgr/tcp_server_uv.c src/core/lib/iomgr/tcp_server_windows.c src/core/lib/iomgr/tcp_uv.c @@ -1523,6 +1527,8 @@ add_library(grpc_test_util src/core/lib/iomgr/tcp_client_windows.c src/core/lib/iomgr/tcp_posix.c src/core/lib/iomgr/tcp_server_posix.c + src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c + src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c src/core/lib/iomgr/tcp_server_uv.c src/core/lib/iomgr/tcp_server_windows.c src/core/lib/iomgr/tcp_uv.c @@ -1769,6 +1775,8 @@ add_library(grpc_unsecure src/core/lib/iomgr/tcp_client_windows.c src/core/lib/iomgr/tcp_posix.c src/core/lib/iomgr/tcp_server_posix.c + src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c + src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c src/core/lib/iomgr/tcp_server_uv.c src/core/lib/iomgr/tcp_server_windows.c src/core/lib/iomgr/tcp_uv.c @@ -2376,6 +2384,8 @@ add_library(grpc++_cronet src/core/lib/iomgr/tcp_client_windows.c src/core/lib/iomgr/tcp_posix.c src/core/lib/iomgr/tcp_server_posix.c + src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c + src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c src/core/lib/iomgr/tcp_server_uv.c src/core/lib/iomgr/tcp_server_windows.c src/core/lib/iomgr/tcp_uv.c diff --git a/Makefile b/Makefile index a9242dddbdd..14c5a36c957 100644 --- a/Makefile +++ b/Makefile @@ -2799,6 +2799,8 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ + src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ src/core/lib/iomgr/tcp_server_windows.c \ src/core/lib/iomgr/tcp_uv.c \ @@ -3111,6 +3113,8 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ + src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ src/core/lib/iomgr/tcp_server_windows.c \ src/core/lib/iomgr/tcp_uv.c \ @@ -3414,6 +3418,8 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ + src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ src/core/lib/iomgr/tcp_server_windows.c \ src/core/lib/iomgr/tcp_uv.c \ @@ -3640,6 +3646,8 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ + src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ src/core/lib/iomgr/tcp_server_windows.c \ src/core/lib/iomgr/tcp_uv.c \ @@ -4249,6 +4257,8 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ + src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ src/core/lib/iomgr/tcp_server_windows.c \ src/core/lib/iomgr/tcp_uv.c \ diff --git a/binding.gyp b/binding.gyp index c521a27c300..ec21a97806f 100644 --- a/binding.gyp +++ b/binding.gyp @@ -667,6 +667,8 @@ 'src/core/lib/iomgr/tcp_client_windows.c', 'src/core/lib/iomgr/tcp_posix.c', 'src/core/lib/iomgr/tcp_server_posix.c', + 'src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c', + 'src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c', 'src/core/lib/iomgr/tcp_server_uv.c', 'src/core/lib/iomgr/tcp_server_windows.c', 'src/core/lib/iomgr/tcp_uv.c', diff --git a/build.yaml b/build.yaml index a32a8a06d34..36817a55067 100644 --- a/build.yaml +++ b/build.yaml @@ -228,6 +228,7 @@ filegroups: - src/core/lib/iomgr/tcp_client_posix.h - src/core/lib/iomgr/tcp_posix.h - src/core/lib/iomgr/tcp_server.h + - src/core/lib/iomgr/tcp_server_utils_posix.h - src/core/lib/iomgr/tcp_uv.h - src/core/lib/iomgr/tcp_windows.h - src/core/lib/iomgr/time_averaged_stats.h @@ -337,6 +338,8 @@ filegroups: - src/core/lib/iomgr/tcp_client_windows.c - src/core/lib/iomgr/tcp_posix.c - src/core/lib/iomgr/tcp_server_posix.c + - src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c + - src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c - src/core/lib/iomgr/tcp_server_uv.c - src/core/lib/iomgr/tcp_server_windows.c - src/core/lib/iomgr/tcp_uv.c diff --git a/config.m4 b/config.m4 index 90536e503ed..181e44c52c8 100644 --- a/config.m4 +++ b/config.m4 @@ -140,6 +140,8 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ + src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ src/core/lib/iomgr/tcp_server_windows.c \ src/core/lib/iomgr/tcp_uv.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 027babcda46..1d002247d4b 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -309,6 +309,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/tcp_client_posix.h', 'src/core/lib/iomgr/tcp_posix.h', 'src/core/lib/iomgr/tcp_server.h', + 'src/core/lib/iomgr/tcp_server_utils_posix.h', 'src/core/lib/iomgr/tcp_uv.h', 'src/core/lib/iomgr/tcp_windows.h', 'src/core/lib/iomgr/time_averaged_stats.h', @@ -508,6 +509,8 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/tcp_client_windows.c', 'src/core/lib/iomgr/tcp_posix.c', 'src/core/lib/iomgr/tcp_server_posix.c', + 'src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c', + 'src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c', 'src/core/lib/iomgr/tcp_server_uv.c', 'src/core/lib/iomgr/tcp_server_windows.c', 'src/core/lib/iomgr/tcp_uv.c', @@ -744,6 +747,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/tcp_client_posix.h', 'src/core/lib/iomgr/tcp_posix.h', 'src/core/lib/iomgr/tcp_server.h', + 'src/core/lib/iomgr/tcp_server_utils_posix.h', 'src/core/lib/iomgr/tcp_uv.h', 'src/core/lib/iomgr/tcp_windows.h', 'src/core/lib/iomgr/time_averaged_stats.h', diff --git a/grpc.gemspec b/grpc.gemspec index 8d5b7b2ab1c..edc08cb9b9a 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -226,6 +226,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/tcp_client_posix.h ) s.files += %w( src/core/lib/iomgr/tcp_posix.h ) s.files += %w( src/core/lib/iomgr/tcp_server.h ) + s.files += %w( src/core/lib/iomgr/tcp_server_utils_posix.h ) s.files += %w( src/core/lib/iomgr/tcp_uv.h ) s.files += %w( src/core/lib/iomgr/tcp_windows.h ) s.files += %w( src/core/lib/iomgr/time_averaged_stats.h ) @@ -425,6 +426,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/tcp_client_windows.c ) s.files += %w( src/core/lib/iomgr/tcp_posix.c ) s.files += %w( src/core/lib/iomgr/tcp_server_posix.c ) + s.files += %w( src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c ) + s.files += %w( src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c ) s.files += %w( src/core/lib/iomgr/tcp_server_uv.c ) s.files += %w( src/core/lib/iomgr/tcp_server_windows.c ) s.files += %w( src/core/lib/iomgr/tcp_uv.c ) diff --git a/package.xml b/package.xml index 4167bef26e3..21eeee4d3ef 100644 --- a/package.xml +++ b/package.xml @@ -235,6 +235,7 @@ + @@ -434,6 +435,8 @@ + + diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index 61ffb02a716..1e9e854d6b5 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -42,10 +42,6 @@ #include "src/core/lib/iomgr/tcp_server.h" -#ifdef GRPC_HAVE_IFADDRS -#include -#endif /* GRPC_HAVE_IFADDRS */ - #include #include #include @@ -70,6 +66,7 @@ #include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/iomgr/socket_utils_posix.h" #include "src/core/lib/iomgr/tcp_posix.h" +#include "src/core/lib/iomgr/tcp_server_utils_posix.h" #include "src/core/lib/iomgr/unix_sockets_posix.h" #include "src/core/lib/support/string.h" @@ -78,29 +75,6 @@ static gpr_once s_init_max_accept_queue_size; static int s_max_accept_queue_size; -/* one listening port */ -typedef struct grpc_tcp_listener grpc_tcp_listener; -struct grpc_tcp_listener { - int fd; - grpc_fd *emfd; - grpc_tcp_server *server; - grpc_resolved_address addr; - int port; - unsigned port_index; - unsigned fd_index; - grpc_closure read_closure; - grpc_closure destroyed_closure; - struct grpc_tcp_listener *next; - /* sibling is a linked list of all listeners for a given port. add_port and - clone_port place all new listeners in the same sibling list. A member of - the 'sibling' list is also a member of the 'next' list. The head of each - sibling list has is_sibling==0, and subsequent members of sibling lists - have is_sibling==1. is_sibling allows separate sibling lists to be - identified while iterating through 'next'. */ - struct grpc_tcp_listener *sibling; - int is_sibling; -}; - /* the overall server */ struct grpc_tcp_server { gpr_refcount refs; @@ -526,11 +500,11 @@ static grpc_error *add_socket_to_server(grpc_tcp_server *s, int fd, /* If successful, add a listener to s for addr, set *dsmode for the socket, and return the *listener. */ -static grpc_error *add_addr_to_server(grpc_tcp_server *s, - const grpc_resolved_address *addr, - unsigned port_index, unsigned fd_index, - grpc_dualstack_mode *dsmode, - grpc_tcp_listener **listener) { +grpc_error *grpc_tcp_server_add_addr(grpc_tcp_server *s, + const grpc_resolved_address *addr, + unsigned port_index, unsigned fd_index, + grpc_dualstack_mode *dsmode, + grpc_tcp_listener **listener) { grpc_resolved_address addr4_copy; int fd; grpc_error *err = @@ -545,39 +519,9 @@ static grpc_error *add_addr_to_server(grpc_tcp_server *s, return add_socket_to_server(s, fd, addr, port_index, fd_index, listener); } -/* Bind to "::" to get a port number not used by any address. */ -static grpc_error *get_unused_port(int *port) { - grpc_resolved_address wild; - grpc_sockaddr_make_wildcard6(0, &wild); - grpc_dualstack_mode dsmode; - int fd; - grpc_error *err = - grpc_create_dualstack_socket(&wild, SOCK_STREAM, 0, &dsmode, &fd); - if (err != GRPC_ERROR_NONE) { - return err; - } - if (dsmode == GRPC_DSMODE_IPV4) { - grpc_sockaddr_make_wildcard4(0, &wild); - } - if (bind(fd, (const struct sockaddr *)wild.addr, (socklen_t)wild.len) != 0) { - err = GRPC_OS_ERROR(errno, "bind"); - close(fd); - return err; - } - if (getsockname(fd, (struct sockaddr *)wild.addr, (socklen_t *)&wild.len) != - 0) { - err = GRPC_OS_ERROR(errno, "getsockname"); - close(fd); - return err; - } - close(fd); - *port = grpc_sockaddr_get_port(&wild); - return *port <= 0 ? GRPC_ERROR_CREATE("Bad port") : GRPC_ERROR_NONE; -} - /* Return the listener in s with address addr or NULL. */ -static grpc_tcp_listener *find_listener_with_addr(grpc_tcp_server *s, - grpc_resolved_address *addr) { +grpc_tcp_listener *grpc_tcp_server_find_listener_with_addr( + grpc_tcp_server *s, grpc_resolved_address *addr) { grpc_tcp_listener *l; gpr_mu_lock(&s->mu); for (l = s->head; l != NULL; l = l->next) { @@ -592,108 +536,6 @@ static grpc_tcp_listener *find_listener_with_addr(grpc_tcp_server *s, return l; } -/* Get all addresses assigned to network interfaces on the machine and create a - listener for each. requested_port is the port to use for every listener, or 0 - to select one random port that will be used for every listener. Set *out_port - to the port selected. Return GRPC_ERROR_NONE only if all listeners were - added. */ -static grpc_error *add_all_local_addrs_to_server(grpc_tcp_server *s, - unsigned port_index, - int requested_port, - int *out_port) { -#ifndef GRPC_HAVE_IFADDRS - return GRPC_ERROR_CREATE("no ifaddrs available"); -#else /* ifndef GRPC_HAVE_IFADDRS */ - struct ifaddrs *ifa = NULL; - struct ifaddrs *ifa_it; - unsigned fd_index = 0; - grpc_tcp_listener *sp = NULL; - grpc_error *err = GRPC_ERROR_NONE; - if (requested_port == 0) { - /* Note: There could be a race where some local addrs can listen on the - selected port and some can't. The sane way to handle this would be to - retry by recreating the whole grpc_tcp_server. Backing out individual - listeners and orphaning the FDs looks like too much trouble. */ - if ((err = get_unused_port(&requested_port)) != GRPC_ERROR_NONE) { - return err; - } else if (requested_port <= 0) { - return GRPC_ERROR_CREATE("Bad get_unused_port()"); - } - gpr_log(GPR_DEBUG, "Picked unused port %d", requested_port); - } - if (getifaddrs(&ifa) != 0 || ifa == NULL) { - return GRPC_OS_ERROR(errno, "getifaddrs"); - } - for (ifa_it = ifa; ifa_it != NULL; ifa_it = ifa_it->ifa_next) { - grpc_resolved_address addr; - char *addr_str = NULL; - grpc_dualstack_mode dsmode; - grpc_tcp_listener *new_sp = NULL; - const char *ifa_name = (ifa_it->ifa_name ? ifa_it->ifa_name : ""); - if (ifa_it->ifa_addr == NULL) { - continue; - } else if (ifa_it->ifa_addr->sa_family == AF_INET) { - addr.len = sizeof(struct sockaddr_in); - } else if (ifa_it->ifa_addr->sa_family == AF_INET6) { - addr.len = sizeof(struct sockaddr_in6); - } else { - continue; - } - memcpy(addr.addr, ifa_it->ifa_addr, addr.len); - if (!grpc_sockaddr_set_port(&addr, requested_port)) { - /* Should never happen, because we check sa_family above. */ - err = GRPC_ERROR_CREATE("Failed to set port"); - break; - } - if (grpc_sockaddr_to_string(&addr_str, &addr, 0) < 0) { - addr_str = gpr_strdup(""); - } - gpr_log(GPR_DEBUG, - "Adding local addr from interface %s flags 0x%x to server: %s", - ifa_name, ifa_it->ifa_flags, addr_str); - /* We could have multiple interfaces with the same address (e.g., bonding), - so look for duplicates. */ - if (find_listener_with_addr(s, &addr) != NULL) { - gpr_log(GPR_DEBUG, "Skipping duplicate addr %s on interface %s", addr_str, - ifa_name); - gpr_free(addr_str); - continue; - } - if ((err = add_addr_to_server(s, &addr, port_index, fd_index, &dsmode, - &new_sp)) != GRPC_ERROR_NONE) { - char *err_str = NULL; - grpc_error *root_err; - if (gpr_asprintf(&err_str, "Failed to add listener: %s", addr_str) < 0) { - err_str = gpr_strdup("Failed to add listener"); - } - root_err = GRPC_ERROR_CREATE(err_str); - gpr_free(err_str); - gpr_free(addr_str); - err = grpc_error_add_child(root_err, err); - break; - } else { - GPR_ASSERT(requested_port == new_sp->port); - ++fd_index; - if (sp != NULL) { - new_sp->is_sibling = 1; - sp->sibling = new_sp; - } - sp = new_sp; - } - gpr_free(addr_str); - } - freeifaddrs(ifa); - if (err != GRPC_ERROR_NONE) { - return err; - } else if (sp == NULL) { - return GRPC_ERROR_CREATE("No local addresses"); - } else { - *out_port = sp->port; - return GRPC_ERROR_NONE; - } -#endif /* ifndef GRPC_HAVE_IFADDRS */ -} - /* Treat :: or 0.0.0.0 as a family-agnostic wildcard. */ static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s, unsigned port_index, @@ -709,17 +551,15 @@ static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s, grpc_error *v4_err = GRPC_ERROR_NONE; *out_port = -1; -#ifdef GRPC_HAVE_IFADDRS - if (s->expand_wildcard_addrs) { - return add_all_local_addrs_to_server(s, port_index, requested_port, - out_port); + if (grpc_tcp_server_have_ifaddrs() && s->expand_wildcard_addrs) { + return grpc_tcp_server_add_all_local_addrs(s, port_index, requested_port, + out_port); } -#endif /* GRPC_HAVE_IFADDRS */ grpc_sockaddr_make_wildcards(requested_port, &wild4, &wild6); /* Try listening on IPv6 first. */ - if ((v6_err = add_addr_to_server(s, &wild6, port_index, fd_index, &dsmode, - &sp)) == GRPC_ERROR_NONE) { + if ((v6_err = grpc_tcp_server_add_addr(s, &wild6, port_index, fd_index, + &dsmode, &sp)) == GRPC_ERROR_NONE) { ++fd_index; requested_port = *out_port = sp->port; if (dsmode == GRPC_DSMODE_DUALSTACK || dsmode == GRPC_DSMODE_IPV4) { @@ -728,8 +568,8 @@ static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s, } /* If we got a v6-only socket or nothing, try adding 0.0.0.0. */ grpc_sockaddr_set_port(&wild4, requested_port); - if ((v4_err = add_addr_to_server(s, &wild4, port_index, fd_index, &dsmode, - &sp2)) == GRPC_ERROR_NONE) { + if ((v4_err = grpc_tcp_server_add_addr(s, &wild4, port_index, fd_index, + &dsmode, &sp2)) == GRPC_ERROR_NONE) { *out_port = sp2->port; if (sp != NULL) { sp2->is_sibling = 1; @@ -839,7 +679,7 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s, if (grpc_sockaddr_to_v4mapped(addr, &addr6_v4mapped)) { addr = &addr6_v4mapped; } - if ((err = add_addr_to_server(s, addr, port_index, 0, &dsmode, &sp)) == + if ((err = grpc_tcp_server_add_addr(s, addr, port_index, 0, &dsmode, &sp)) == GRPC_ERROR_NONE) { *out_port = sp->port; } diff --git a/src/core/lib/iomgr/tcp_server_utils_posix.h b/src/core/lib/iomgr/tcp_server_utils_posix.h new file mode 100644 index 00000000000..d39f9085fda --- /dev/null +++ b/src/core/lib/iomgr/tcp_server_utils_posix.h @@ -0,0 +1,89 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_TCP_SERVER_UTILS_POSIX_H +#define GRPC_CORE_LIB_IOMGR_TCP_SERVER_UTILS_POSIX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/resolve_address.h" +#include "src/core/lib/iomgr/socket_utils_posix.h" +#include "src/core/lib/iomgr/tcp_server.h" + +/* one listening port */ +typedef struct grpc_tcp_listener { + int fd; + grpc_fd *emfd; + grpc_tcp_server *server; + grpc_resolved_address addr; + int port; + unsigned port_index; + unsigned fd_index; + grpc_closure read_closure; + grpc_closure destroyed_closure; + struct grpc_tcp_listener *next; + /* sibling is a linked list of all listeners for a given port. add_port and + clone_port place all new listeners in the same sibling list. A member of + the 'sibling' list is also a member of the 'next' list. The head of each + sibling list has is_sibling==0, and subsequent members of sibling lists + have is_sibling==1. is_sibling allows separate sibling lists to be + identified while iterating through 'next'. */ + struct grpc_tcp_listener *sibling; + int is_sibling; +} grpc_tcp_listener; + +/* If successful, add a listener to \a s for \a addr, set \a dsmode for the + socket, and return the \a listener. */ +grpc_error *grpc_tcp_server_add_addr(grpc_tcp_server *s, + const grpc_resolved_address *addr, + unsigned port_index, unsigned fd_index, + grpc_dualstack_mode *dsmode, + grpc_tcp_listener **listener); + +/* Return the listener in \a s with address \a addr or NULL. */ +grpc_tcp_listener *grpc_tcp_server_find_listener_with_addr( + grpc_tcp_server *s, grpc_resolved_address *addr); + +/* Get all addresses assigned to network interfaces on the machine and create a + listener for each. requested_port is the port to use for every listener, or 0 + to select one random port that will be used for every listener. Set *out_port + to the port selected. Return GRPC_ERROR_NONE only if all listeners were + added. */ +grpc_error *grpc_tcp_server_add_all_local_addrs(grpc_tcp_server *s, + unsigned port_index, + int requested_port, + int *out_port); + +/* Ruturn true if the platform supports ifaddrs */ +bool grpc_tcp_server_have_ifaddrs(void); + +#endif /* GRPC_CORE_LIB_IOMGR_TCP_SERVER_UTILS_POSIX_H */ diff --git a/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c b/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c new file mode 100644 index 00000000000..05eb96f73ed --- /dev/null +++ b/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c @@ -0,0 +1,178 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +#ifdef GRPC_HAVE_IFADDRS + +#include "src/core/lib/iomgr/tcp_server_utils_posix.h" + +#include +#include +#include +#include + +#include +#include +#include + +#include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/sockaddr.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" + +/* Bind to "::" to get a port number not used by any address. */ +static grpc_error *get_unused_port(int *port) { + grpc_resolved_address wild; + grpc_sockaddr_make_wildcard6(0, &wild); + grpc_dualstack_mode dsmode; + int fd; + grpc_error *err = + grpc_create_dualstack_socket(&wild, SOCK_STREAM, 0, &dsmode, &fd); + if (err != GRPC_ERROR_NONE) { + return err; + } + if (dsmode == GRPC_DSMODE_IPV4) { + grpc_sockaddr_make_wildcard4(0, &wild); + } + if (bind(fd, (const struct sockaddr *)wild.addr, (socklen_t)wild.len) != 0) { + err = GRPC_OS_ERROR(errno, "bind"); + close(fd); + return err; + } + if (getsockname(fd, (struct sockaddr *)wild.addr, (socklen_t *)&wild.len) != + 0) { + err = GRPC_OS_ERROR(errno, "getsockname"); + close(fd); + return err; + } + close(fd); + *port = grpc_sockaddr_get_port(&wild); + return *port <= 0 ? GRPC_ERROR_CREATE("Bad port") : GRPC_ERROR_NONE; +} + +grpc_error *grpc_tcp_server_add_all_local_addrs(grpc_tcp_server *s, + unsigned port_index, + int requested_port, + int *out_port) { + struct ifaddrs *ifa = NULL; + struct ifaddrs *ifa_it; + unsigned fd_index = 0; + grpc_tcp_listener *sp = NULL; + grpc_error *err = GRPC_ERROR_NONE; + if (requested_port == 0) { + /* Note: There could be a race where some local addrs can listen on the + selected port and some can't. The sane way to handle this would be to + retry by recreating the whole grpc_tcp_server. Backing out individual + listeners and orphaning the FDs looks like too much trouble. */ + if ((err = get_unused_port(&requested_port)) != GRPC_ERROR_NONE) { + return err; + } else if (requested_port <= 0) { + return GRPC_ERROR_CREATE("Bad get_unused_port()"); + } + gpr_log(GPR_DEBUG, "Picked unused port %d", requested_port); + } + if (getifaddrs(&ifa) != 0 || ifa == NULL) { + return GRPC_OS_ERROR(errno, "getifaddrs"); + } + for (ifa_it = ifa; ifa_it != NULL; ifa_it = ifa_it->ifa_next) { + grpc_resolved_address addr; + char *addr_str = NULL; + grpc_dualstack_mode dsmode; + grpc_tcp_listener *new_sp = NULL; + const char *ifa_name = (ifa_it->ifa_name ? ifa_it->ifa_name : ""); + if (ifa_it->ifa_addr == NULL) { + continue; + } else if (ifa_it->ifa_addr->sa_family == AF_INET) { + addr.len = sizeof(struct sockaddr_in); + } else if (ifa_it->ifa_addr->sa_family == AF_INET6) { + addr.len = sizeof(struct sockaddr_in6); + } else { + continue; + } + memcpy(addr.addr, ifa_it->ifa_addr, addr.len); + if (!grpc_sockaddr_set_port(&addr, requested_port)) { + /* Should never happen, because we check sa_family above. */ + err = GRPC_ERROR_CREATE("Failed to set port"); + break; + } + if (grpc_sockaddr_to_string(&addr_str, &addr, 0) < 0) { + addr_str = gpr_strdup(""); + } + gpr_log(GPR_DEBUG, + "Adding local addr from interface %s flags 0x%x to server: %s", + ifa_name, ifa_it->ifa_flags, addr_str); + /* We could have multiple interfaces with the same address (e.g., bonding), + so look for duplicates. */ + if (grpc_tcp_server_find_listener_with_addr(s, &addr) != NULL) { + gpr_log(GPR_DEBUG, "Skipping duplicate addr %s on interface %s", addr_str, + ifa_name); + gpr_free(addr_str); + continue; + } + if ((err = grpc_tcp_server_add_addr(s, &addr, port_index, fd_index, &dsmode, + &new_sp)) != GRPC_ERROR_NONE) { + char *err_str = NULL; + grpc_error *root_err; + if (gpr_asprintf(&err_str, "Failed to add listener: %s", addr_str) < 0) { + err_str = gpr_strdup("Failed to add listener"); + } + root_err = GRPC_ERROR_CREATE(err_str); + gpr_free(err_str); + gpr_free(addr_str); + err = grpc_error_add_child(root_err, err); + break; + } else { + GPR_ASSERT(requested_port == new_sp->port); + ++fd_index; + if (sp != NULL) { + new_sp->is_sibling = 1; + sp->sibling = new_sp; + } + sp = new_sp; + } + gpr_free(addr_str); + } + freeifaddrs(ifa); + if (err != GRPC_ERROR_NONE) { + return err; + } else if (sp == NULL) { + return GRPC_ERROR_CREATE("No local addresses"); + } else { + *out_port = sp->port; + return GRPC_ERROR_NONE; + } +} + +bool grpc_tcp_server_have_ifaddrs(void) { return true; } + +#endif /* GRPC_HAVE_IFADDRS */ diff --git a/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c b/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c new file mode 100644 index 00000000000..fb21d62d053 --- /dev/null +++ b/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c @@ -0,0 +1,49 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +#ifndef GRPC_HAVE_IFADDRS + +#include "src/core/lib/iomgr/tcp_server_utils_posix.h" + +grpc_error *grpc_tcp_server_add_all_local_addrs(grpc_tcp_server *s, + unsigned port_index, + int requested_port, + int *out_port) { + return GRPC_ERROR_CREATE("no ifaddrs available"); +} + +bool grpc_tcp_server_have_ifaddrs(void) { return false; } + +#endif /* ifndef GRPC_HAVE_IFADDRS */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index a9f20e6d2a8..00b2b8f9026 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -134,6 +134,8 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/tcp_client_windows.c', 'src/core/lib/iomgr/tcp_posix.c', 'src/core/lib/iomgr/tcp_server_posix.c', + 'src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c', + 'src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c', 'src/core/lib/iomgr/tcp_server_uv.c', 'src/core/lib/iomgr/tcp_server_windows.c', 'src/core/lib/iomgr/tcp_uv.c', diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 131a0134519..f9e4775b738 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1130,6 +1130,9 @@ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_posix.h \ src/core/lib/iomgr/tcp_server.h \ src/core/lib/iomgr/tcp_server_posix.c \ +src/core/lib/iomgr/tcp_server_utils_posix.h \ +src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ +src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ src/core/lib/iomgr/tcp_server_windows.c \ src/core/lib/iomgr/tcp_uv.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 767c2b2e368..38fe87ac463 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7456,6 +7456,7 @@ "src/core/lib/iomgr/tcp_client_posix.h", "src/core/lib/iomgr/tcp_posix.h", "src/core/lib/iomgr/tcp_server.h", + "src/core/lib/iomgr/tcp_server_utils_posix.h", "src/core/lib/iomgr/tcp_uv.h", "src/core/lib/iomgr/tcp_windows.h", "src/core/lib/iomgr/time_averaged_stats.h", @@ -7636,6 +7637,9 @@ "src/core/lib/iomgr/tcp_posix.h", "src/core/lib/iomgr/tcp_server.h", "src/core/lib/iomgr/tcp_server_posix.c", + "src/core/lib/iomgr/tcp_server_utils_posix.h", + "src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c", + "src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c", "src/core/lib/iomgr/tcp_server_uv.c", "src/core/lib/iomgr/tcp_server_windows.c", "src/core/lib/iomgr/tcp_uv.c", diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index fde60be3e20..7b48dfbf597 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -357,6 +357,7 @@ + @@ -618,6 +619,10 @@ + + + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 8edbbc22bed..8ba8a4f8fb9 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -181,6 +181,12 @@ src\core\lib\iomgr + + src\core\lib\iomgr + + + src\core\lib\iomgr + src\core\lib\iomgr @@ -944,6 +950,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index e7c9fb71f33..d078db1c3c7 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -252,6 +252,7 @@ + @@ -461,6 +462,10 @@ + + + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 3d36948aaef..9e2d817143e 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -238,6 +238,12 @@ src\core\lib\iomgr + + src\core\lib\iomgr + + + src\core\lib\iomgr + src\core\lib\iomgr @@ -728,6 +734,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 22f4740b8fa..b6f10b3d7c1 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -347,6 +347,7 @@ + @@ -585,6 +586,10 @@ + + + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 5021cb47d8d..03662d4044c 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -184,6 +184,12 @@ src\core\lib\iomgr + + src\core\lib\iomgr + + + src\core\lib\iomgr + src\core\lib\iomgr @@ -854,6 +860,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr From 20a72b8b83bcbfd2d5a3756e1cbee813d9ff76af Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 13 Mar 2017 19:10:54 -0700 Subject: [PATCH 142/188] clang fmt --- src/core/lib/iomgr/error.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index 1009194a312..1127fff756f 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -140,7 +140,8 @@ grpc_error *grpc_error_ref(grpc_error *err, const char *file, int line, const char *func) { if (grpc_error_is_special(err)) return err; gpr_log(GPR_DEBUG, "%p: %" PRIdPTR " -> %" PRIdPTR " [%s:%d %s]", err, - gpr_atm_no_barrier_load(&err->atomics.refs.count), gpr_atm_no_barrier_load(&err->atomics.refs.count) + 1, file, line, + gpr_atm_no_barrier_load(&err->atomics.refs.count), + gpr_atm_no_barrier_load(&err->atomics.refs.count) + 1, file, line, func); gpr_ref(&err->atomics.refs); return err; @@ -192,7 +193,8 @@ void grpc_error_unref(grpc_error *err, const char *file, int line, const char *func) { if (grpc_error_is_special(err)) return; gpr_log(GPR_DEBUG, "%p: %" PRIdPTR " -> %" PRIdPTR " [%s:%d %s]", err, - gpr_atm_no_barrier_load(&err->atomics.refs.count), gpr_atm_no_barrier_load(&err->atomics.refs.count) - 1, file, line, + gpr_atm_no_barrier_load(&err->atomics.refs.count), + gpr_atm_no_barrier_load(&err->atomics.refs.count) - 1, file, line, func); if (gpr_unref(&err->atomics.refs)) { error_destroy(err); From 40d55f9dac7c732e568a2d0a559d15d74f8fde6e Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Tue, 14 Mar 2017 00:01:35 +0000 Subject: [PATCH 143/188] Add import prefix to Python generator This is needed by a customer with a nontraditional codebase. --- src/compiler/python_generator.cc | 22 ++++++++++++++-------- src/compiler/python_generator.h | 3 +++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index 4841da8da8e..242ce06a16e 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -101,18 +101,20 @@ class IndentScope { // TODO(https://github.com/google/protobuf/issues/888): // Export `ModuleName` from protobuf's // `src/google/protobuf/compiler/python/python_generator.cc` file. -grpc::string ModuleName(const grpc::string& filename) { +grpc::string ModuleName(const grpc::string& filename, + const grpc::string& import_prefix) { grpc::string basename = StripProto(filename); basename = StringReplace(basename, "-", "_"); basename = StringReplace(basename, "/", "."); - return basename + "_pb2"; + return import_prefix + basename + "_pb2"; } // TODO(https://github.com/google/protobuf/issues/888): // Export `ModuleAlias` from protobuf's // `src/google/protobuf/compiler/python/python_generator.cc` file. -grpc::string ModuleAlias(const grpc::string& filename) { - grpc::string module_name = ModuleName(filename); +grpc::string ModuleAlias(const grpc::string& filename, + const grpc::string& import_prefix) { + grpc::string module_name = ModuleName(filename, import_prefix); // We can't have dots in the module name, so we replace each with _dot_. // But that could lead to a collision between a.b and a_dot_b, so we also // duplicate each underscore. @@ -189,7 +191,7 @@ bool PrivateGenerator::GetModuleAndMessagePath(const Descriptor* type, grpc::string generator_file_name = file->name(); grpc::string module; if (generator_file_name != file_name || generate_in_pb2_grpc) { - module = ModuleAlias(file_name) + "."; + module = ModuleAlias(file_name, config.import_prefix) + "."; } else { module = ""; } @@ -666,8 +668,10 @@ bool PrivateGenerator::PrintPreamble() { for (int k = 0; k < 2; ++k) { const Descriptor* type = types[k]; grpc::string type_file_name = type->file()->name(); - grpc::string module_name = ModuleName(type_file_name); - grpc::string module_alias = ModuleAlias(type_file_name); + grpc::string module_name = + ModuleName(type_file_name, config.import_prefix); + grpc::string module_alias = + ModuleAlias(type_file_name, config.import_prefix); imports_set.insert(std::make_tuple(module_name, module_alias)); } } @@ -766,7 +770,9 @@ pair PrivateGenerator::GetGrpcServices() { } // namespace GeneratorConfiguration::GeneratorConfiguration() - : grpc_package_root("grpc"), beta_package_root("grpc.beta") {} + : grpc_package_root("grpc"), + beta_package_root("grpc.beta"), + import_prefix("") {} PythonGrpcGenerator::PythonGrpcGenerator(const GeneratorConfiguration& config) : config_(config) {} diff --git a/src/compiler/python_generator.h b/src/compiler/python_generator.h index 6a95255d40e..b91059fad77 100644 --- a/src/compiler/python_generator.h +++ b/src/compiler/python_generator.h @@ -45,7 +45,10 @@ namespace grpc_python_generator { struct GeneratorConfiguration { GeneratorConfiguration(); grpc::string grpc_package_root; + // TODO(https://github.com/grpc/grpc/issues/8622): Drop this. grpc::string beta_package_root; + // TODO(https://github.com/google/protobuf/issues/888): Drop this. + grpc::string import_prefix; }; class PythonGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { From 2e0788ac76ee7bd6dcbcead1ca25d061c2c3abe7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Mar 2017 06:55:44 -0700 Subject: [PATCH 144/188] Move config application to *just* prior to subchannel pick - this is both the point where we *know* we have a config (or not), and the last moment before wait_until_ready-ness is not an issue --- src/core/ext/client_channel/client_channel.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index d1ce186fec0..f2475cf6aee 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -796,7 +796,6 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, } else { /* Create call on subchannel. */ grpc_subchannel_call *subchannel_call = NULL; - apply_final_configuration_locked(exec_ctx, elem); grpc_error *new_error = grpc_connected_subchannel_create_call( exec_ctx, calld->connected_subchannel, calld->pollent, calld->path, calld->call_start_time, calld->deadline, &subchannel_call); @@ -894,6 +893,7 @@ static bool pick_subchannel_locked( } GPR_ASSERT(error == GRPC_ERROR_NONE); if (chand->lb_policy != NULL) { + apply_final_configuration_locked(exec_ctx, elem); grpc_lb_policy *lb_policy = chand->lb_policy; GRPC_LB_POLICY_REF(lb_policy, "pick_subchannel"); // If the application explicitly set wait_for_ready, use that. @@ -1025,7 +1025,6 @@ static void start_transport_stream_op_locked_inner(grpc_exec_ctx *exec_ctx, if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && calld->connected_subchannel != NULL) { grpc_subchannel_call *subchannel_call = NULL; - apply_final_configuration_locked(exec_ctx, elem); grpc_error *error = grpc_connected_subchannel_create_call( exec_ctx, calld->connected_subchannel, calld->pollent, calld->path, calld->call_start_time, calld->deadline, &subchannel_call); From dd9b597bd7ba13b135bf0e5ae292a9ad38d7d856 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Mar 2017 07:39:45 -0700 Subject: [PATCH 145/188] UBSAN cleanup --- Makefile | 2 +- build.yaml | 6 +++--- tools/run_tests/generated/configs.json | 2 +- tools/ubsan_suppressions.txt | 7 +++++++ 4 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 tools/ubsan_suppressions.txt diff --git a/Makefile b/Makefile index 57e877beb01..addcc914b13 100644 --- a/Makefile +++ b/Makefile @@ -189,7 +189,7 @@ CC_ubsan = clang CXX_ubsan = clang++ LD_ubsan = clang LDXX_ubsan = clang++ -CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge -fsanitize=undefined,unsigned-integer-overflow -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs +CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs LDFLAGS_ubsan = -fsanitize=undefined,unsigned-integer-overflow DEFINES_ubsan = NDEBUG diff --git a/build.yaml b/build.yaml index 9ff37d59e17..321a8ac381d 100644 --- a/build.yaml +++ b/build.yaml @@ -4090,8 +4090,8 @@ configs: TSAN_OPTIONS: suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 ubsan: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=undefined,unsigned-integer-overflow - -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs + CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer + -Wno-unused-command-line-argument -Wvarargs CXX: clang++ DEFINES: NDEBUG LD: clang @@ -4099,7 +4099,7 @@ configs: LDXX: clang++ compile_the_world: true test_environ: - UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 + UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1:suppressions=tools/ubsan_suppressions.txt defaults: benchmark: CPPFLAGS: -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX diff --git a/tools/run_tests/generated/configs.json b/tools/run_tests/generated/configs.json index 9173bd7c19c..69e0f447743 100644 --- a/tools/run_tests/generated/configs.json +++ b/tools/run_tests/generated/configs.json @@ -58,7 +58,7 @@ { "config": "ubsan", "environ": { - "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1" + "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1:suppressions=tools/ubsan_suppressions.txt" } }, { diff --git a/tools/ubsan_suppressions.txt b/tools/ubsan_suppressions.txt new file mode 100644 index 00000000000..9869f98a22e --- /dev/null +++ b/tools/ubsan_suppressions.txt @@ -0,0 +1,7 @@ +# boringssl stuff +nonnull-attribute:bn_wexpand +nonnull-attribute:CBB_add_bytes +nonnull-attribute:rsa_blinding_get +nonnull-attribute:ssl_copy_key_material +alignment:CRYPTO_cbc128_encrypt + From ff23f801c5fbb67d42b70c4ba0ba28ae2a7d31ae Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Mar 2017 07:50:56 -0700 Subject: [PATCH 146/188] Fix crash --- test/cpp/microbenchmarks/bm_arena.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/cpp/microbenchmarks/bm_arena.cc b/test/cpp/microbenchmarks/bm_arena.cc index 33e073f7495..770c0b6d474 100644 --- a/test/cpp/microbenchmarks/bm_arena.cc +++ b/test/cpp/microbenchmarks/bm_arena.cc @@ -48,8 +48,15 @@ BENCHMARK(BM_Arena_NoOp)->Range(1, 1024 * 1024); static void BM_Arena_ManyAlloc(benchmark::State& state) { gpr_arena* a = gpr_arena_create(state.range(0)); + const size_t realloc_after = + 1024 * 1024 * 1024 / ((state.range(1) + 15) & 0xffffff0u); while (state.KeepRunning()) { gpr_arena_alloc(a, state.range(1)); + // periodically recreate arena to avoid OOM + if (state.iterations() % realloc_after == 0) { + gpr_arena_destroy(a); + a = gpr_arena_create(state.range(0)); + } } gpr_arena_destroy(a); } From a6bec8fc3159b54fc447a029b6260b3d6c478e6b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Mar 2017 08:26:04 -0700 Subject: [PATCH 147/188] Auto-call-arena-sizing --- src/core/lib/support/arena.c | 1 + src/core/lib/surface/call.c | 6 ++++-- src/core/lib/surface/channel.c | 32 ++++++++++++++++++++++++++++++++ src/core/lib/surface/channel.h | 3 +++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/core/lib/support/arena.c b/src/core/lib/support/arena.c index 6610acd3a07..8ce29aaab34 100644 --- a/src/core/lib/support/arena.c +++ b/src/core/lib/support/arena.c @@ -53,6 +53,7 @@ struct gpr_arena { gpr_arena *gpr_arena_create(size_t initial_size) { initial_size = ROUND_UP_TO_ALIGNMENT_SIZE(initial_size); + gpr_log(GPR_DEBUG, "arena create: %" PRIdPTR, initial_size); gpr_arena *a = gpr_zalloc(sizeof(gpr_arena) + initial_size); a->initial_zone.size_end = initial_size; return a; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 84c401db35a..00719be9abc 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -277,7 +277,8 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, grpc_channel_get_channel_stack(args->channel); grpc_call *call; GPR_TIMER_BEGIN("grpc_call_create", 0); - gpr_arena *arena = gpr_arena_create(8192); + gpr_arena *arena = + gpr_arena_create(grpc_channel_get_call_size_estimate(args->channel)); call = gpr_arena_alloc(arena, sizeof(grpc_call) + channel_stack->call_stack_size); call->arena = arena; @@ -436,8 +437,9 @@ void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c REF_ARG) { static void release_call(grpc_exec_ctx *exec_ctx, void *call, grpc_error *error) { grpc_call *c = call; + grpc_channel_update_call_size_estimate(c->channel, + gpr_arena_destroy(c->arena)); GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, c->channel, "call"); - gpr_arena_destroy(c->arena); } static void set_status_value_directly(grpc_status_code status, void *dest); diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index d6acd392c1a..56beb7d96e6 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -68,6 +68,8 @@ struct grpc_channel { grpc_compression_options compression_options; grpc_mdelem default_authority; + gpr_atm call_size_estimate; + gpr_mu registered_call_mu; registered_call *registered_calls; @@ -115,6 +117,10 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target, gpr_mu_init(&channel->registered_call_mu); channel->registered_calls = NULL; + gpr_atm_no_barrier_store( + &channel->call_size_estimate, + CHANNEL_STACK_FROM_CHANNEL(channel)->call_stack_size); + grpc_compression_options_init(&channel->compression_options); for (size_t i = 0; i < args->num_args; i++) { if (0 == strcmp(args->args[i].key, GRPC_ARG_DEFAULT_AUTHORITY)) { @@ -177,6 +183,32 @@ done: return channel; } +size_t grpc_channel_get_call_size_estimate(grpc_channel *channel) { +#define ROUND_UP_SIZE 256 + return ((size_t)gpr_atm_no_barrier_load(&channel->call_size_estimate) + + ROUND_UP_SIZE) & + ~(size_t)(ROUND_UP_SIZE - 1); +} + +void grpc_channel_update_call_size_estimate(grpc_channel *channel, + size_t size) { + size_t cur = (size_t)gpr_atm_no_barrier_load(&channel->call_size_estimate); + if (cur < size) { + /* size grew: update estimate */ + gpr_atm_no_barrier_cas(&channel->call_size_estimate, (gpr_atm)cur, + (gpr_atm)size); + /* if we lose: never mind, something else will likely update soon enough */ + } else if (cur == size) { + /* no change: holding pattern */ + } else if (cur > 0) { + /* size shrank: decrease estimate */ + gpr_atm_no_barrier_cas( + &channel->call_size_estimate, (gpr_atm)cur, + (gpr_atm)(GPR_MIN(cur - 1, (255 * cur + size) / 256))); + /* if we lose: never mind, something else will likely update soon enough */ + } +} + char *grpc_channel_get_target(grpc_channel *channel) { GRPC_API_TRACE("grpc_channel_get_target(channel=%p)", 1, (channel)); return gpr_strdup(channel->target); diff --git a/src/core/lib/surface/channel.h b/src/core/lib/surface/channel.h index 3a441d7adde..c4aebd8b9be 100644 --- a/src/core/lib/surface/channel.h +++ b/src/core/lib/surface/channel.h @@ -66,6 +66,9 @@ grpc_mdelem grpc_channel_get_reffed_status_elem(grpc_exec_ctx *exec_ctx, grpc_channel *channel, int status_code); +size_t grpc_channel_get_call_size_estimate(grpc_channel *channel); +void grpc_channel_update_call_size_estimate(grpc_channel *channel, size_t size); + #ifdef GRPC_STREAM_REFCOUNT_DEBUG void grpc_channel_internal_ref(grpc_channel *channel, const char *reason); void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx, grpc_channel *channel, From fb9d11204388b524b942950b47fef24116eab243 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Mar 2017 09:26:27 -0700 Subject: [PATCH 148/188] Review comments --- src/core/ext/client_channel/client_channel.c | 4 ++-- src/core/ext/client_channel/subchannel.c | 14 +++++++------- .../transport/chttp2/transport/incoming_metadata.c | 2 +- src/core/lib/support/arena.c | 1 - test/core/channel/channel_stack_test.c | 2 +- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index f6550292a0a..ba842c79162 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -755,7 +755,7 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, } else { /* Create call on subchannel. */ grpc_subchannel_call *subchannel_call = NULL; - grpc_connected_subchannel_call_args call_args = { + const grpc_connected_subchannel_call_args call_args = { .pollent = calld->pollent, .path = calld->path, .start_time = calld->call_start_time, @@ -988,7 +988,7 @@ static void start_transport_stream_op_locked_inner(grpc_exec_ctx *exec_ctx, if (calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && calld->connected_subchannel != NULL) { grpc_subchannel_call *subchannel_call = NULL; - grpc_connected_subchannel_call_args call_args = { + const grpc_connected_subchannel_call_args call_args = { .pollent = calld->pollent, .path = calld->path, .start_time = calld->call_start_time, diff --git a/src/core/ext/client_channel/subchannel.c b/src/core/ext/client_channel/subchannel.c index e4b669c5828..cef08a04ea9 100644 --- a/src/core/ext/client_channel/subchannel.c +++ b/src/core/ext/client_channel/subchannel.c @@ -778,13 +778,13 @@ grpc_error *grpc_connected_subchannel_create_call( args->arena, sizeof(grpc_subchannel_call) + chanstk->call_stack_size); grpc_call_stack *callstk = SUBCHANNEL_CALL_TO_CALL_STACK(*call); (*call)->connection = con; // Ref is added below. - grpc_call_element_args call_args = {.call_stack = callstk, - .server_transport_data = NULL, - .context = NULL, - .path = args->path, - .start_time = args->start_time, - .deadline = args->deadline, - .arena = args->arena}; + const grpc_call_element_args call_args = {.call_stack = callstk, + .server_transport_data = NULL, + .context = NULL, + .path = args->path, + .start_time = args->start_time, + .deadline = args->deadline, + .arena = args->arena}; grpc_error *error = grpc_call_stack_init( exec_ctx, chanstk, 1, subchannel_call_destroy, *call, &call_args); if (error != GRPC_ERROR_NONE) { diff --git a/src/core/ext/transport/chttp2/transport/incoming_metadata.c b/src/core/ext/transport/chttp2/transport/incoming_metadata.c index ab900fdff26..da0a34d32a0 100644 --- a/src/core/ext/transport/chttp2/transport/incoming_metadata.c +++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.c @@ -42,8 +42,8 @@ void grpc_chttp2_incoming_metadata_buffer_init( grpc_chttp2_incoming_metadata_buffer *buffer, gpr_arena *arena) { - grpc_metadata_batch_init(&buffer->batch); buffer->arena = arena; + grpc_metadata_batch_init(&buffer->batch); buffer->batch.deadline = gpr_inf_future(GPR_CLOCK_REALTIME); } diff --git a/src/core/lib/support/arena.c b/src/core/lib/support/arena.c index 8ce29aaab34..6610acd3a07 100644 --- a/src/core/lib/support/arena.c +++ b/src/core/lib/support/arena.c @@ -53,7 +53,6 @@ struct gpr_arena { gpr_arena *gpr_arena_create(size_t initial_size) { initial_size = ROUND_UP_TO_ALIGNMENT_SIZE(initial_size); - gpr_log(GPR_DEBUG, "arena create: %" PRIdPTR, initial_size); gpr_arena *a = gpr_zalloc(sizeof(gpr_arena) + initial_size); a->initial_zone.size_end = initial_size; return a; diff --git a/test/core/channel/channel_stack_test.c b/test/core/channel/channel_stack_test.c index 0c424c1ec48..af551c4928e 100644 --- a/test/core/channel/channel_stack_test.c +++ b/test/core/channel/channel_stack_test.c @@ -139,7 +139,7 @@ static void test_create_channel_stack(void) { GPR_ASSERT(*channel_data == 0); call_stack = gpr_malloc(channel_stack->call_stack_size); - grpc_call_element_args args = { + const grpc_call_element_args args = { .call_stack = call_stack, .server_transport_data = NULL, .context = NULL, From 46bec6bc7377d73b01366c3b8ae0f9d9bab93601 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 14 Mar 2017 09:39:41 -0700 Subject: [PATCH 149/188] Correct Kokoro scripts --- tools/internal_ci/linux/grpc_master_sanitizers.sh | 2 +- tools/internal_ci/linux/grpc_portability.sh | 2 +- tools/internal_ci/linux/grpc_portability_build_only.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/internal_ci/linux/grpc_master_sanitizers.sh b/tools/internal_ci/linux/grpc_master_sanitizers.sh index 2cd0efe1588..d22387fb203 100755 --- a/tools/internal_ci/linux/grpc_master_sanitizers.sh +++ b/tools/internal_ci/linux/grpc_master_sanitizers.sh @@ -37,4 +37,4 @@ git submodule update --init # download docker images from dockerhub export DOCKERHUB_ORGANIZATION=grpctesting -tools/run_tests/run_tests_matrix.sh -f sanitizers linux +tools/run_tests/run_tests_matrix.py -f sanitizers linux diff --git a/tools/internal_ci/linux/grpc_portability.sh b/tools/internal_ci/linux/grpc_portability.sh index c7ac3bbcbee..58d3c58e707 100755 --- a/tools/internal_ci/linux/grpc_portability.sh +++ b/tools/internal_ci/linux/grpc_portability.sh @@ -37,4 +37,4 @@ git submodule update --init # download docker images from dockerhub export DOCKERHUB_ORGANIZATION=grpctesting -tools/run_tests/run_tests_matrix.sh -f portability linux +tools/run_tests/run_tests_matrix.py -f portability linux diff --git a/tools/internal_ci/linux/grpc_portability_build_only.sh b/tools/internal_ci/linux/grpc_portability_build_only.sh index edd6f916872..80b5c4cb960 100755 --- a/tools/internal_ci/linux/grpc_portability_build_only.sh +++ b/tools/internal_ci/linux/grpc_portability_build_only.sh @@ -37,4 +37,4 @@ git submodule update --init # download docker images from dockerhub export DOCKERHUB_ORGANIZATION=grpctesting -tools/run_tests/run_tests_matrix.sh -f portability linux --build_only +tools/run_tests/run_tests_matrix.py -f portability linux --build_only From 90f43d4fa4d833b9ae693ee02a1b956f7188971a Mon Sep 17 00:00:00 2001 From: yang-g Date: Tue, 14 Mar 2017 10:58:44 -0700 Subject: [PATCH 150/188] add missing header --- test/cpp/qps/qps_json_driver.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cpp/qps/qps_json_driver.cc b/test/cpp/qps/qps_json_driver.cc index ddaaa7ca75e..bd2c1f0ec67 100644 --- a/test/cpp/qps/qps_json_driver.cc +++ b/test/cpp/qps/qps_json_driver.cc @@ -31,6 +31,7 @@ * */ +#include #include #include From d26250be35a0e0921cb97f43b13ce14343ee4c7d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Mar 2017 11:01:52 -0700 Subject: [PATCH 151/188] Get growth right --- src/core/lib/support/arena.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/lib/support/arena.c b/src/core/lib/support/arena.c index 6610acd3a07..7bcb983f245 100644 --- a/src/core/lib/support/arena.c +++ b/src/core/lib/support/arena.c @@ -78,8 +78,7 @@ void *gpr_arena_alloc(gpr_arena *arena, size_t size) { while (start > z->size_end) { zone *next_z = (zone *)gpr_atm_acq_load(&z->next_atm); if (next_z == NULL) { - size_t next_z_size = - GPR_MAX((size_t)gpr_atm_no_barrier_load(&arena->size_so_far), size); + size_t next_z_size = (size_t)gpr_atm_no_barrier_load(&arena->size_so_far); next_z = gpr_zalloc(sizeof(zone) + next_z_size); next_z->size_begin = z->size_end; next_z->size_end = z->size_end + next_z_size; From 4564b8cde72f74b88ec2ddf4943768e809dff94b Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 14 Mar 2017 01:21:57 -0700 Subject: [PATCH 152/188] Copy value at SetPointerWithVtable --- src/cpp/common/channel_arguments.cc | 4 +++- test/cpp/common/channel_arguments_test.cc | 7 ------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc index 34fd4491d0a..eddcacc3325 100644 --- a/src/cpp/common/channel_arguments.cc +++ b/src/cpp/common/channel_arguments.cc @@ -111,8 +111,10 @@ void ChannelArguments::SetSocketMutator(grpc_socket_mutator* mutator) { for (auto it = args_.begin(); it != args_.end(); ++it) { if (it->type == mutator_arg.type && grpc::string(it->key) == grpc::string(mutator_arg.key)) { + GPR_ASSERT(!replaced); it->value.pointer.vtable->destroy(&exec_ctx, it->value.pointer.p); it->value.pointer = mutator_arg.value.pointer; + replaced = true; } } grpc_exec_ctx_finish(&exec_ctx); @@ -195,7 +197,7 @@ void ChannelArguments::SetPointerWithVtable( arg.type = GRPC_ARG_POINTER; strings_.push_back(key); arg.key = const_cast(strings_.back().c_str()); - arg.value.pointer.p = value; + arg.value.pointer.p = vtable->copy(value); arg.value.pointer.vtable = vtable; args_.push_back(arg); } diff --git a/test/cpp/common/channel_arguments_test.cc b/test/cpp/common/channel_arguments_test.cc index 190d32ce060..9bcc9f99f68 100644 --- a/test/cpp/common/channel_arguments_test.cc +++ b/test/cpp/common/channel_arguments_test.cc @@ -230,13 +230,6 @@ TEST_F(ChannelArgumentsTest, SetSocketMutator) { EXPECT_TRUE(HasArg(arg1)); // arg0 is replaced by arg1 EXPECT_FALSE(HasArg(arg0)); - - // arg0 is destroyed by grpc_socket_mutator_to_arg(mutator1) - { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - arg1.value.pointer.vtable->destroy(&exec_ctx, arg1.value.pointer.p); - grpc_exec_ctx_finish(&exec_ctx); - } } TEST_F(ChannelArgumentsTest, SetUserAgentPrefix) { From 58796c8589d3a805032599ab8bf40a48360be091 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Tue, 14 Mar 2017 20:58:24 +0100 Subject: [PATCH 153/188] G stands for... --- build.yaml | 2 +- src/core/lib/surface/version.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.yaml b/build.yaml index 7c43df32502..2c0f6db1adb 100644 --- a/build.yaml +++ b/build.yaml @@ -13,7 +13,7 @@ settings: '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here core_version: 3.0.0-dev - g_stands_for: green + g_stands_for: gentle version: 1.3.0-dev filegroups: - name: census diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c index 1143a9e044e..ba80bd801ee 100644 --- a/src/core/lib/surface/version.c +++ b/src/core/lib/surface/version.c @@ -38,4 +38,4 @@ const char *grpc_version_string(void) { return "3.0.0-dev"; } -const char *grpc_g_stands_for(void) { return "green"; } +const char *grpc_g_stands_for(void) { return "gentle"; } From 008c1f09115b23196d192d9ddd92b431a824edd7 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Tue, 14 Mar 2017 21:12:44 +0100 Subject: [PATCH 154/188] Updating g_stands_for.md --- doc/g_stands_for.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md index 53a1fdf193c..d2fc7a50f95 100644 --- a/doc/g_stands_for.md +++ b/doc/g_stands_for.md @@ -7,3 +7,4 @@ future), and the corresponding version numbers that used them: - 1.0 'g' stands for 'gRPC' - 1.1 'g' stands for 'good' - 1.2 'g' stands for 'green' +- 1.3 'g' stands for 'gentle' From 456ddd83df4ee8305fdd209893acd43a1444e362 Mon Sep 17 00:00:00 2001 From: Michael Warres Date: Tue, 7 Mar 2017 14:02:51 -0500 Subject: [PATCH 155/188] Remove grpc_udp_server dependency on grpc_server. --- src/core/lib/iomgr/udp_server.c | 16 ++++++++-------- src/core/lib/iomgr/udp_server.h | 12 ++++++------ test/core/iomgr/udp_server_test.c | 8 ++++---- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index d1bcd89af1f..71e295770a4 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -109,8 +109,8 @@ struct grpc_udp_server { grpc_pollset **pollsets; /* number of pollsets in the pollsets array */ size_t pollset_count; - /* The parent grpc server */ - grpc_server *grpc_server; + /* opaque object to pass to callbacks */ + void *user_data; }; grpc_udp_server *grpc_udp_server_create(void) { @@ -178,7 +178,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { /* Call the orphan_cb to signal that the FD is about to be closed and * should no longer be used. */ GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(exec_ctx, sp->emfd); + sp->orphan_cb(exec_ctx, sp->emfd, sp->server->user_data); grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL, "udp_listener_shutdown"); @@ -204,7 +204,7 @@ void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, if (s->active_ports) { for (sp = s->head; sp; sp = sp->next) { GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(exec_ctx, sp->emfd); + sp->orphan_cb(exec_ctx, sp->emfd, sp->server->user_data); grpc_fd_shutdown(exec_ctx, sp->emfd, GRPC_ERROR_CREATE("Server destroyed")); } @@ -299,7 +299,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { /* Tell the registered callback that data is available to read. */ GPR_ASSERT(sp->read_cb); - sp->read_cb(exec_ctx, sp->emfd, sp->server->grpc_server); + sp->read_cb(exec_ctx, sp->emfd, sp->server->user_data); /* Re-arm the notification event so we get another chance to read. */ grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure); @@ -322,7 +322,7 @@ static void on_write(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { /* Tell the registered callback that the socket is writeable. */ GPR_ASSERT(sp->write_cb); - sp->write_cb(exec_ctx, sp->emfd); + sp->write_cb(exec_ctx, sp->emfd, sp->server->user_data); /* Re-arm the notification event so we get another chance to write. */ grpc_fd_notify_on_write(exec_ctx, sp->emfd, &sp->write_closure); @@ -464,13 +464,13 @@ int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned port_index) { void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, grpc_pollset **pollsets, size_t pollset_count, - grpc_server *server) { + void *user_data) { size_t i; gpr_mu_lock(&s->mu); grpc_udp_listener *sp; GPR_ASSERT(s->active_ports == 0); s->pollsets = pollsets; - s->grpc_server = server; + s->user_data = user_data; sp = s->head; while (sp != NULL) { diff --git a/src/core/lib/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h index ed63fa7d817..90842a47f09 100644 --- a/src/core/lib/iomgr/udp_server.h +++ b/src/core/lib/iomgr/udp_server.h @@ -47,23 +47,23 @@ typedef struct grpc_udp_server grpc_udp_server; /* Called when data is available to read from the socket. */ typedef void (*grpc_udp_server_read_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, - struct grpc_server *server); + void *user_data); /* Called when the socket is writeable. */ -typedef void (*grpc_udp_server_write_cb)(grpc_exec_ctx *exec_ctx, - grpc_fd *emfd); +typedef void (*grpc_udp_server_write_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, + void *user_data); /* Called when the grpc_fd is about to be orphaned (and the FD closed). */ typedef void (*grpc_udp_server_orphan_cb)(grpc_exec_ctx *exec_ctx, - grpc_fd *emfd); + grpc_fd *emfd, void *user_data); /* Create a server, initially not bound to any ports */ grpc_udp_server *grpc_udp_server_create(void); -/* Start listening to bound ports */ +/* Start listening to bound ports. user_data is passed to callbacks. */ void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *udp_server, grpc_pollset **pollsets, size_t pollset_count, - struct grpc_server *server); + void *user_data); int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned port_index); diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index d57a37b2a97..396ec959cdc 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -62,8 +62,7 @@ static int g_number_of_writes = 0; static int g_number_of_bytes_read = 0; static int g_number_of_orphan_calls = 0; -static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, - grpc_server *server) { +static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { char read_buffer[512]; ssize_t byte_count; @@ -79,7 +78,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, gpr_mu_unlock(g_mu); } -static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd) { +static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { gpr_mu_lock(g_mu); g_number_of_writes++; @@ -88,7 +87,8 @@ static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd) { gpr_mu_unlock(g_mu); } -static void on_fd_orphaned(grpc_exec_ctx *exec_ctx, grpc_fd *emfd) { +static void on_fd_orphaned(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, + void *user_data) { gpr_log(GPR_INFO, "gRPC FD about to be orphaned: %d", grpc_fd_wrapped_fd(emfd)); g_number_of_orphan_calls++; From 68745bbf8a0193a8a2c2691281365a1e9d488863 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 14 Mar 2017 17:51:07 -0700 Subject: [PATCH 156/188] Resolve dependency issues --- BUILD | 1 + CMakeLists.txt | 5 + Makefile | 5 + binding.gyp | 1 + build.yaml | 1 + config.m4 | 1 + gRPC-Core.podspec | 1 + grpc.gemspec | 1 + package.xml | 1 + src/core/lib/iomgr/tcp_server_posix.c | 229 +----------------- src/core/lib/iomgr/tcp_server_utils_posix.h | 53 +++- .../lib/iomgr/tcp_server_utils_posix_common.c | 220 +++++++++++++++++ .../iomgr/tcp_server_utils_posix_ifaddrs.c | 19 +- .../iomgr/tcp_server_utils_posix_noifaddrs.c | 4 +- src/python/grpcio/grpc_core_dependencies.py | 1 + tools/doxygen/Doxyfile.core.internal | 1 + .../generated/sources_and_headers.json | 1 + vsprojects/vcxproj/grpc/grpc.vcxproj | 2 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 3 + .../grpc_test_util/grpc_test_util.vcxproj | 2 + .../grpc_test_util.vcxproj.filters | 3 + .../grpc_unsecure/grpc_unsecure.vcxproj | 2 + .../grpc_unsecure.vcxproj.filters | 3 + 23 files changed, 325 insertions(+), 235 deletions(-) create mode 100644 src/core/lib/iomgr/tcp_server_utils_posix_common.c diff --git a/BUILD b/BUILD index dc6431484e3..371ab200d22 100644 --- a/BUILD +++ b/BUILD @@ -481,6 +481,7 @@ grpc_cc_library( "src/core/lib/iomgr/tcp_client_windows.c", "src/core/lib/iomgr/tcp_posix.c", "src/core/lib/iomgr/tcp_server_posix.c", + "src/core/lib/iomgr/tcp_server_utils_posix_common.c", "src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c", "src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c", "src/core/lib/iomgr/tcp_server_uv.c", diff --git a/CMakeLists.txt b/CMakeLists.txt index cfc4555834a..c7001aa721c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -914,6 +914,7 @@ add_library(grpc src/core/lib/iomgr/tcp_client_windows.c src/core/lib/iomgr/tcp_posix.c src/core/lib/iomgr/tcp_server_posix.c + src/core/lib/iomgr/tcp_server_utils_posix_common.c src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c src/core/lib/iomgr/tcp_server_uv.c @@ -1225,6 +1226,7 @@ add_library(grpc_cronet src/core/lib/iomgr/tcp_client_windows.c src/core/lib/iomgr/tcp_posix.c src/core/lib/iomgr/tcp_server_posix.c + src/core/lib/iomgr/tcp_server_utils_posix_common.c src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c src/core/lib/iomgr/tcp_server_uv.c @@ -1527,6 +1529,7 @@ add_library(grpc_test_util src/core/lib/iomgr/tcp_client_windows.c src/core/lib/iomgr/tcp_posix.c src/core/lib/iomgr/tcp_server_posix.c + src/core/lib/iomgr/tcp_server_utils_posix_common.c src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c src/core/lib/iomgr/tcp_server_uv.c @@ -1775,6 +1778,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/tcp_client_windows.c src/core/lib/iomgr/tcp_posix.c src/core/lib/iomgr/tcp_server_posix.c + src/core/lib/iomgr/tcp_server_utils_posix_common.c src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c src/core/lib/iomgr/tcp_server_uv.c @@ -2384,6 +2388,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/tcp_client_windows.c src/core/lib/iomgr/tcp_posix.c src/core/lib/iomgr/tcp_server_posix.c + src/core/lib/iomgr/tcp_server_utils_posix_common.c src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c src/core/lib/iomgr/tcp_server_uv.c diff --git a/Makefile b/Makefile index 14c5a36c957..e31b7514e1d 100644 --- a/Makefile +++ b/Makefile @@ -2799,6 +2799,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_common.c \ src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ @@ -3113,6 +3114,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_common.c \ src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ @@ -3418,6 +3420,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_common.c \ src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ @@ -3646,6 +3649,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_common.c \ src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ @@ -4257,6 +4261,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_common.c \ src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ diff --git a/binding.gyp b/binding.gyp index ec21a97806f..96843c0d6b7 100644 --- a/binding.gyp +++ b/binding.gyp @@ -667,6 +667,7 @@ 'src/core/lib/iomgr/tcp_client_windows.c', 'src/core/lib/iomgr/tcp_posix.c', 'src/core/lib/iomgr/tcp_server_posix.c', + 'src/core/lib/iomgr/tcp_server_utils_posix_common.c', 'src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c', 'src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c', 'src/core/lib/iomgr/tcp_server_uv.c', diff --git a/build.yaml b/build.yaml index 36817a55067..75bacbde632 100644 --- a/build.yaml +++ b/build.yaml @@ -338,6 +338,7 @@ filegroups: - src/core/lib/iomgr/tcp_client_windows.c - src/core/lib/iomgr/tcp_posix.c - src/core/lib/iomgr/tcp_server_posix.c + - src/core/lib/iomgr/tcp_server_utils_posix_common.c - src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c - src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c - src/core/lib/iomgr/tcp_server_uv.c diff --git a/config.m4 b/config.m4 index 181e44c52c8..6d6f9419cd1 100644 --- a/config.m4 +++ b/config.m4 @@ -140,6 +140,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/tcp_client_windows.c \ src/core/lib/iomgr/tcp_posix.c \ src/core/lib/iomgr/tcp_server_posix.c \ + src/core/lib/iomgr/tcp_server_utils_posix_common.c \ src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 1d002247d4b..16c17c2bf6e 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -509,6 +509,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/tcp_client_windows.c', 'src/core/lib/iomgr/tcp_posix.c', 'src/core/lib/iomgr/tcp_server_posix.c', + 'src/core/lib/iomgr/tcp_server_utils_posix_common.c', 'src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c', 'src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c', 'src/core/lib/iomgr/tcp_server_uv.c', diff --git a/grpc.gemspec b/grpc.gemspec index edc08cb9b9a..da9535ff469 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -426,6 +426,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/tcp_client_windows.c ) s.files += %w( src/core/lib/iomgr/tcp_posix.c ) s.files += %w( src/core/lib/iomgr/tcp_server_posix.c ) + s.files += %w( src/core/lib/iomgr/tcp_server_utils_posix_common.c ) s.files += %w( src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c ) s.files += %w( src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c ) s.files += %w( src/core/lib/iomgr/tcp_server_uv.c ) diff --git a/package.xml b/package.xml index 21eeee4d3ef..aeeb0d59867 100644 --- a/package.xml +++ b/package.xml @@ -435,6 +435,7 @@ + diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index 1e9e854d6b5..b3644518f54 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -44,10 +44,8 @@ #include #include -#include #include #include -#include #include #include #include @@ -70,56 +68,6 @@ #include "src/core/lib/iomgr/unix_sockets_posix.h" #include "src/core/lib/support/string.h" -#define MIN_SAFE_ACCEPT_QUEUE_SIZE 100 - -static gpr_once s_init_max_accept_queue_size; -static int s_max_accept_queue_size; - -/* the overall server */ -struct grpc_tcp_server { - gpr_refcount refs; - /* Called whenever accept() succeeds on a server port. */ - grpc_tcp_server_cb on_accept_cb; - void *on_accept_cb_arg; - - gpr_mu mu; - - /* active port count: how many ports are actually still listening */ - size_t active_ports; - /* destroyed port count: how many ports are completely destroyed */ - size_t destroyed_ports; - - /* is this server shutting down? */ - bool shutdown; - /* have listeners been shutdown? */ - bool shutdown_listeners; - /* use SO_REUSEPORT */ - bool so_reuseport; - /* expand wildcard addresses to a list of all local addresses */ - bool expand_wildcard_addrs; - - /* linked list of server ports */ - grpc_tcp_listener *head; - grpc_tcp_listener *tail; - unsigned nports; - - /* List of closures passed to shutdown_starting_add(). */ - grpc_closure_list shutdown_starting; - - /* shutdown callback */ - grpc_closure *shutdown_complete; - - /* all pollsets interested in new connections */ - grpc_pollset **pollsets; - /* number of pollsets in the pollsets array */ - size_t pollset_count; - - /* next pollset to assign a channel to */ - gpr_atm next_pollset_to_assign; - - grpc_resource_quota *resource_quota; -}; - static gpr_once check_init = GPR_ONCE_INIT; static bool has_so_reuseport = false; @@ -278,99 +226,6 @@ static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { } } -/* get max listen queue size on linux */ -static void init_max_accept_queue_size(void) { - int n = SOMAXCONN; - char buf[64]; - FILE *fp = fopen("/proc/sys/net/core/somaxconn", "r"); - if (fp == NULL) { - /* 2.4 kernel. */ - s_max_accept_queue_size = SOMAXCONN; - return; - } - if (fgets(buf, sizeof buf, fp)) { - char *end; - long i = strtol(buf, &end, 10); - if (i > 0 && i <= INT_MAX && end && *end == 0) { - n = (int)i; - } - } - fclose(fp); - s_max_accept_queue_size = n; - - if (s_max_accept_queue_size < MIN_SAFE_ACCEPT_QUEUE_SIZE) { - gpr_log(GPR_INFO, - "Suspiciously small accept queue (%d) will probably lead to " - "connection drops", - s_max_accept_queue_size); - } -} - -static int get_max_accept_queue_size(void) { - gpr_once_init(&s_init_max_accept_queue_size, init_max_accept_queue_size); - return s_max_accept_queue_size; -} - -/* Prepare a recently-created socket for listening. */ -static grpc_error *prepare_socket(int fd, const grpc_resolved_address *addr, - bool so_reuseport, int *port) { - grpc_resolved_address sockname_temp; - grpc_error *err = GRPC_ERROR_NONE; - - GPR_ASSERT(fd >= 0); - - if (so_reuseport && !grpc_is_unix_socket(addr)) { - err = grpc_set_socket_reuse_port(fd, 1); - if (err != GRPC_ERROR_NONE) goto error; - } - - err = grpc_set_socket_nonblocking(fd, 1); - if (err != GRPC_ERROR_NONE) goto error; - err = grpc_set_socket_cloexec(fd, 1); - if (err != GRPC_ERROR_NONE) goto error; - if (!grpc_is_unix_socket(addr)) { - err = grpc_set_socket_low_latency(fd, 1); - if (err != GRPC_ERROR_NONE) goto error; - err = grpc_set_socket_reuse_addr(fd, 1); - if (err != GRPC_ERROR_NONE) goto error; - } - err = grpc_set_socket_no_sigpipe_if_possible(fd); - if (err != GRPC_ERROR_NONE) goto error; - - GPR_ASSERT(addr->len < ~(socklen_t)0); - if (bind(fd, (struct sockaddr *)addr->addr, (socklen_t)addr->len) < 0) { - err = GRPC_OS_ERROR(errno, "bind"); - goto error; - } - - if (listen(fd, get_max_accept_queue_size()) < 0) { - err = GRPC_OS_ERROR(errno, "listen"); - goto error; - } - - sockname_temp.len = sizeof(struct sockaddr_storage); - - if (getsockname(fd, (struct sockaddr *)sockname_temp.addr, - (socklen_t *)&sockname_temp.len) < 0) { - err = GRPC_OS_ERROR(errno, "getsockname"); - goto error; - } - - *port = grpc_sockaddr_get_port(&sockname_temp); - return GRPC_ERROR_NONE; - -error: - GPR_ASSERT(err != GRPC_ERROR_NONE); - if (fd >= 0) { - close(fd); - } - grpc_error *ret = grpc_error_set_int( - GRPC_ERROR_CREATE_REFERENCING("Unable to configure socket", &err, 1), - GRPC_ERROR_INT_FD, fd); - GRPC_ERROR_UNREF(err); - return ret; -} - /* event manager callback when reads are ready */ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { grpc_tcp_listener *sp = arg; @@ -454,88 +309,6 @@ error: } } -static grpc_error *add_socket_to_server(grpc_tcp_server *s, int fd, - const grpc_resolved_address *addr, - unsigned port_index, unsigned fd_index, - grpc_tcp_listener **listener) { - grpc_tcp_listener *sp = NULL; - int port = -1; - char *addr_str; - char *name; - - grpc_error *err = prepare_socket(fd, addr, s->so_reuseport, &port); - if (err == GRPC_ERROR_NONE) { - GPR_ASSERT(port > 0); - grpc_sockaddr_to_string(&addr_str, addr, 1); - gpr_asprintf(&name, "tcp-server-listener:%s", addr_str); - gpr_mu_lock(&s->mu); - s->nports++; - GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server"); - sp = gpr_malloc(sizeof(grpc_tcp_listener)); - sp->next = NULL; - if (s->head == NULL) { - s->head = sp; - } else { - s->tail->next = sp; - } - s->tail = sp; - sp->server = s; - sp->fd = fd; - sp->emfd = grpc_fd_create(fd, name); - memcpy(&sp->addr, addr, sizeof(grpc_resolved_address)); - sp->port = port; - sp->port_index = port_index; - sp->fd_index = fd_index; - sp->is_sibling = 0; - sp->sibling = NULL; - GPR_ASSERT(sp->emfd); - gpr_mu_unlock(&s->mu); - gpr_free(addr_str); - gpr_free(name); - } - - *listener = sp; - return err; -} - -/* If successful, add a listener to s for addr, set *dsmode for the socket, and - return the *listener. */ -grpc_error *grpc_tcp_server_add_addr(grpc_tcp_server *s, - const grpc_resolved_address *addr, - unsigned port_index, unsigned fd_index, - grpc_dualstack_mode *dsmode, - grpc_tcp_listener **listener) { - grpc_resolved_address addr4_copy; - int fd; - grpc_error *err = - grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, dsmode, &fd); - if (err != GRPC_ERROR_NONE) { - return err; - } - if (*dsmode == GRPC_DSMODE_IPV4 && - grpc_sockaddr_is_v4mapped(addr, &addr4_copy)) { - addr = &addr4_copy; - } - return add_socket_to_server(s, fd, addr, port_index, fd_index, listener); -} - -/* Return the listener in s with address addr or NULL. */ -grpc_tcp_listener *grpc_tcp_server_find_listener_with_addr( - grpc_tcp_server *s, grpc_resolved_address *addr) { - grpc_tcp_listener *l; - gpr_mu_lock(&s->mu); - for (l = s->head; l != NULL; l = l->next) { - if (l->addr.len != addr->len) { - continue; - } - if (memcmp(l->addr.addr, addr->addr, addr->len) == 0) { - break; - } - } - gpr_mu_unlock(&s->mu); - return l; -} - /* Treat :: or 0.0.0.0 as a family-agnostic wildcard. */ static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s, unsigned port_index, @@ -607,7 +380,7 @@ static grpc_error *clone_port(grpc_tcp_listener *listener, unsigned count) { err = grpc_create_dualstack_socket(&listener->addr, SOCK_STREAM, 0, &dsmode, &fd); if (err != GRPC_ERROR_NONE) return err; - err = prepare_socket(fd, &listener->addr, true, &port); + err = grpc_tcp_server_prepare_socket(fd, &listener->addr, true, &port); if (err != GRPC_ERROR_NONE) return err; listener->server->nports++; grpc_sockaddr_to_string(&addr_str, &listener->addr, 1); diff --git a/src/core/lib/iomgr/tcp_server_utils_posix.h b/src/core/lib/iomgr/tcp_server_utils_posix.h index d39f9085fda..f5dc8532f9f 100644 --- a/src/core/lib/iomgr/tcp_server_utils_posix.h +++ b/src/core/lib/iomgr/tcp_server_utils_posix.h @@ -61,6 +61,51 @@ typedef struct grpc_tcp_listener { int is_sibling; } grpc_tcp_listener; +/* the overall server */ +struct grpc_tcp_server { + gpr_refcount refs; + /* Called whenever accept() succeeds on a server port. */ + grpc_tcp_server_cb on_accept_cb; + void *on_accept_cb_arg; + + gpr_mu mu; + + /* active port count: how many ports are actually still listening */ + size_t active_ports; + /* destroyed port count: how many ports are completely destroyed */ + size_t destroyed_ports; + + /* is this server shutting down? */ + bool shutdown; + /* have listeners been shutdown? */ + bool shutdown_listeners; + /* use SO_REUSEPORT */ + bool so_reuseport; + /* expand wildcard addresses to a list of all local addresses */ + bool expand_wildcard_addrs; + + /* linked list of server ports */ + grpc_tcp_listener *head; + grpc_tcp_listener *tail; + unsigned nports; + + /* List of closures passed to shutdown_starting_add(). */ + grpc_closure_list shutdown_starting; + + /* shutdown callback */ + grpc_closure *shutdown_complete; + + /* all pollsets interested in new connections */ + grpc_pollset **pollsets; + /* number of pollsets in the pollsets array */ + size_t pollset_count; + + /* next pollset to assign a channel to */ + gpr_atm next_pollset_to_assign; + + grpc_resource_quota *resource_quota; +}; + /* If successful, add a listener to \a s for \a addr, set \a dsmode for the socket, and return the \a listener. */ grpc_error *grpc_tcp_server_add_addr(grpc_tcp_server *s, @@ -69,10 +114,6 @@ grpc_error *grpc_tcp_server_add_addr(grpc_tcp_server *s, grpc_dualstack_mode *dsmode, grpc_tcp_listener **listener); -/* Return the listener in \a s with address \a addr or NULL. */ -grpc_tcp_listener *grpc_tcp_server_find_listener_with_addr( - grpc_tcp_server *s, grpc_resolved_address *addr); - /* Get all addresses assigned to network interfaces on the machine and create a listener for each. requested_port is the port to use for every listener, or 0 to select one random port that will be used for every listener. Set *out_port @@ -83,6 +124,10 @@ grpc_error *grpc_tcp_server_add_all_local_addrs(grpc_tcp_server *s, int requested_port, int *out_port); +/* Prepare a recently-created socket for listening. */ +grpc_error *grpc_tcp_server_prepare_socket(int fd, + const grpc_resolved_address *addr, + bool so_reuseport, int *port); /* Ruturn true if the platform supports ifaddrs */ bool grpc_tcp_server_have_ifaddrs(void); diff --git a/src/core/lib/iomgr/tcp_server_utils_posix_common.c b/src/core/lib/iomgr/tcp_server_utils_posix_common.c new file mode 100644 index 00000000000..e45e27d5ab6 --- /dev/null +++ b/src/core/lib/iomgr/tcp_server_utils_posix_common.c @@ -0,0 +1,220 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +#ifdef GRPC_HAVE_IFADDRS + +#include "src/core/lib/iomgr/tcp_server_utils_posix.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/sockaddr.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" +#include "src/core/lib/iomgr/unix_sockets_posix.h" + +#define MIN_SAFE_ACCEPT_QUEUE_SIZE 100 + +static gpr_once s_init_max_accept_queue_size; +static int s_max_accept_queue_size; + +/* get max listen queue size on linux */ +static void init_max_accept_queue_size(void) { + int n = SOMAXCONN; + char buf[64]; + FILE *fp = fopen("/proc/sys/net/core/somaxconn", "r"); + if (fp == NULL) { + /* 2.4 kernel. */ + s_max_accept_queue_size = SOMAXCONN; + return; + } + if (fgets(buf, sizeof buf, fp)) { + char *end; + long i = strtol(buf, &end, 10); + if (i > 0 && i <= INT_MAX && end && *end == 0) { + n = (int)i; + } + } + fclose(fp); + s_max_accept_queue_size = n; + + if (s_max_accept_queue_size < MIN_SAFE_ACCEPT_QUEUE_SIZE) { + gpr_log(GPR_INFO, + "Suspiciously small accept queue (%d) will probably lead to " + "connection drops", + s_max_accept_queue_size); + } +} + +static int get_max_accept_queue_size(void) { + gpr_once_init(&s_init_max_accept_queue_size, init_max_accept_queue_size); + return s_max_accept_queue_size; +} + +static grpc_error *add_socket_to_server(grpc_tcp_server *s, int fd, + const grpc_resolved_address *addr, + unsigned port_index, unsigned fd_index, + grpc_tcp_listener **listener) { + grpc_tcp_listener *sp = NULL; + int port = -1; + char *addr_str; + char *name; + + grpc_error *err = + grpc_tcp_server_prepare_socket(fd, addr, s->so_reuseport, &port); + if (err == GRPC_ERROR_NONE) { + GPR_ASSERT(port > 0); + grpc_sockaddr_to_string(&addr_str, addr, 1); + gpr_asprintf(&name, "tcp-server-listener:%s", addr_str); + gpr_mu_lock(&s->mu); + s->nports++; + GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server"); + sp = gpr_malloc(sizeof(grpc_tcp_listener)); + sp->next = NULL; + if (s->head == NULL) { + s->head = sp; + } else { + s->tail->next = sp; + } + s->tail = sp; + sp->server = s; + sp->fd = fd; + sp->emfd = grpc_fd_create(fd, name); + memcpy(&sp->addr, addr, sizeof(grpc_resolved_address)); + sp->port = port; + sp->port_index = port_index; + sp->fd_index = fd_index; + sp->is_sibling = 0; + sp->sibling = NULL; + GPR_ASSERT(sp->emfd); + gpr_mu_unlock(&s->mu); + gpr_free(addr_str); + gpr_free(name); + } + + *listener = sp; + return err; +} + +/* If successful, add a listener to s for addr, set *dsmode for the socket, and + return the *listener. */ +grpc_error *grpc_tcp_server_add_addr(grpc_tcp_server *s, + const grpc_resolved_address *addr, + unsigned port_index, unsigned fd_index, + grpc_dualstack_mode *dsmode, + grpc_tcp_listener **listener) { + grpc_resolved_address addr4_copy; + int fd; + grpc_error *err = + grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, dsmode, &fd); + if (err != GRPC_ERROR_NONE) { + return err; + } + if (*dsmode == GRPC_DSMODE_IPV4 && + grpc_sockaddr_is_v4mapped(addr, &addr4_copy)) { + addr = &addr4_copy; + } + return add_socket_to_server(s, fd, addr, port_index, fd_index, listener); +} + +/* Prepare a recently-created socket for listening. */ +grpc_error *grpc_tcp_server_prepare_socket(int fd, + const grpc_resolved_address *addr, + bool so_reuseport, int *port) { + grpc_resolved_address sockname_temp; + grpc_error *err = GRPC_ERROR_NONE; + + GPR_ASSERT(fd >= 0); + + if (so_reuseport && !grpc_is_unix_socket(addr)) { + err = grpc_set_socket_reuse_port(fd, 1); + if (err != GRPC_ERROR_NONE) goto error; + } + + err = grpc_set_socket_nonblocking(fd, 1); + if (err != GRPC_ERROR_NONE) goto error; + err = grpc_set_socket_cloexec(fd, 1); + if (err != GRPC_ERROR_NONE) goto error; + if (!grpc_is_unix_socket(addr)) { + err = grpc_set_socket_low_latency(fd, 1); + if (err != GRPC_ERROR_NONE) goto error; + err = grpc_set_socket_reuse_addr(fd, 1); + if (err != GRPC_ERROR_NONE) goto error; + } + err = grpc_set_socket_no_sigpipe_if_possible(fd); + if (err != GRPC_ERROR_NONE) goto error; + + GPR_ASSERT(addr->len < ~(socklen_t)0); + if (bind(fd, (struct sockaddr *)addr->addr, (socklen_t)addr->len) < 0) { + err = GRPC_OS_ERROR(errno, "bind"); + goto error; + } + + if (listen(fd, get_max_accept_queue_size()) < 0) { + err = GRPC_OS_ERROR(errno, "listen"); + goto error; + } + + sockname_temp.len = sizeof(struct sockaddr_storage); + + if (getsockname(fd, (struct sockaddr *)sockname_temp.addr, + (socklen_t *)&sockname_temp.len) < 0) { + err = GRPC_OS_ERROR(errno, "getsockname"); + goto error; + } + + *port = grpc_sockaddr_get_port(&sockname_temp); + return GRPC_ERROR_NONE; + +error: + GPR_ASSERT(err != GRPC_ERROR_NONE); + if (fd >= 0) { + close(fd); + } + grpc_error *ret = grpc_error_set_int( + GRPC_ERROR_CREATE_REFERENCING("Unable to configure socket", &err, 1), + GRPC_ERROR_INT_FD, fd); + GRPC_ERROR_UNREF(err); + return ret; +} + +#endif /* GRPC_HAVE_IFADDRS */ diff --git a/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c b/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c index 05eb96f73ed..6354a6bdc10 100644 --- a/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c +++ b/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c @@ -50,6 +50,23 @@ #include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/iomgr/sockaddr_utils.h" +/* Return the listener in s with address addr or NULL. */ +static grpc_tcp_listener *find_listener_with_addr(grpc_tcp_server *s, + grpc_resolved_address *addr) { + grpc_tcp_listener *l; + gpr_mu_lock(&s->mu); + for (l = s->head; l != NULL; l = l->next) { + if (l->addr.len != addr->len) { + continue; + } + if (memcmp(l->addr.addr, addr->addr, addr->len) == 0) { + break; + } + } + gpr_mu_unlock(&s->mu); + return l; +} + /* Bind to "::" to get a port number not used by any address. */ static grpc_error *get_unused_port(int *port) { grpc_resolved_address wild; @@ -133,7 +150,7 @@ grpc_error *grpc_tcp_server_add_all_local_addrs(grpc_tcp_server *s, ifa_name, ifa_it->ifa_flags, addr_str); /* We could have multiple interfaces with the same address (e.g., bonding), so look for duplicates. */ - if (grpc_tcp_server_find_listener_with_addr(s, &addr) != NULL) { + if (find_listener_with_addr(s, &addr) != NULL) { gpr_log(GPR_DEBUG, "Skipping duplicate addr %s on interface %s", addr_str, ifa_name); gpr_free(addr_str); diff --git a/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c b/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c index fb21d62d053..95c3198be6f 100644 --- a/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c +++ b/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c @@ -33,7 +33,7 @@ #include "src/core/lib/iomgr/port.h" -#ifndef GRPC_HAVE_IFADDRS +#if defined(GRPC_POSIX_SOCKET) && !defined(GRPC_HAVE_IFADDRS) #include "src/core/lib/iomgr/tcp_server_utils_posix.h" @@ -46,4 +46,4 @@ grpc_error *grpc_tcp_server_add_all_local_addrs(grpc_tcp_server *s, bool grpc_tcp_server_have_ifaddrs(void) { return false; } -#endif /* ifndef GRPC_HAVE_IFADDRS */ +#endif /* defined(GRPC_POSIX_SOCKET) && !defined(GRPC_HAVE_IFADDRS) */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 00b2b8f9026..ea3a51be13d 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -134,6 +134,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/tcp_client_windows.c', 'src/core/lib/iomgr/tcp_posix.c', 'src/core/lib/iomgr/tcp_server_posix.c', + 'src/core/lib/iomgr/tcp_server_utils_posix_common.c', 'src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c', 'src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c', 'src/core/lib/iomgr/tcp_server_uv.c', diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index f9e4775b738..a1287cd8942 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1131,6 +1131,7 @@ src/core/lib/iomgr/tcp_posix.h \ src/core/lib/iomgr/tcp_server.h \ src/core/lib/iomgr/tcp_server_posix.c \ src/core/lib/iomgr/tcp_server_utils_posix.h \ +src/core/lib/iomgr/tcp_server_utils_posix_common.c \ src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c \ src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c \ src/core/lib/iomgr/tcp_server_uv.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 38fe87ac463..9ab9836c4bb 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7638,6 +7638,7 @@ "src/core/lib/iomgr/tcp_server.h", "src/core/lib/iomgr/tcp_server_posix.c", "src/core/lib/iomgr/tcp_server_utils_posix.h", + "src/core/lib/iomgr/tcp_server_utils_posix_common.c", "src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.c", "src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.c", "src/core/lib/iomgr/tcp_server_uv.c", diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 7b48dfbf597..5e3b027663d 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -619,6 +619,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 8ba8a4f8fb9..d75ca766c0d 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -181,6 +181,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index d078db1c3c7..62969e31acf 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -462,6 +462,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 9e2d817143e..30088101f5f 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -238,6 +238,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index b6f10b3d7c1..11ac8bd4b2b 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -586,6 +586,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 03662d4044c..414e2a50b89 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -184,6 +184,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr From 51006fea710ef7e4faefae653432f55947639b76 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Mar 2017 08:07:02 -0700 Subject: [PATCH 157/188] Fix use-after-free --- src/core/lib/surface/call.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 00719be9abc..9a8870b86a4 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -437,9 +437,10 @@ void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c REF_ARG) { static void release_call(grpc_exec_ctx *exec_ctx, void *call, grpc_error *error) { grpc_call *c = call; - grpc_channel_update_call_size_estimate(c->channel, + grpc_channel *channel = c->channel; + grpc_channel_update_call_size_estimate(channel, gpr_arena_destroy(c->arena)); - GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, c->channel, "call"); + GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "call"); } static void set_status_value_directly(grpc_status_code status, void *dest); From dbad370aa5a5480824388d77207d3d0b2aa94a1c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Mar 2017 08:21:19 -0700 Subject: [PATCH 158/188] Fix use-after-free, cronet compiles --- .../cronet/transport/cronet_transport.c | 46 +++++++++++-------- src/core/lib/surface/call.c | 3 +- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 01a03533daf..e827423a2af 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -177,6 +177,7 @@ struct op_storage { }; struct stream_obj { + gpr_arena *arena; struct op_and_state *oas; grpc_transport_stream_op *curr_op; grpc_cronet_transport *curr_ct; @@ -451,15 +452,18 @@ static void on_response_headers_received( gpr_mu_lock(&s->mu); memset(&s->state.rs.initial_metadata, 0, sizeof(s->state.rs.initial_metadata)); - grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.initial_metadata); + grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.initial_metadata, + s->arena); for (size_t i = 0; i < headers->count; i++) { - grpc_chttp2_incoming_metadata_buffer_add( - &s->state.rs.initial_metadata, - grpc_mdelem_from_slices( - &exec_ctx, grpc_slice_intern(grpc_slice_from_static_string( - headers->headers[i].key)), - grpc_slice_intern( - grpc_slice_from_static_string(headers->headers[i].value)))); + GRPC_LOG_IF_ERROR( + "on_response_headers_received", + grpc_chttp2_incoming_metadata_buffer_add( + &exec_ctx, &s->state.rs.initial_metadata, + grpc_mdelem_from_slices( + &exec_ctx, grpc_slice_intern(grpc_slice_from_static_string( + headers->headers[i].key)), + grpc_slice_intern(grpc_slice_from_static_string( + headers->headers[i].value))))); } s->state.state_callback_received[OP_RECV_INITIAL_METADATA] = true; if (!(s->state.state_op_done[OP_CANCEL_ERROR] || @@ -549,17 +553,20 @@ static void on_response_trailers_received( memset(&s->state.rs.trailing_metadata, 0, sizeof(s->state.rs.trailing_metadata)); s->state.rs.trailing_metadata_valid = false; - grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.trailing_metadata); + grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.trailing_metadata, + s->arena); for (size_t i = 0; i < trailers->count; i++) { CRONET_LOG(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, trailers->headers[i].value); - grpc_chttp2_incoming_metadata_buffer_add( - &s->state.rs.trailing_metadata, - grpc_mdelem_from_slices( - &exec_ctx, grpc_slice_intern(grpc_slice_from_static_string( - trailers->headers[i].key)), - grpc_slice_intern( - grpc_slice_from_static_string(trailers->headers[i].value)))); + GRPC_LOG_IF_ERROR( + "on_response_trailers_received", + grpc_chttp2_incoming_metadata_buffer_add( + &exec_ctx, &s->state.rs.trailing_metadata, + grpc_mdelem_from_slices( + &exec_ctx, grpc_slice_intern(grpc_slice_from_static_string( + trailers->headers[i].key)), + grpc_slice_intern(grpc_slice_from_static_string( + trailers->headers[i].value))))); s->state.rs.trailing_metadata_valid = true; if (0 == strcmp(trailers->headers[i].key, "grpc-status") && 0 != strcmp(trailers->headers[i].value, "0")) { @@ -1174,7 +1181,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_stream_refcount *refcount, - const void *server_data) { + const void *server_data, gpr_arena *arena) { stream_obj *s = (stream_obj *)gs; memset(&s->storage, 0, sizeof(s->storage)); s->storage.head = NULL; @@ -1194,6 +1201,7 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->curr_gs = gs; s->curr_ct = (grpc_cronet_transport *)gt; + s->arena = arena; gpr_mu_init(&s->mu); return 0; @@ -1238,9 +1246,11 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, } static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, void *and_free_memory) { + grpc_stream *gs, + grpc_closure *then_schedule_closure) { stream_obj *s = (stream_obj *)gs; GRPC_ERROR_UNREF(s->state.cancel_error); + grpc_closure_sched(exec_ctx, then_schedule_closure, GRPC_ERROR_NONE); } static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {} diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 9a8870b86a4..f1285b923d0 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -438,8 +438,7 @@ static void release_call(grpc_exec_ctx *exec_ctx, void *call, grpc_error *error) { grpc_call *c = call; grpc_channel *channel = c->channel; - grpc_channel_update_call_size_estimate(channel, - gpr_arena_destroy(c->arena)); + grpc_channel_update_call_size_estimate(channel, gpr_arena_destroy(c->arena)); GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "call"); } From 527937031495287be5c301614055d1eb6a078241 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 15 Mar 2017 08:38:24 -0700 Subject: [PATCH 159/188] Fix compilation --- src/cpp/common/channel_filter.h | 3 ++- test/cpp/microbenchmarks/bm_call_create.cc | 26 +++++++++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/cpp/common/channel_filter.h b/src/cpp/common/channel_filter.h index 79c4bab985b..494d5d64d7f 100644 --- a/src/cpp/common/channel_filter.h +++ b/src/cpp/common/channel_filter.h @@ -318,7 +318,8 @@ class ChannelFilter final { static void DestroyCallElement(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *and_free_memory) { + grpc_closure *then_call_closure) { + GPR_ASSERT(then_call_closure == NULL); reinterpret_cast(elem->call_data)->~CallDataType(); } diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index 014e2b96b55..5ef40abb970 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -232,7 +232,7 @@ static void SetPollsetOrPollsetSet(grpc_exec_ctx *exec_ctx, static void DestroyCallElem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *and_free_memory) {} + grpc_closure *then_sched_closure) {} grpc_error *InitChannelElem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_channel_element_args *args) { @@ -275,7 +275,7 @@ const char *name; /* implementation of grpc_transport_init_stream */ int InitStream(grpc_exec_ctx *exec_ctx, grpc_transport *self, grpc_stream *stream, grpc_stream_refcount *refcount, - const void *server_data) { + const void *server_data, gpr_arena *arena) { return 0; } @@ -299,7 +299,7 @@ void PerformOp(grpc_exec_ctx *exec_ctx, grpc_transport *self, /* implementation of grpc_transport_destroy_stream */ void DestroyStream(grpc_exec_ctx *exec_ctx, grpc_transport *self, - grpc_stream *stream, void *and_free_memory) {} + grpc_stream *stream, grpc_closure *then_sched_closure) {} /* implementation of grpc_transport_destroy */ void Destroy(grpc_exec_ctx *exec_ctx, grpc_transport *self) {} @@ -394,7 +394,7 @@ static void BM_IsolatedFilter(benchmark::State &state) { grpc_channel_stack *channel_stack = static_cast(gpr_zalloc(channel_size)); GPR_ASSERT(GRPC_LOG_IF_ERROR( - "call_stack_init", + "channel_stack_init", grpc_channel_stack_init(&exec_ctx, 1, FilterDestroy, channel_stack, &filters[0], filters.size(), &channel_args, fixture.flags & REQUIRES_TRANSPORT @@ -409,15 +409,29 @@ static void BM_IsolatedFilter(benchmark::State &state) { grpc_slice method = grpc_slice_from_static_string("/foo/bar"); grpc_call_final_info final_info; TestOp test_op_data; + grpc_call_element_args call_args; + call_args.call_stack = call_stack; + call_args.server_transport_data = NULL; + call_args.context = NULL; + call_args.path = method; + call_args.start_time = start_time; + call_args.deadline = deadline; + const int kArenaSize = 4096; + call_args.arena = gpr_arena_create(kArenaSize); while (state.KeepRunning()) { GRPC_ERROR_UNREF(grpc_call_stack_init(&exec_ctx, channel_stack, 1, - DoNothing, NULL, NULL, NULL, method, - start_time, deadline, call_stack)); + DoNothing, NULL, &call_args)); typename TestOp::Op op(&exec_ctx, &test_op_data, call_stack); grpc_call_stack_destroy(&exec_ctx, call_stack, &final_info, NULL); op.Finish(&exec_ctx); grpc_exec_ctx_flush(&exec_ctx); + // recreate arena every 64k iterations to avoid oom + if (0 == (state.iterations() & 0xffff)) { + gpr_arena_destroy(call_args.arena); + call_args.arena = gpr_arena_create(kArenaSize); + } } + gpr_arena_destroy(call_args.arena); grpc_channel_stack_destroy(&exec_ctx, channel_stack); grpc_exec_ctx_finish(&exec_ctx); gpr_free(channel_stack); From 2ac189e60626b6dbcec48feca8d79614db00baee Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 15 Mar 2017 10:02:57 -0700 Subject: [PATCH 160/188] Fix proxy mapper initialization to work with third-party plugins. --- src/core/ext/client_channel/proxy_mapper_registry.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/core/ext/client_channel/proxy_mapper_registry.c b/src/core/ext/client_channel/proxy_mapper_registry.c index 2c44b9d4903..0935ddbdbd1 100644 --- a/src/core/ext/client_channel/proxy_mapper_registry.c +++ b/src/core/ext/client_channel/proxy_mapper_registry.c @@ -94,6 +94,14 @@ static void grpc_proxy_mapper_list_destroy(grpc_proxy_mapper_list* list) { grpc_proxy_mapper_destroy(list->list[i]); } gpr_free(list->list); + // Clean up in case we re-initialze later. + // TODO(ctiller): This should ideally live in + // grpc_proxy_mapper_registry_init(). However, if we did this there, + // then we would do it AFTER we start registering proxy mappers from + // third-party plugins, so they'd never show up (and would leak memory). + // We probably need some sort of dependency system for plugins to fix + // this. + memset(list, 0, sizeof(*list)); } // @@ -102,9 +110,7 @@ static void grpc_proxy_mapper_list_destroy(grpc_proxy_mapper_list* list) { static grpc_proxy_mapper_list g_proxy_mapper_list; -void grpc_proxy_mapper_registry_init() { - memset(&g_proxy_mapper_list, 0, sizeof(g_proxy_mapper_list)); -} +void grpc_proxy_mapper_registry_init() {} void grpc_proxy_mapper_registry_shutdown() { grpc_proxy_mapper_list_destroy(&g_proxy_mapper_list); From 39e2e09f20dcd55339a01cef4b7b95b2d6ff30c9 Mon Sep 17 00:00:00 2001 From: Yong Ni Date: Tue, 14 Mar 2017 13:50:57 -0700 Subject: [PATCH 161/188] created gcr upload script for version compatiblity tests. --- tools/gcp/utils/gcr_upload.py | 119 ++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100755 tools/gcp/utils/gcr_upload.py diff --git a/tools/gcp/utils/gcr_upload.py b/tools/gcp/utils/gcr_upload.py new file mode 100755 index 00000000000..b22f8731f68 --- /dev/null +++ b/tools/gcp/utils/gcr_upload.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python2.7 +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Upload docker images to Google Container Registry.""" + +from __future__ import print_function + +import argparse +import atexit +import os +import shutil +import subprocess +import tempfile + +argp = argparse.ArgumentParser(description='Run interop tests.') +argp.add_argument('--gcr_path', + default='gcr.io/grpc-testing', + help='Path of docker images in Google Container Registry') + +argp.add_argument('--gcr_tag', + default='latest', + help='the tag string for the images to upload') + +argp.add_argument('--with_files', + default=[], + nargs='+', + help='additional files to include in the docker image') + +argp.add_argument('--with_file_dest', + default='/var/local/image_info', + help='Destination directory for with_files inside docker image') + +argp.add_argument('--images', + default=[], + nargs='+', + help='local docker images in the form of repo:tag ' + + '(i.e. grpc_interop_java:26328ad8) to upload') + +argp.add_argument('--keep', + action='store_true', + help='keep the created local images after uploading to GCR') + + +args = argp.parse_args() + +def upload_to_gcr(image): + """Tags and Pushes a docker image in Google Containger Registry. + + image: docker image name, i.e. grpc_interop_java:26328ad8 + + A docker image image_foo:tag_old will be uploaded as + /image_foo: + after inserting extra with_files under with_file_dest in the image. The + original image name will be stored as label original_name:"image_foo:tag_old". + """ + tag_idx = image.find(':') + if tag_idx == -1: + print('Failed to parse docker image name %s' % image) + return False + new_tag = '%s/%s:%s' % (args.gcr_path, image[:tag_idx], args.gcr_tag) + + lines = ['FROM ' + image] + lines.append('LABEL original_name="%s"' % image) + + temp_dir = tempfile.mkdtemp() + atexit.register(lambda: subprocess.call(['rm', '-rf', temp_dir])) + + # Copy with_files inside the tmp directory, which will be the docker build + # context. + for f in args.with_files: + shutil.copy(f, temp_dir) + lines.append('COPY %s %s/' % (os.path.basename(f), args.with_file_dest)) + + # Create a Dockerfile. + with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as f: + f.write('\n'.join(lines)) + + build_cmd = ['docker', 'build', '--rm', '--tag', new_tag, temp_dir] + subprocess.check_output(build_cmd) + + if not args.keep: + atexit.register(lambda: subprocess.call(['docker', 'rmi', new_tag])) + + # Upload to GCR. + if args.gcr_path: + subprocess.call(['gcloud', 'docker', '--', 'push', new_tag]) + + return True + + +for image in args.images: + upload_to_gcr(image) From 1f8342933f17fab652cb00b38c3186ca57227a9d Mon Sep 17 00:00:00 2001 From: Robbie Shade Date: Fri, 10 Mar 2017 09:22:22 -0500 Subject: [PATCH 162/188] Add test feature flags --- test/core/end2end/end2end_tests.h | 7 +++++++ test/core/end2end/tests/network_status_change.c | 4 ++++ test/core/end2end/tests/resource_quota_server.c | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/test/core/end2end/end2end_tests.h b/test/core/end2end/end2end_tests.h index cb0afd9cd99..cdb26a67e9d 100644 --- a/test/core/end2end/end2end_tests.h +++ b/test/core/end2end/end2end_tests.h @@ -39,12 +39,15 @@ typedef struct grpc_end2end_test_fixture grpc_end2end_test_fixture; typedef struct grpc_end2end_test_config grpc_end2end_test_config; +/* Test feature flags. */ #define FEATURE_MASK_SUPPORTS_DELAYED_CONNECTION 1 #define FEATURE_MASK_SUPPORTS_HOSTNAME_VERIFICATION 2 #define FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS 4 #define FEATURE_MASK_SUPPORTS_REQUEST_PROXYING 8 #define FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL 16 #define FEATURE_MASK_SUPPORTS_AUTHORITY_HEADER 32 +#define FEATURE_MASK_DOES_NOT_SUPPORT_RESOURCE_QUOTA_SERVER 64 +#define FEATURE_MASK_DOES_NOT_SUPPORT_NETWORK_STATUS_CHANGE 128 #define FAIL_AUTH_CHECK_SERVER_ARG_NAME "fail_auth_check" @@ -56,8 +59,12 @@ struct grpc_end2end_test_fixture { }; struct grpc_end2end_test_config { + /* A descriptive name for this test fixture. */ const char *name; + + /* Which features are supported by this fixture. See feature flags above. */ uint32_t feature_mask; + grpc_end2end_test_fixture (*create_fixture)(grpc_channel_args *client_args, grpc_channel_args *server_args); void (*init_client)(grpc_end2end_test_fixture *f, diff --git a/test/core/end2end/tests/network_status_change.c b/test/core/end2end/tests/network_status_change.c index 7540ce93a1b..d7a4106459d 100644 --- a/test/core/end2end/tests/network_status_change.c +++ b/test/core/end2end/tests/network_status_change.c @@ -240,6 +240,10 @@ static void test_invoke_network_status_change(grpc_end2end_test_config config) { } void network_status_change(grpc_end2end_test_config config) { + if (config.feature_mask & + FEATURE_MASK_DOES_NOT_SUPPORT_NETWORK_STATUS_CHANGE) { + return; + } test_invoke_network_status_change(config); } diff --git a/test/core/end2end/tests/resource_quota_server.c b/test/core/end2end/tests/resource_quota_server.c index 4f9ed7a3a17..db26b4480e0 100644 --- a/test/core/end2end/tests/resource_quota_server.c +++ b/test/core/end2end/tests/resource_quota_server.c @@ -113,6 +113,10 @@ static grpc_slice generate_random_slice() { } void resource_quota_server(grpc_end2end_test_config config) { + if (config.feature_mask & + FEATURE_MASK_DOES_NOT_SUPPORT_RESOURCE_QUOTA_SERVER) { + return; + } grpc_resource_quota *resource_quota = grpc_resource_quota_create("test_server"); grpc_resource_quota_resize(resource_quota, 5 * 1024 * 1024); From 30ddc63fdf668f37d9e5c104eb9def44170dc883 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 21 Feb 2017 12:28:28 -0800 Subject: [PATCH 163/188] Suppress race caused by cached_size tracking in protobuf --- tools/tsan_suppressions.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/tsan_suppressions.txt b/tools/tsan_suppressions.txt index 765fa098f92..e0c79072288 100644 --- a/tools/tsan_suppressions.txt +++ b/tools/tsan_suppressions.txt @@ -6,6 +6,8 @@ race:cleanse_ctr race:ssleay_rand_add race:ssleay_rand_bytes race:__sleep_for -# protobuf has an idempotent write race in ByteSize +# protobuf has an idempotent write race in ByteSize/GetCachedSize # https://github.com/google/protobuf/issues/2169 race:ByteSize +race:ByteSizeLong +race:GetCachedSize From c7f0d6ac5ad1405f0094a4251a6320ae97eabfa0 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 15 Mar 2017 15:05:34 -0700 Subject: [PATCH 164/188] Increase test timeout for Linux portability --- tools/internal_ci/linux/grpc_portability.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_portability.cfg b/tools/internal_ci/linux/grpc_portability.cfg index 0c25661d624..5cc49f10462 100644 --- a/tools/internal_ci/linux/grpc_portability.cfg +++ b/tools/internal_ci/linux/grpc_portability.cfg @@ -31,7 +31,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_portability.sh" -timeout_mins: 360 +timeout_mins: 720 action { define_artifacts { regex: "**/*sponge_log.xml" From 488d41912567708ad4934ad35b41ac7275d65ec9 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Wed, 15 Mar 2017 23:09:52 +0000 Subject: [PATCH 165/188] Update health to current gRPC code elements --- src/python/grpcio_health_checking/grpc_health/v1/health.py | 3 ++- .../grpcio_tests/tests/health_check/_health_servicer_test.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/python/grpcio_health_checking/grpc_health/v1/health.py b/src/python/grpcio_health_checking/grpc_health/v1/health.py index f0f11cf84b0..e92c2659b7e 100644 --- a/src/python/grpcio_health_checking/grpc_health/v1/health.py +++ b/src/python/grpcio_health_checking/grpc_health/v1/health.py @@ -33,9 +33,10 @@ import threading import grpc from grpc_health.v1 import health_pb2 +from grpc_health.v1 import health_pb2_grpc -class HealthServicer(health_pb2.HealthServicer): +class HealthServicer(health_pb2_grpc.HealthServicer): """Servicer handling RPCs for service statuses.""" def __init__(self): diff --git a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py index 363b4c5f994..1bc8669dad9 100644 --- a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py +++ b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py @@ -34,6 +34,7 @@ import grpc from grpc.framework.foundation import logging_pool from grpc_health.v1 import health from grpc_health.v1 import health_pb2 +from grpc_health.v1 import health_pb2_grpc from tests.unit.framework.common import test_constants @@ -52,11 +53,11 @@ class HealthServicerTest(unittest.TestCase): server_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY) self._server = grpc.server(server_pool) port = self._server.add_insecure_port('[::]:0') - health_pb2.add_HealthServicer_to_server(servicer, self._server) + health_pb2_grpc.add_HealthServicer_to_server(servicer, self._server) self._server.start() channel = grpc.insecure_channel('localhost:%d' % port) - self._stub = health_pb2.HealthStub(channel) + self._stub = health_pb2_grpc.HealthStub(channel) def test_empty_service(self): request = health_pb2.HealthCheckRequest() From 94a4ed93a1bd88a29886bcc377e1eba48013c7ae Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 15 Mar 2017 17:15:39 -0700 Subject: [PATCH 166/188] Fix the error in flush read that was causing jenkins timeout --- src/core/ext/transport/cronet/transport/cronet_transport.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index fabfaf8a270..6499eff3e93 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -62,7 +62,7 @@ } while (0) /* TODO (makdharma): Hook up into the wider tracing mechanism */ -int grpc_cronet_trace = 0; +int grpc_cronet_trace = 1; enum e_op_result { ACTION_TAKEN_WITH_CALLBACK, @@ -272,7 +272,7 @@ static void maybe_flush_read(stream_obj *s) { /* Whenever the evaluation of any of the two condition is changed, we check * whether we should enter the flush read state. */ if (s->state.pending_recv_trailing_metadata && s->state.fail_state) { - if (!s->state.flush_read) { + if (!s->state.flush_read && !s->state.rs.read_stream_closed) { CRONET_LOG(GPR_DEBUG, "%p: Flush read", s); s->state.flush_read = true; null_and_maybe_free_read_buffer(s); From 98e282a58519e3854973a1486aaba3d90f15a866 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 17 Feb 2017 18:05:23 -0800 Subject: [PATCH 167/188] Eliminate std::placeholders --- test/cpp/qps/server_async.cc | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index b3a06aeaf53..b58d91eea63 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -103,24 +103,25 @@ class AsyncQpsServerTest final : public grpc::testing::Server { server_ = builder.BuildAndStart(); - using namespace std::placeholders; - auto process_rpc_bound = - std::bind(process_rpc, config.payload_config(), _1, _2); + std::bind(process_rpc, config.payload_config(), std::placeholders::_1, + std::placeholders::_2); for (int i = 0; i < 15000; i++) { for (int j = 0; j < num_threads; j++) { if (request_unary_function) { - auto request_unary = - std::bind(request_unary_function, &async_service_, _1, _2, _3, - srv_cqs_[j].get(), srv_cqs_[j].get(), _4); + auto request_unary = std::bind( + request_unary_function, &async_service_, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3, srv_cqs_[j].get(), + srv_cqs_[j].get(), std::placeholders::_4); contexts_.emplace_back( new ServerRpcContextUnaryImpl(request_unary, process_rpc_bound)); } if (request_streaming_function) { - auto request_streaming = - std::bind(request_streaming_function, &async_service_, _1, _2, - srv_cqs_[j].get(), srv_cqs_[j].get(), _3); + auto request_streaming = std::bind( + request_streaming_function, &async_service_, + std::placeholders::_1, std::placeholders::_2, srv_cqs_[j].get(), + srv_cqs_[j].get(), std::placeholders::_3); contexts_.emplace_back(new ServerRpcContextStreamingImpl( request_streaming, process_rpc_bound)); } From 364fa33576bc09407b37b3ecca43f7b26d4f7c21 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 16 Mar 2017 08:00:36 -0700 Subject: [PATCH 168/188] Override subchannel address in channel args when proxy mapper is used. --- src/core/ext/client_channel/subchannel.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/core/ext/client_channel/subchannel.c b/src/core/ext/client_channel/subchannel.c index 5df0a9060d0..2528941981e 100644 --- a/src/core/ext/client_channel/subchannel.c +++ b/src/core/ext/client_channel/subchannel.c @@ -340,17 +340,15 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, GPR_ASSERT(new_address != NULL); gpr_free(addr); addr = new_address; - if (new_args != NULL) c->args = new_args; - } - if (c->args == NULL) { - static const char *keys_to_remove[] = {GRPC_ARG_SUBCHANNEL_ADDRESS}; - grpc_arg new_arg = grpc_create_subchannel_address_arg(addr); - c->args = grpc_channel_args_copy_and_add_and_remove( - args->args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), &new_arg, - 1); - gpr_free(new_arg.value.string); } + static const char *keys_to_remove[] = {GRPC_ARG_SUBCHANNEL_ADDRESS}; + grpc_arg new_arg = grpc_create_subchannel_address_arg(addr); gpr_free(addr); + c->args = grpc_channel_args_copy_and_add_and_remove( + new_args != NULL ? new_args : args->args, keys_to_remove, + GPR_ARRAY_SIZE(keys_to_remove), &new_arg, 1); + gpr_free(new_arg.value.string); + if (new_args != NULL) grpc_channel_args_destroy(exec_ctx, new_args); c->root_external_state_watcher.next = c->root_external_state_watcher.prev = &c->root_external_state_watcher; grpc_closure_init(&c->connected, subchannel_connected, c, From 1292906a7cbbc21748d356e9317bdc83817ab043 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 16 Mar 2017 09:36:58 -0700 Subject: [PATCH 169/188] Disable cronet trace --- src/core/ext/transport/cronet/transport/cronet_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 6499eff3e93..e1873603306 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -62,7 +62,7 @@ } while (0) /* TODO (makdharma): Hook up into the wider tracing mechanism */ -int grpc_cronet_trace = 1; +int grpc_cronet_trace = 0; enum e_op_result { ACTION_TAKEN_WITH_CALLBACK, From 2ccd502ed4e3fffa2cccf7d3e0aa7103caf37bf0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 16 Mar 2017 09:57:46 -0700 Subject: [PATCH 170/188] Add cast --- src/core/lib/surface/channel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index 56beb7d96e6..2b700b2f675 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -119,7 +119,7 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target, gpr_atm_no_barrier_store( &channel->call_size_estimate, - CHANNEL_STACK_FROM_CHANNEL(channel)->call_stack_size); + (gpr_atm)CHANNEL_STACK_FROM_CHANNEL(channel)->call_stack_size); grpc_compression_options_init(&channel->compression_options); for (size_t i = 0; i < args->num_args; i++) { From 0e0149d3f3fecd7d899700647221fbe30a99f77c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 16 Mar 2017 10:20:28 -0700 Subject: [PATCH 171/188] Restore missing benchmarks --- .../bm_fullstack_streaming_ping_pong.cc | 229 ++++++++++++------ 1 file changed, 156 insertions(+), 73 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc index dc0e7d769ab..513974fde19 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc @@ -54,86 +54,144 @@ auto& force_library_initialization = Library::get(); static void* tag(intptr_t x) { return reinterpret_cast(x); } -template -static void BM_PumpStreamClientToServer(benchmark::State& state) { +// Repeatedly makes Streaming Bidi calls (exchanging a configurable number of +// messages in each call) in a loop on a single channel +// +// First parmeter (i.e state.range(0)): Message size (in bytes) to use +// Second parameter (i.e state.range(1)): Number of ping pong messages. +// Note: One ping-pong means two messages (one from client to server and +// the other from server to client): +template +static void BM_StreamingPingPong(benchmark::State& state) { + TrackCounters track_counters; + const int msg_size = state.range(0); + const int max_ping_pongs = state.range(1); + EchoTestService::AsyncService service; std::unique_ptr fixture(new Fixture(&service)); { + EchoResponse send_response; + EchoResponse recv_response; EchoRequest send_request; EchoRequest recv_request; - if (state.range(0) > 0) { - send_request.set_message(std::string(state.range(0), 'a')); + + if (msg_size > 0) { + send_request.set_message(std::string(msg_size, 'a')); + send_response.set_message(std::string(msg_size, 'b')); } - Status recv_status; - ServerContext svr_ctx; - ServerAsyncReaderWriter response_rw(&svr_ctx); - service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), - fixture->cq(), tag(0)); + std::unique_ptr stub( EchoTestService::NewStub(fixture->channel())); - ClientContext cli_ctx; - auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); - void* t; - bool ok; - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - GPR_ASSERT(ok); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); - } - response_rw.Read(&recv_request, tag(0)); + while (state.KeepRunning()) { - GPR_TIMER_SCOPE("BenchmarkCycle", 0); - request_rw->Write(send_request, tag(1)); - while (true) { + ServerContext svr_ctx; + ServerContextMutator svr_ctx_mut(&svr_ctx); + ServerAsyncReaderWriter response_rw(&svr_ctx); + service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), + fixture->cq(), tag(0)); + + ClientContext cli_ctx; + ClientContextMutator cli_ctx_mut(&cli_ctx); + auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); + + // Establish async stream between client side and server side + void* t; + bool ok; + int need_tags = (1 << 0) | (1 << 1); + while (need_tags) { GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - if (t == tag(0)) { - response_rw.Read(&recv_request, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + + // Send 'max_ping_pongs' number of ping pong messages + int ping_pong_cnt = 0; + while (ping_pong_cnt < max_ping_pongs) { + request_rw->Write(send_request, tag(0)); // Start client send + response_rw.Read(&recv_request, tag(1)); // Start server recv + request_rw->Read(&recv_response, tag(2)); // Start client recv + + need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + + // If server recv is complete, start the server send operation + if (i == 1) { + response_rw.Write(send_response, tag(3)); + } + + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); } + + ping_pong_cnt++; } - } - request_rw->WritesDone(tag(1)); - need_tags = (1 << 0) | (1 << 1); - while (need_tags) { - GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - int i = (int)(intptr_t)t; - GPR_ASSERT(need_tags & (1 << i)); - need_tags &= ~(1 << i); + + request_rw->WritesDone(tag(0)); + response_rw.Finish(Status::OK, tag(1)); + + Status recv_status; + request_rw->Finish(&recv_status, tag(2)); + + need_tags = (1 << 0) | (1 << 1) | (1 << 2); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + + GPR_ASSERT(recv_status.ok()); } } + fixture->Finish(state); fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); + state.SetBytesProcessed(msg_size * state.iterations() * max_ping_pongs * 2); + track_counters.Finish(state); } -template -static void BM_PumpStreamServerToClient(benchmark::State& state) { +// Repeatedly sends ping pong messages in a single streaming Bidi call in a loop +// First parmeter (i.e state.range(0)): Message size (in bytes) to use +template +static void BM_StreamingPingPongMsgs(benchmark::State& state) { + TrackCounters track_counters; + const int msg_size = state.range(0); + EchoTestService::AsyncService service; std::unique_ptr fixture(new Fixture(&service)); { EchoResponse send_response; EchoResponse recv_response; - if (state.range(0) > 0) { - send_response.set_message(std::string(state.range(0), 'a')); + EchoRequest send_request; + EchoRequest recv_request; + + if (msg_size > 0) { + send_request.set_message(std::string(msg_size, 'a')); + send_response.set_message(std::string(msg_size, 'b')); } - Status recv_status; + + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + ServerContext svr_ctx; + ServerContextMutator svr_ctx_mut(&svr_ctx); ServerAsyncReaderWriter response_rw(&svr_ctx); service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(), fixture->cq(), tag(0)); - std::unique_ptr stub( - EchoTestService::NewStub(fixture->channel())); + ClientContext cli_ctx; + ClientContextMutator cli_ctx_mut(&cli_ctx); auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1)); - int need_tags = (1 << 0) | (1 << 1); + + // Establish async stream between client side and server side void* t; bool ok; + int need_tags = (1 << 0) | (1 << 1); while (need_tags) { GPR_ASSERT(fixture->cq()->Next(&t, &ok)); GPR_ASSERT(ok); @@ -141,54 +199,79 @@ static void BM_PumpStreamServerToClient(benchmark::State& state) { GPR_ASSERT(need_tags & (1 << i)); need_tags &= ~(1 << i); } - request_rw->Read(&recv_response, tag(0)); + while (state.KeepRunning()) { GPR_TIMER_SCOPE("BenchmarkCycle", 0); - response_rw.Write(send_response, tag(1)); - while (true) { + request_rw->Write(send_request, tag(0)); // Start client send + response_rw.Read(&recv_request, tag(1)); // Start server recv + request_rw->Read(&recv_response, tag(2)); // Start client recv + + need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3); + while (need_tags) { GPR_ASSERT(fixture->cq()->Next(&t, &ok)); - if (t == tag(0)) { - request_rw->Read(&recv_response, tag(0)); - } else if (t == tag(1)) { - break; - } else { - GPR_ASSERT(false); + GPR_ASSERT(ok); + int i = (int)(intptr_t)t; + + // If server recv is complete, start the server send operation + if (i == 1) { + response_rw.Write(send_response, tag(3)); } + + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); } } + + request_rw->WritesDone(tag(0)); response_rw.Finish(Status::OK, tag(1)); - need_tags = (1 << 0) | (1 << 1); + Status recv_status; + request_rw->Finish(&recv_status, tag(2)); + + need_tags = (1 << 0) | (1 << 1) | (1 << 2); while (need_tags) { GPR_ASSERT(fixture->cq()->Next(&t, &ok)); int i = (int)(intptr_t)t; GPR_ASSERT(need_tags & (1 << i)); need_tags &= ~(1 << i); } + + GPR_ASSERT(recv_status.ok()); } + fixture->Finish(state); fixture.reset(); - state.SetBytesProcessed(state.range(0) * state.iterations()); + state.SetBytesProcessed(msg_size * state.iterations() * 2); + track_counters.Finish(state); } /******************************************************************************* * CONFIGURATIONS */ -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS) - ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) +// Generate Args for StreamingPingPong benchmarks. Currently generates args for +// only "small streams" (i.e streams with 0, 1 or 2 messages) +static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) { + int msg_size = 0; + + b->Args({0, 0}); // spl case: 0 ping-pong msgs (msg_size doesn't matter here) + + for (msg_size = 0; msg_size <= 128 * 1024 * 1024; + msg_size == 0 ? msg_size++ : msg_size *= 8) { + b->Args({msg_size, 1}); + b->Args({msg_size, 2}); + } +} + +BENCHMARK_TEMPLATE(BM_StreamingPingPong, InProcessCHTTP2, NoOpMutator, + NoOpMutator) + ->Apply(StreamingPingPongArgs); +BENCHMARK_TEMPLATE(BM_StreamingPingPong, TCP, NoOpMutator, NoOpMutator) + ->Apply(StreamingPingPongArgs); + +BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, InProcessCHTTP2, NoOpMutator, + NoOpMutator) ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) +BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, TCP, NoOpMutator, NoOpMutator) ->Range(0, 128 * 1024 * 1024); } // namespace testing From 06d4cbce2d8127e9942910830f78472cce13881e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 16 Mar 2017 13:35:58 -0700 Subject: [PATCH 172/188] Fix BUILD --- BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/BUILD b/BUILD index ca0a1c56076..47017bf8f0c 100644 --- a/BUILD +++ b/BUILD @@ -308,6 +308,7 @@ grpc_cc_library( srcs = [ "src/core/lib/profiling/basic_timers.c", "src/core/lib/profiling/stap_timers.c", + "src/core/lib/support/arena.c", "src/core/lib/support/alloc.c", "src/core/lib/support/avl.c", "src/core/lib/support/backoff.c", @@ -352,6 +353,7 @@ grpc_cc_library( "src/core/lib/support/wrap_memcpy.c", ], hdrs = [ + "src/core/lib/support/arena.h", "src/core/lib/profiling/timers.h", "src/core/lib/support/backoff.h", "src/core/lib/support/block_annotate.h", From 9df13b26d2ce0f14b0499b45e5aa01c48108e3f5 Mon Sep 17 00:00:00 2001 From: yang-g Date: Thu, 16 Mar 2017 14:14:18 -0700 Subject: [PATCH 173/188] Add missing files --- BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/BUILD b/BUILD index ca0a1c56076..5a2e7a72e9e 100644 --- a/BUILD +++ b/BUILD @@ -1132,6 +1132,7 @@ grpc_cc_library( "src/cpp/common/rpc_method.cc", "src/cpp/common/version_cc.cc", "src/cpp/server/async_generic_service.cc", + "src/cpp/server/channel_argument_option.cc", "src/cpp/server/create_default_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.cc", "src/cpp/server/health/default_health_check_service.cc", @@ -1173,6 +1174,7 @@ grpc_cc_library( "include/grpc++/grpc++.h", "include/grpc++/health_check_service_interface.h", "include/grpc++/impl/call.h", + "include/grpc++/impl/channel_argument_option.h", "include/grpc++/impl/client_unary_call.h", "include/grpc++/impl/codegen/core_codegen.h", "include/grpc++/impl/grpc_library.h", From 5ae42a15cfe0c99711da4865541eb6e6e45dbd48 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 10 Mar 2017 12:04:49 -0800 Subject: [PATCH 174/188] stop http2 test server with error code when failures occur --- test/http2_test/http2_test_server.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/test/http2_test/http2_test_server.py b/test/http2_test/http2_test_server.py index abde3433ad2..81ee1df9f61 100644 --- a/test/http2_test/http2_test_server.py +++ b/test/http2_test/http2_test_server.py @@ -31,6 +31,7 @@ import argparse import logging +import sys import twisted import twisted.internet import twisted.internet.endpoints @@ -55,7 +56,7 @@ _TEST_CASE_MAPPING = { class H2Factory(twisted.internet.protocol.Factory): def __init__(self, testcase): - logging.info('Creating H2Factory for new connection.') + logging.info('Creating H2Factory for new connection (%s)', testcase) self._num_streams = 0 self._testcase = testcase @@ -83,6 +84,19 @@ def parse_arguments(): ) return parser.parse_args() +exit_code = 0 + +def listen(endpoint, test_case): + deferred = endpoint.listen(H2Factory(test_case)) + def listen_error(reason): + # If listening fails, we stop the reactor and exit the program + # with exit_code = 1. + global exit_code + exit_code = 1 + logging.error('Listening failed: %s' % reason.value) + twisted.internet.reactor.stop() + deferred.addErrback(listen_error) + def start_test_servers(base_port): """ Start one server per test case on incrementing port numbers beginning with base_port """ @@ -92,7 +106,9 @@ def start_test_servers(base_port): logging.warning('serving on port %d : %s'%(portnum, test_case)) endpoint = twisted.internet.endpoints.TCP4ServerEndpoint( twisted.internet.reactor, portnum, backlog=128) - endpoint.listen(H2Factory(test_case)) + # Wait until the reactor is running before calling endpoint.listen(). + twisted.internet.reactor.callWhenRunning(listen, endpoint, test_case) + index += 1 if __name__ == '__main__': @@ -102,3 +118,4 @@ if __name__ == '__main__': args = parse_arguments() start_test_servers(args.base_port) twisted.internet.reactor.run() + sys.exit(exit_code) From 69e2f8216978c4a9293cd5e9f0dbda91e404599e Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 10 Mar 2017 12:21:05 -0800 Subject: [PATCH 175/188] add _ --- test/http2_test/http2_test_server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/http2_test/http2_test_server.py b/test/http2_test/http2_test_server.py index 81ee1df9f61..85c356b5ef1 100644 --- a/test/http2_test/http2_test_server.py +++ b/test/http2_test/http2_test_server.py @@ -54,6 +54,8 @@ _TEST_CASE_MAPPING = { 'max_streams': test_max_streams.TestcaseSettingsMaxStreams, } +_exit_code = 0 + class H2Factory(twisted.internet.protocol.Factory): def __init__(self, testcase): logging.info('Creating H2Factory for new connection (%s)', testcase) @@ -84,14 +86,12 @@ def parse_arguments(): ) return parser.parse_args() -exit_code = 0 - def listen(endpoint, test_case): deferred = endpoint.listen(H2Factory(test_case)) def listen_error(reason): # If listening fails, we stop the reactor and exit the program # with exit_code = 1. - global exit_code + global _exit_code exit_code = 1 logging.error('Listening failed: %s' % reason.value) twisted.internet.reactor.stop() @@ -118,4 +118,4 @@ if __name__ == '__main__': args = parse_arguments() start_test_servers(args.base_port) twisted.internet.reactor.run() - sys.exit(exit_code) + sys.exit(_exit_code) From 95f48c154b1073393fae4d412f4c905194e7f96c Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 10 Mar 2017 14:01:11 -0800 Subject: [PATCH 176/188] really add _ --- test/http2_test/http2_test_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/http2_test/http2_test_server.py b/test/http2_test/http2_test_server.py index 85c356b5ef1..46c3e00d18f 100644 --- a/test/http2_test/http2_test_server.py +++ b/test/http2_test/http2_test_server.py @@ -90,9 +90,9 @@ def listen(endpoint, test_case): deferred = endpoint.listen(H2Factory(test_case)) def listen_error(reason): # If listening fails, we stop the reactor and exit the program - # with exit_code = 1. + # with exit code 1. global _exit_code - exit_code = 1 + _exit_code = 1 logging.error('Listening failed: %s' % reason.value) twisted.internet.reactor.stop() deferred.addErrback(listen_error) From 61335ec086b9c40c5c0d099f53dde1b602db5955 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Mon, 13 Mar 2017 13:13:30 -0700 Subject: [PATCH 177/188] use docker port mapping for http2 server --- tools/run_tests/run_interop_tests.py | 30 ++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 29d5a9aaa00..f51088e237b 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -636,10 +636,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, '--server_host=%s' % server_host, ] if test_case in _HTTP2_BADSERVER_TEST_CASES: - # We are running the http2_badserver_interop test. Adjust command line accordingly. - offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case) - client_options = common_options + ['--server_port=%s' % - (int(server_port)+offset)] + client_options = common_options + ['--server_port=%s' % server_port] cmdline = bash_cmdline(language.client_cmd_http2interop(client_options)) cwd = language.http2_cwd else: @@ -685,18 +682,11 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): if language.safename == 'http2': # we are running the http2 interop server. Open next N ports beginning # with the server port. These ports are used for http2 interop test - # (one test case per port). We also attach the docker container running - # the server to local network, so we don't have to mess with port mapping - port_args = [ - '-p', str(_DEFAULT_SERVER_PORT+0), - '-p', str(_DEFAULT_SERVER_PORT+1), - '-p', str(_DEFAULT_SERVER_PORT+2), - '-p', str(_DEFAULT_SERVER_PORT+3), - '-p', str(_DEFAULT_SERVER_PORT+4), - '-p', str(_DEFAULT_SERVER_PORT+5), - '-p', str(_DEFAULT_SERVER_PORT+6), - '--net=host', - ] + # (one test case per port). + port_args = list( + itertools.chain.from_iterable(('-p', str(_DEFAULT_SERVER_PORT + i)) + for i in range( + len(_HTTP2_BADSERVER_TEST_CASES)))) else: port_args = ['-p', str(_DEFAULT_SERVER_PORT)] @@ -936,6 +926,7 @@ client_manual_cmd_log = [] if args.manual_run else None # Start interop servers. server_jobs = {} server_addresses = {} +http2_badserver_ports = () try: for s in servers: lang = str(s) @@ -957,6 +948,10 @@ try: if not args.manual_run: job = dockerjob.DockerJob(spec) server_jobs[lang] = job + http2_badserver_ports = tuple([ + job.mapped_port(_DEFAULT_SERVER_PORT + i) + for i in range(len(_HTTP2_BADSERVER_TEST_CASES)) + ]) else: # don't run the server, set server port to a placeholder value server_addresses[lang] = ('localhost', '${SERVER_PORT}') @@ -1043,11 +1038,12 @@ try: if args.http2_badserver_interop: for language in languages_http2_badserver_interop: for test_case in _HTTP2_BADSERVER_TEST_CASES: + offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case) test_job = cloud_to_cloud_jobspec(language, test_case, str(http2InteropServer), 'localhost', - _DEFAULT_SERVER_PORT, + http2_badserver_ports[offset], docker_image=docker_images.get(str(language)), manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) From bee6ec2473f56227faef3cd7559a19137ad5b15e Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Mon, 13 Mar 2017 15:40:51 -0700 Subject: [PATCH 178/188] refactor common_options, correct output for manual run --- tools/run_tests/run_interop_tests.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index f51088e237b..ac57e5a9c50 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -634,14 +634,13 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, common_options = [ '--test_case=%s' % test_case, '--server_host=%s' % server_host, + '--server_port=%s' % server_port, ] if test_case in _HTTP2_BADSERVER_TEST_CASES: - client_options = common_options + ['--server_port=%s' % server_port] - cmdline = bash_cmdline(language.client_cmd_http2interop(client_options)) + cmdline = bash_cmdline(language.client_cmd_http2interop(common_options)) cwd = language.http2_cwd else: - client_options = interop_only_options + common_options + ['--server_port=%s' % server_port] - cmdline = bash_cmdline(language.client_cmd(client_options)) + cmdline = bash_cmdline(language.client_cmd(common_options+interop_only_options)) cwd = language.client_cwd environ = language.global_env() @@ -1039,11 +1038,15 @@ try: for language in languages_http2_badserver_interop: for test_case in _HTTP2_BADSERVER_TEST_CASES: offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case) + if not args.manual_run: + server_port = http2_badserver_ports[offset] + else: + server_port = _DEFAULT_SERVER_PORT+offset test_job = cloud_to_cloud_jobspec(language, test_case, str(http2InteropServer), 'localhost', - http2_badserver_ports[offset], + server_port, docker_image=docker_images.get(str(language)), manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) From 22afddf53c1de182082a022ee108f27db117d2d9 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Wed, 15 Mar 2017 15:43:43 -0700 Subject: [PATCH 179/188] add health check for http2 server --- test/http2_test/http2_server_health_check.py | 49 +++++++++++++++++++ .../interoptest/grpc_interop_http2/Dockerfile | 2 +- tools/run_tests/python_utils/dockerjob.py | 19 +++++++ tools/run_tests/run_interop_tests.py | 21 ++++++-- 4 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 test/http2_test/http2_server_health_check.py diff --git a/test/http2_test/http2_server_health_check.py b/test/http2_test/http2_server_health_check.py new file mode 100644 index 00000000000..dd9402b8557 --- /dev/null +++ b/test/http2_test/http2_server_health_check.py @@ -0,0 +1,49 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import hyper +import sys + +# Utility to healthcheck the http2 server. Used when starting the server to +# verify that the server is live before tests begin. +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--server_host', type=str, default='localhost') + parser.add_argument('--server_port', type=int, default=8080) + args = parser.parse_args() + server_host = args.server_host + server_port = args.server_port + conn = hyper.HTTP20Connection('%s:%d' % (server_host, server_port)) + conn.request('POST', '/grpc.testing.TestService/UnaryCall') + resp = conn.get_response() + if resp.headers.get('grpc-encoding') is None: + sys.exit(1) + else: + sys.exit(0) diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 66d9b4f6403..acc984acb0d 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -47,7 +47,7 @@ RUN pip install pip --upgrade RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 -RUN pip install twisted h2 +RUN pip install twisted h2 hyper # Define the default command. CMD ["bash"] diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py index 0869c5cee9b..82466d3a93f 100755 --- a/tools/run_tests/python_utils/dockerjob.py +++ b/tools/run_tests/python_utils/dockerjob.py @@ -70,6 +70,22 @@ def docker_mapped_port(cid, port, timeout_seconds=15): (port, cid)) +def wait_for_healthy(cid, shortname, timeout_seconds): + """Wait timeout_seconds for the container to become healthy""" + started = time.time() + while time.time() - started < timeout_seconds: + try: + output = subprocess.check_output( + ['docker', 'inspect', '--format="{{.State.Health.Status}}"', cid], + stderr=_DEVNULL) + if output.strip('\n') == 'healthy': + return + except subprocess.CalledProcessError as e: + pass + raise Exception('Timed out waiting for %s (%s) to pass health check' % + (shortname, cid)) + + def finish_jobs(jobs): """Kills given docker containers and waits for corresponding jobs to finish""" for job in jobs: @@ -113,6 +129,9 @@ class DockerJob: def mapped_port(self, port): return docker_mapped_port(self._container_name, port) + def wait_for_healthy(self, timeout_seconds=15): + wait_for_healthy(self._container_name, self._spec.shortname, timeout_seconds) + def kill(self, suppress_failure=False): """Sends kill signal to the container.""" if suppress_failure: diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index ac57e5a9c50..fb10a442d0b 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -678,23 +678,32 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT, '--use_tls=%s' % ('false' if insecure else 'true')])) environ = language.global_env() + docker_args = ['--name=%s' % container_name] if language.safename == 'http2': # we are running the http2 interop server. Open next N ports beginning # with the server port. These ports are used for http2 interop test # (one test case per port). - port_args = list( + docker_args += list( itertools.chain.from_iterable(('-p', str(_DEFAULT_SERVER_PORT + i)) for i in range( len(_HTTP2_BADSERVER_TEST_CASES)))) + docker_args += [ + '--health-cmd=python test/http2_test/http2_server_health_check.py ' + '--server_host=%s --server_port=%d' + % ('localhost', _DEFAULT_SERVER_PORT), + '--health-interval=1s', + '--health-retries=5', + '--health-timeout=1s', + ] + else: - port_args = ['-p', str(_DEFAULT_SERVER_PORT)] + docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] docker_cmdline = docker_run_cmdline(cmdline, image=docker_image, cwd=language.server_cwd, environ=environ, - docker_args=port_args + - ['--name=%s' % container_name]) + docker_args=docker_args) if manual_cmd_log is not None: manual_cmd_log.append(manual_cmdline(docker_cmdline)) server_job = jobset.JobSpec( @@ -881,7 +890,8 @@ languages = set(_LANGUAGES[l] languages_http2_badserver_interop = set() if args.http2_badserver_interop: languages_http2_badserver_interop = set( - _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS) + _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS + if 'all' in args.language or l in args.language) http2Interop = Http2Client() if args.http2_interop else None http2InteropServer = Http2Server() if args.http2_badserver_interop else None @@ -946,6 +956,7 @@ try: manual_cmd_log=server_manual_cmd_log) if not args.manual_run: job = dockerjob.DockerJob(spec) + job.wait_for_healthy(timeout_seconds=15) server_jobs[lang] = job http2_badserver_ports = tuple([ job.mapped_port(_DEFAULT_SERVER_PORT + i) From d3b160010e220e7c95dde1f81428ed1654ff4570 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 16 Mar 2017 07:28:25 -0700 Subject: [PATCH 180/188] add comments for health check docker flags --- tools/run_tests/run_interop_tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index fb10a442d0b..48f297804c1 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -687,6 +687,14 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): itertools.chain.from_iterable(('-p', str(_DEFAULT_SERVER_PORT + i)) for i in range( len(_HTTP2_BADSERVER_TEST_CASES)))) + # Enable docker's healthcheck mechanism. + # This runs a Python script inside the container every second. The script + # pings the http2 server to verify it is ready. The 'health-retries' flag + # specifies the number of consecutive failures before docker will report + # the container's status as 'unhealthy'. Prior to the first 'health_retries' + # failures or the first success, the status will be 'starting'. 'docker ps' + # or 'docker inspect' can be used to see the health of the container on the + # command line. docker_args += [ '--health-cmd=python test/http2_test/http2_server_health_check.py ' '--server_host=%s --server_port=%d' From d86a19efd0bb011fbadc304434c2512c4607f8b4 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 16 Mar 2017 07:34:39 -0700 Subject: [PATCH 181/188] check error for docker inspect --- tools/run_tests/python_utils/dockerjob.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py index 82466d3a93f..cdf5759f8c9 100755 --- a/tools/run_tests/python_utils/dockerjob.py +++ b/tools/run_tests/python_utils/dockerjob.py @@ -76,8 +76,9 @@ def wait_for_healthy(cid, shortname, timeout_seconds): while time.time() - started < timeout_seconds: try: output = subprocess.check_output( - ['docker', 'inspect', '--format="{{.State.Health.Status}}"', cid], - stderr=_DEVNULL) + ['docker', 'inspect', '--format="{{.State.Health.Status}}"', cid]) + #stderr=_DEVNULL) + print(output) if output.strip('\n') == 'healthy': return except subprocess.CalledProcessError as e: From 92a5d28a4c0afc4cfd1f7eaba5d876a53a314986 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 16 Mar 2017 07:53:50 -0700 Subject: [PATCH 182/188] increase wait for healthy timeout --- tools/run_tests/python_utils/dockerjob.py | 4 +++- tools/run_tests/run_interop_tests.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py index cdf5759f8c9..a29555428bc 100755 --- a/tools/run_tests/python_utils/dockerjob.py +++ b/tools/run_tests/python_utils/dockerjob.py @@ -83,6 +83,8 @@ def wait_for_healthy(cid, shortname, timeout_seconds): return except subprocess.CalledProcessError as e: pass + time.sleep(1) + print(subprocess.check_output(['docker', 'ps'])) raise Exception('Timed out waiting for %s (%s) to pass health check' % (shortname, cid)) @@ -130,7 +132,7 @@ class DockerJob: def mapped_port(self, port): return docker_mapped_port(self._container_name, port) - def wait_for_healthy(self, timeout_seconds=15): + def wait_for_healthy(self, timeout_seconds): wait_for_healthy(self._container_name, self._spec.shortname, timeout_seconds) def kill(self, suppress_failure=False): diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 48f297804c1..b5977bf4f3a 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -964,7 +964,7 @@ try: manual_cmd_log=server_manual_cmd_log) if not args.manual_run: job = dockerjob.DockerJob(spec) - job.wait_for_healthy(timeout_seconds=15) + job.wait_for_healthy(timeout_seconds=240) server_jobs[lang] = job http2_badserver_ports = tuple([ job.mapped_port(_DEFAULT_SERVER_PORT + i) From f9d0804daadf3cc22d406eaf6d603a3dda9e0460 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 16 Mar 2017 09:06:49 -0700 Subject: [PATCH 183/188] disable wait for healthy --- tools/run_tests/run_interop_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index b5977bf4f3a..55e729ace3c 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -964,7 +964,8 @@ try: manual_cmd_log=server_manual_cmd_log) if not args.manual_run: job = dockerjob.DockerJob(spec) - job.wait_for_healthy(timeout_seconds=240) + #job.wait_for_healthy(timeout_seconds=240) + print(subprocess.check_output(['docker', 'version'])) server_jobs[lang] = job http2_badserver_ports = tuple([ job.mapped_port(_DEFAULT_SERVER_PORT + i) From 6ce6166bd0451db14893b896384517bb95fbfea3 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 16 Mar 2017 10:27:00 -0700 Subject: [PATCH 184/188] reenable health check, disable other interop tests --- tools/jenkins/run_interop.sh | 2 +- tools/run_tests/run_interop_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/jenkins/run_interop.sh b/tools/jenkins/run_interop.sh index 2a9fc662a9e..18e8b644381 100755 --- a/tools/jenkins/run_interop.sh +++ b/tools/jenkins/run_interop.sh @@ -36,4 +36,4 @@ export LANG=en_US.UTF-8 # Enter the gRPC repo root cd $(dirname $0)/../.. -tools/run_tests/run_interop_tests.py -l all -s all --cloud_to_prod --cloud_to_prod_auth --use_docker --http2_interop --http2_badserver_interop -t -j 12 $@ || true +tools/run_tests/run_interop_tests.py -l c++ --use_docker --http2_badserver_interop -t -j 12 $@ || true diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 55e729ace3c..a5164727597 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -964,7 +964,7 @@ try: manual_cmd_log=server_manual_cmd_log) if not args.manual_run: job = dockerjob.DockerJob(spec) - #job.wait_for_healthy(timeout_seconds=240) + job.wait_for_healthy(timeout_seconds=240) print(subprocess.check_output(['docker', 'version'])) server_jobs[lang] = job http2_badserver_ports = tuple([ From 246c0eee336fd657f82c9f4503295164efbdf9a6 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 16 Mar 2017 12:42:10 -0700 Subject: [PATCH 185/188] delay health check, increase timeout --- tools/run_tests/run_interop_tests.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index a5164727597..579114939f0 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -957,6 +957,7 @@ try: # don't run the server, set server port to a placeholder value server_addresses[lang] = ('localhost', '${SERVER_PORT}') + http2_server_job = None if args.http2_badserver_interop: # launch a HTTP2 server emulator that creates edge cases lang = str(http2InteropServer) @@ -964,8 +965,8 @@ try: manual_cmd_log=server_manual_cmd_log) if not args.manual_run: job = dockerjob.DockerJob(spec) - job.wait_for_healthy(timeout_seconds=240) - print(subprocess.check_output(['docker', 'version'])) + #job.wait_for_healthy(timeout_seconds=240) + http2_server_job = job server_jobs[lang] = job http2_badserver_ports = tuple([ job.mapped_port(_DEFAULT_SERVER_PORT + i) @@ -1055,6 +1056,8 @@ try: jobs.append(test_job) if args.http2_badserver_interop: + print(subprocess.check_output(['docker', 'ps'])) + http2_server_job.wait_for_healthy(timeout_seconds=600) for language in languages_http2_badserver_interop: for test_case in _HTTP2_BADSERVER_TEST_CASES: offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case) From 2f59cf2dc361e274bf97550447d5a72c8f239704 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 16 Mar 2017 13:44:09 -0700 Subject: [PATCH 186/188] move health check to just before running test cases --- tools/jenkins/run_interop.sh | 2 +- tools/run_tests/python_utils/dockerjob.py | 6 ++---- tools/run_tests/run_interop_tests.py | 23 ++++++++--------------- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/tools/jenkins/run_interop.sh b/tools/jenkins/run_interop.sh index 18e8b644381..2a9fc662a9e 100755 --- a/tools/jenkins/run_interop.sh +++ b/tools/jenkins/run_interop.sh @@ -36,4 +36,4 @@ export LANG=en_US.UTF-8 # Enter the gRPC repo root cd $(dirname $0)/../.. -tools/run_tests/run_interop_tests.py -l c++ --use_docker --http2_badserver_interop -t -j 12 $@ || true +tools/run_tests/run_interop_tests.py -l all -s all --cloud_to_prod --cloud_to_prod_auth --use_docker --http2_interop --http2_badserver_interop -t -j 12 $@ || true diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py index a29555428bc..709fc121a9b 100755 --- a/tools/run_tests/python_utils/dockerjob.py +++ b/tools/run_tests/python_utils/dockerjob.py @@ -76,15 +76,13 @@ def wait_for_healthy(cid, shortname, timeout_seconds): while time.time() - started < timeout_seconds: try: output = subprocess.check_output( - ['docker', 'inspect', '--format="{{.State.Health.Status}}"', cid]) - #stderr=_DEVNULL) - print(output) + ['docker', 'inspect', '--format="{{.State.Health.Status}}"', cid], + stderr=_DEVNULL) if output.strip('\n') == 'healthy': return except subprocess.CalledProcessError as e: pass time.sleep(1) - print(subprocess.check_output(['docker', 'ps'])) raise Exception('Timed out waiting for %s (%s) to pass health check' % (shortname, cid)) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 579114939f0..ce4dfb863ea 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -701,7 +701,7 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): % ('localhost', _DEFAULT_SERVER_PORT), '--health-interval=1s', '--health-retries=5', - '--health-timeout=1s', + '--health-timeout=10s', ] else: @@ -957,21 +957,15 @@ try: # don't run the server, set server port to a placeholder value server_addresses[lang] = ('localhost', '${SERVER_PORT}') - http2_server_job = None + http2_badserver_job = None if args.http2_badserver_interop: # launch a HTTP2 server emulator that creates edge cases lang = str(http2InteropServer) spec = server_jobspec(http2InteropServer, docker_images.get(lang), manual_cmd_log=server_manual_cmd_log) if not args.manual_run: - job = dockerjob.DockerJob(spec) - #job.wait_for_healthy(timeout_seconds=240) - http2_server_job = job - server_jobs[lang] = job - http2_badserver_ports = tuple([ - job.mapped_port(_DEFAULT_SERVER_PORT + i) - for i in range(len(_HTTP2_BADSERVER_TEST_CASES)) - ]) + http2_badserver_job = dockerjob.DockerJob(spec) + server_jobs[lang] = http2_badserver_job else: # don't run the server, set server port to a placeholder value server_addresses[lang] = ('localhost', '${SERVER_PORT}') @@ -1056,15 +1050,14 @@ try: jobs.append(test_job) if args.http2_badserver_interop: - print(subprocess.check_output(['docker', 'ps'])) - http2_server_job.wait_for_healthy(timeout_seconds=600) + if not args.manual_run: + http2_badserver_job.wait_for_healthy(timeout_seconds=600) for language in languages_http2_badserver_interop: for test_case in _HTTP2_BADSERVER_TEST_CASES: offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case) + server_port = _DEFAULT_SERVER_PORT+offset if not args.manual_run: - server_port = http2_badserver_ports[offset] - else: - server_port = _DEFAULT_SERVER_PORT+offset + server_port = http2_badserver_job.mapped_port(server_port) test_job = cloud_to_cloud_jobspec(language, test_case, str(http2InteropServer), From 01deb9d748db3fc7d04b7d511bec5aec594a8610 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 16 Mar 2017 15:53:55 -0700 Subject: [PATCH 187/188] add hyper to Dockerfile.template --- .../interoptest/grpc_interop_http2/Dockerfile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template index 6204c3e2cb2..12b99844253 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template @@ -33,7 +33,7 @@ <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> - RUN pip install twisted h2 + RUN pip install twisted h2 hyper # Define the default command. CMD ["bash"] From cb2f3f6ba6f8097b6a15d311d6eed9709c655963 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Fri, 17 Mar 2017 14:30:09 -0700 Subject: [PATCH 188/188] Downgrade error severity to INFO when failing to add a listener --- src/core/lib/iomgr/tcp_server_posix.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index b3644518f54..e242631fc0b 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -350,8 +350,20 @@ static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s, } } if (*out_port > 0) { - GRPC_LOG_IF_ERROR("Failed to add :: listener", v6_err); - GRPC_LOG_IF_ERROR("Failed to add 0.0.0.0 listener", v4_err); + if (v6_err != GRPC_ERROR_NONE) { + gpr_log(GPR_INFO, + "Failed to add :: listener, " + "the environment may not support IPv6: %s", + grpc_error_string(v6_err)); + GRPC_ERROR_UNREF(v6_err); + } + if (v4_err != GRPC_ERROR_NONE) { + gpr_log(GPR_INFO, + "Failed to add 0.0.0.0 listener, " + "the environment may not support IPv4: %s", + grpc_error_string(v4_err)); + GRPC_ERROR_UNREF(v4_err); + } return GRPC_ERROR_NONE; } else { grpc_error *root_err =