diff --git a/BUILD b/BUILD index b298615f786..264fe259b2c 100644 --- a/BUILD +++ b/BUILD @@ -204,6 +204,7 @@ cc_library( "src/core/lib/iomgr/pollset_windows.h", "src/core/lib/iomgr/port.h", "src/core/lib/iomgr/resolve_address.h", + "src/core/lib/iomgr/resource_quota.h", "src/core/lib/iomgr/sockaddr.h", "src/core/lib/iomgr/sockaddr_posix.h", "src/core/lib/iomgr/sockaddr_utils.h", @@ -212,6 +213,7 @@ cc_library( "src/core/lib/iomgr/socket_utils_posix.h", "src/core/lib/iomgr/socket_windows.h", "src/core/lib/iomgr/tcp_client.h", + "src/core/lib/iomgr/tcp_client_posix.h", "src/core/lib/iomgr/tcp_posix.h", "src/core/lib/iomgr/tcp_server.h", "src/core/lib/iomgr/tcp_uv.h", @@ -375,6 +377,7 @@ cc_library( "src/core/lib/iomgr/resolve_address_posix.c", "src/core/lib/iomgr/resolve_address_uv.c", "src/core/lib/iomgr/resolve_address_windows.c", + "src/core/lib/iomgr/resource_quota.c", "src/core/lib/iomgr/sockaddr_utils.c", "src/core/lib/iomgr/socket_utils_common_posix.c", "src/core/lib/iomgr/socket_utils_linux.c", @@ -625,6 +628,7 @@ cc_library( "src/core/lib/iomgr/pollset_windows.h", "src/core/lib/iomgr/port.h", "src/core/lib/iomgr/resolve_address.h", + "src/core/lib/iomgr/resource_quota.h", "src/core/lib/iomgr/sockaddr.h", "src/core/lib/iomgr/sockaddr_posix.h", "src/core/lib/iomgr/sockaddr_utils.h", @@ -633,6 +637,7 @@ cc_library( "src/core/lib/iomgr/socket_utils_posix.h", "src/core/lib/iomgr/socket_windows.h", "src/core/lib/iomgr/tcp_client.h", + "src/core/lib/iomgr/tcp_client_posix.h", "src/core/lib/iomgr/tcp_posix.h", "src/core/lib/iomgr/tcp_server.h", "src/core/lib/iomgr/tcp_uv.h", @@ -781,6 +786,7 @@ cc_library( "src/core/lib/iomgr/resolve_address_posix.c", "src/core/lib/iomgr/resolve_address_uv.c", "src/core/lib/iomgr/resolve_address_windows.c", + "src/core/lib/iomgr/resource_quota.c", "src/core/lib/iomgr/sockaddr_utils.c", "src/core/lib/iomgr/socket_utils_common_posix.c", "src/core/lib/iomgr/socket_utils_linux.c", @@ -1001,6 +1007,7 @@ cc_library( "src/core/lib/iomgr/pollset_windows.h", "src/core/lib/iomgr/port.h", "src/core/lib/iomgr/resolve_address.h", + "src/core/lib/iomgr/resource_quota.h", "src/core/lib/iomgr/sockaddr.h", "src/core/lib/iomgr/sockaddr_posix.h", "src/core/lib/iomgr/sockaddr_utils.h", @@ -1009,6 +1016,7 @@ cc_library( "src/core/lib/iomgr/socket_utils_posix.h", "src/core/lib/iomgr/socket_windows.h", "src/core/lib/iomgr/tcp_client.h", + "src/core/lib/iomgr/tcp_client_posix.h", "src/core/lib/iomgr/tcp_posix.h", "src/core/lib/iomgr/tcp_server.h", "src/core/lib/iomgr/tcp_uv.h", @@ -1149,6 +1157,7 @@ cc_library( "src/core/lib/iomgr/resolve_address_posix.c", "src/core/lib/iomgr/resolve_address_uv.c", "src/core/lib/iomgr/resolve_address_windows.c", + "src/core/lib/iomgr/resource_quota.c", "src/core/lib/iomgr/sockaddr_utils.c", "src/core/lib/iomgr/socket_utils_common_posix.c", "src/core/lib/iomgr/socket_utils_linux.c", @@ -1353,6 +1362,7 @@ cc_library( "src/cpp/common/channel_filter.cc", "src/cpp/common/completion_queue_cc.cc", "src/cpp/common/core_codegen.cc", + "src/cpp/common/resource_quota_cc.cc", "src/cpp/common/rpc_method.cc", "src/cpp/server/async_generic_service.cc", "src/cpp/server/create_default_thread_pool.cc", @@ -1398,6 +1408,7 @@ cc_library( "include/grpc++/impl/thd.h", "include/grpc++/impl/thd_cxx11.h", "include/grpc++/impl/thd_no_cxx11.h", + "include/grpc++/resource_quota.h", "include/grpc++/security/auth_context.h", "include/grpc++/security/auth_metadata_processor.h", "include/grpc++/security/credentials.h", @@ -1503,6 +1514,7 @@ cc_library( "src/cpp/common/channel_filter.cc", "src/cpp/common/completion_queue_cc.cc", "src/cpp/common/core_codegen.cc", + "src/cpp/common/resource_quota_cc.cc", "src/cpp/common/rpc_method.cc", "src/cpp/server/async_generic_service.cc", "src/cpp/server/create_default_thread_pool.cc", @@ -1548,6 +1560,7 @@ cc_library( "include/grpc++/impl/thd.h", "include/grpc++/impl/thd_cxx11.h", "include/grpc++/impl/thd_no_cxx11.h", + "include/grpc++/resource_quota.h", "include/grpc++/security/auth_context.h", "include/grpc++/security/auth_metadata_processor.h", "include/grpc++/security/credentials.h", @@ -1727,6 +1740,7 @@ cc_library( "src/cpp/common/channel_filter.cc", "src/cpp/common/completion_queue_cc.cc", "src/cpp/common/core_codegen.cc", + "src/cpp/common/resource_quota_cc.cc", "src/cpp/common/rpc_method.cc", "src/cpp/server/async_generic_service.cc", "src/cpp/server/create_default_thread_pool.cc", @@ -1772,6 +1786,7 @@ cc_library( "include/grpc++/impl/thd.h", "include/grpc++/impl/thd_cxx11.h", "include/grpc++/impl/thd_no_cxx11.h", + "include/grpc++/resource_quota.h", "include/grpc++/security/auth_context.h", "include/grpc++/security/auth_metadata_processor.h", "include/grpc++/security/credentials.h", @@ -2079,6 +2094,7 @@ objc_library( "src/core/lib/iomgr/resolve_address_posix.c", "src/core/lib/iomgr/resolve_address_uv.c", "src/core/lib/iomgr/resolve_address_windows.c", + "src/core/lib/iomgr/resource_quota.c", "src/core/lib/iomgr/sockaddr_utils.c", "src/core/lib/iomgr/socket_utils_common_posix.c", "src/core/lib/iomgr/socket_utils_linux.c", @@ -2308,6 +2324,7 @@ objc_library( "src/core/lib/iomgr/pollset_windows.h", "src/core/lib/iomgr/port.h", "src/core/lib/iomgr/resolve_address.h", + "src/core/lib/iomgr/resource_quota.h", "src/core/lib/iomgr/sockaddr.h", "src/core/lib/iomgr/sockaddr_posix.h", "src/core/lib/iomgr/sockaddr_utils.h", @@ -2316,6 +2333,7 @@ objc_library( "src/core/lib/iomgr/socket_utils_posix.h", "src/core/lib/iomgr/socket_windows.h", "src/core/lib/iomgr/tcp_client.h", + "src/core/lib/iomgr/tcp_client_posix.h", "src/core/lib/iomgr/tcp_posix.h", "src/core/lib/iomgr/tcp_server.h", "src/core/lib/iomgr/tcp_uv.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b9e6781567..e2e75ed60bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -334,6 +334,7 @@ add_library(grpc src/core/lib/iomgr/resolve_address_posix.c src/core/lib/iomgr/resolve_address_uv.c src/core/lib/iomgr/resolve_address_windows.c + src/core/lib/iomgr/resource_quota.c src/core/lib/iomgr/sockaddr_utils.c src/core/lib/iomgr/socket_utils_common_posix.c src/core/lib/iomgr/socket_utils_linux.c @@ -605,6 +606,7 @@ add_library(grpc_cronet src/core/lib/iomgr/resolve_address_posix.c src/core/lib/iomgr/resolve_address_uv.c src/core/lib/iomgr/resolve_address_windows.c + src/core/lib/iomgr/resource_quota.c src/core/lib/iomgr/sockaddr_utils.c src/core/lib/iomgr/socket_utils_common_posix.c src/core/lib/iomgr/socket_utils_linux.c @@ -848,6 +850,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/resolve_address_posix.c src/core/lib/iomgr/resolve_address_uv.c src/core/lib/iomgr/resolve_address_windows.c + src/core/lib/iomgr/resource_quota.c src/core/lib/iomgr/sockaddr_utils.c src/core/lib/iomgr/socket_utils_common_posix.c src/core/lib/iomgr/socket_utils_linux.c @@ -1061,6 +1064,7 @@ add_library(grpc++ src/cpp/common/channel_filter.cc src/cpp/common/completion_queue_cc.cc src/cpp/common/core_codegen.cc + src/cpp/common/resource_quota_cc.cc src/cpp/common/rpc_method.cc src/cpp/server/async_generic_service.cc src/cpp/server/create_default_thread_pool.cc @@ -1123,6 +1127,7 @@ foreach(_hdr include/grpc++/impl/thd.h include/grpc++/impl/thd_cxx11.h include/grpc++/impl/thd_no_cxx11.h + include/grpc++/resource_quota.h include/grpc++/security/auth_context.h include/grpc++/security/auth_metadata_processor.h include/grpc++/security/credentials.h @@ -1225,6 +1230,7 @@ add_library(grpc++_cronet src/cpp/common/channel_filter.cc src/cpp/common/completion_queue_cc.cc src/cpp/common/core_codegen.cc + src/cpp/common/resource_quota_cc.cc src/cpp/common/rpc_method.cc src/cpp/server/async_generic_service.cc src/cpp/server/create_default_thread_pool.cc @@ -1287,6 +1293,7 @@ foreach(_hdr include/grpc++/impl/thd.h include/grpc++/impl/thd_cxx11.h include/grpc++/impl/thd_no_cxx11.h + include/grpc++/resource_quota.h include/grpc++/security/auth_context.h include/grpc++/security/auth_metadata_processor.h include/grpc++/security/credentials.h @@ -1480,6 +1487,7 @@ add_library(grpc++_unsecure src/cpp/common/channel_filter.cc src/cpp/common/completion_queue_cc.cc src/cpp/common/core_codegen.cc + src/cpp/common/resource_quota_cc.cc src/cpp/common/rpc_method.cc src/cpp/server/async_generic_service.cc src/cpp/server/create_default_thread_pool.cc @@ -1542,6 +1550,7 @@ foreach(_hdr include/grpc++/impl/thd.h include/grpc++/impl/thd_cxx11.h include/grpc++/impl/thd_no_cxx11.h + include/grpc++/resource_quota.h include/grpc++/security/auth_context.h include/grpc++/security/auth_metadata_processor.h include/grpc++/security/credentials.h diff --git a/Makefile b/Makefile index d3fd1846364..b6e5e6a0c89 100644 --- a/Makefile +++ b/Makefile @@ -1008,6 +1008,7 @@ no_server_test: $(BINDIR)/$(CONFIG)/no_server_test percent_decode_fuzzer: $(BINDIR)/$(CONFIG)/percent_decode_fuzzer percent_encode_fuzzer: $(BINDIR)/$(CONFIG)/percent_encode_fuzzer resolve_address_test: $(BINDIR)/$(CONFIG)/resolve_address_test +resource_quota_test: $(BINDIR)/$(CONFIG)/resource_quota_test secure_channel_create_test: $(BINDIR)/$(CONFIG)/secure_channel_create_test secure_endpoint_test: $(BINDIR)/$(CONFIG)/secure_endpoint_test sequential_connectivity_test: $(BINDIR)/$(CONFIG)/sequential_connectivity_test @@ -1332,6 +1333,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/murmur_hash_test \ $(BINDIR)/$(CONFIG)/no_server_test \ $(BINDIR)/$(CONFIG)/resolve_address_test \ + $(BINDIR)/$(CONFIG)/resource_quota_test \ $(BINDIR)/$(CONFIG)/secure_channel_create_test \ $(BINDIR)/$(CONFIG)/secure_endpoint_test \ $(BINDIR)/$(CONFIG)/sequential_connectivity_test \ @@ -1720,6 +1722,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/no_server_test || ( echo test no_server_test failed ; exit 1 ) $(E) "[RUN] Testing resolve_address_test" $(Q) $(BINDIR)/$(CONFIG)/resolve_address_test || ( echo test resolve_address_test failed ; exit 1 ) + $(E) "[RUN] Testing resource_quota_test" + $(Q) $(BINDIR)/$(CONFIG)/resource_quota_test || ( echo test resource_quota_test failed ; exit 1 ) $(E) "[RUN] Testing secure_channel_create_test" $(Q) $(BINDIR)/$(CONFIG)/secure_channel_create_test || ( echo test secure_channel_create_test failed ; exit 1 ) $(E) "[RUN] Testing secure_endpoint_test" @@ -2624,6 +2628,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/resolve_address_posix.c \ src/core/lib/iomgr/resolve_address_uv.c \ src/core/lib/iomgr/resolve_address_windows.c \ + src/core/lib/iomgr/resource_quota.c \ src/core/lib/iomgr/sockaddr_utils.c \ src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ @@ -2913,6 +2918,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/resolve_address_posix.c \ src/core/lib/iomgr/resolve_address_uv.c \ src/core/lib/iomgr/resolve_address_windows.c \ + src/core/lib/iomgr/resource_quota.c \ src/core/lib/iomgr/sockaddr_utils.c \ src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ @@ -3193,6 +3199,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/resolve_address_posix.c \ src/core/lib/iomgr/resolve_address_uv.c \ src/core/lib/iomgr/resolve_address_windows.c \ + src/core/lib/iomgr/resource_quota.c \ src/core/lib/iomgr/sockaddr_utils.c \ src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ @@ -3402,6 +3409,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/resolve_address_posix.c \ src/core/lib/iomgr/resolve_address_uv.c \ src/core/lib/iomgr/resolve_address_windows.c \ + src/core/lib/iomgr/resource_quota.c \ src/core/lib/iomgr/sockaddr_utils.c \ src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ @@ -3698,6 +3706,7 @@ LIBGRPC++_SRC = \ src/cpp/common/channel_filter.cc \ src/cpp/common/completion_queue_cc.cc \ src/cpp/common/core_codegen.cc \ + src/cpp/common/resource_quota_cc.cc \ src/cpp/common/rpc_method.cc \ src/cpp/server/async_generic_service.cc \ src/cpp/server/create_default_thread_pool.cc \ @@ -3743,6 +3752,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/thd.h \ include/grpc++/impl/thd_cxx11.h \ include/grpc++/impl/thd_no_cxx11.h \ + include/grpc++/resource_quota.h \ include/grpc++/security/auth_context.h \ include/grpc++/security/auth_metadata_processor.h \ include/grpc++/security/credentials.h \ @@ -3891,6 +3901,7 @@ LIBGRPC++_CRONET_SRC = \ src/cpp/common/channel_filter.cc \ src/cpp/common/completion_queue_cc.cc \ src/cpp/common/core_codegen.cc \ + src/cpp/common/resource_quota_cc.cc \ src/cpp/common/rpc_method.cc \ src/cpp/server/async_generic_service.cc \ src/cpp/server/create_default_thread_pool.cc \ @@ -3936,6 +3947,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/thd.h \ include/grpc++/impl/thd_cxx11.h \ include/grpc++/impl/thd_no_cxx11.h \ + include/grpc++/resource_quota.h \ include/grpc++/security/auth_context.h \ include/grpc++/security/auth_metadata_processor.h \ include/grpc++/security/credentials.h \ @@ -4471,6 +4483,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/cpp/common/channel_filter.cc \ src/cpp/common/completion_queue_cc.cc \ src/cpp/common/core_codegen.cc \ + src/cpp/common/resource_quota_cc.cc \ src/cpp/common/rpc_method.cc \ src/cpp/server/async_generic_service.cc \ src/cpp/server/create_default_thread_pool.cc \ @@ -4516,6 +4529,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/thd.h \ include/grpc++/impl/thd_cxx11.h \ include/grpc++/impl/thd_no_cxx11.h \ + include/grpc++/resource_quota.h \ include/grpc++/security/auth_context.h \ include/grpc++/security/auth_metadata_processor.h \ include/grpc++/security/credentials.h \ @@ -6950,6 +6964,7 @@ LIBEND2END_TESTS_SRC = \ test/core/end2end/tests/registered_call.c \ test/core/end2end/tests/request_with_flags.c \ test/core/end2end/tests/request_with_payload.c \ + test/core/end2end/tests/resource_quota_server.c \ test/core/end2end/tests/server_finishes_request.c \ test/core/end2end/tests/shutdown_finishes_calls.c \ test/core/end2end/tests/shutdown_finishes_tags.c \ @@ -7032,6 +7047,7 @@ LIBEND2END_NOSEC_TESTS_SRC = \ test/core/end2end/tests/registered_call.c \ test/core/end2end/tests/request_with_flags.c \ test/core/end2end/tests/request_with_payload.c \ + test/core/end2end/tests/resource_quota_server.c \ test/core/end2end/tests/server_finishes_request.c \ test/core/end2end/tests/shutdown_finishes_calls.c \ test/core/end2end/tests/shutdown_finishes_tags.c \ @@ -10428,6 +10444,38 @@ endif endif +RESOURCE_QUOTA_TEST_SRC = \ + test/core/iomgr/resource_quota_test.c \ + +RESOURCE_QUOTA_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOURCE_QUOTA_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resource_quota_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/resource_quota_test: $(RESOURCE_QUOTA_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(RESOURCE_QUOTA_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/resource_quota_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/iomgr/resource_quota_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_resource_quota_test: $(RESOURCE_QUOTA_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOURCE_QUOTA_TEST_OBJS:.o=.dep) +endif +endif + + SECURE_CHANNEL_CREATE_TEST_SRC = \ test/core/surface/secure_channel_create_test.c \ diff --git a/binding.gyp b/binding.gyp index 23628f819b3..9f3b75aaf3d 100644 --- a/binding.gyp +++ b/binding.gyp @@ -612,6 +612,7 @@ 'src/core/lib/iomgr/resolve_address_posix.c', 'src/core/lib/iomgr/resolve_address_uv.c', 'src/core/lib/iomgr/resolve_address_windows.c', + 'src/core/lib/iomgr/resource_quota.c', 'src/core/lib/iomgr/sockaddr_utils.c', 'src/core/lib/iomgr/socket_utils_common_posix.c', 'src/core/lib/iomgr/socket_utils_linux.c', diff --git a/build.yaml b/build.yaml index 8bc8729278d..2824391ef20 100644 --- a/build.yaml +++ b/build.yaml @@ -208,6 +208,7 @@ filegroups: - src/core/lib/iomgr/pollset_windows.h - src/core/lib/iomgr/port.h - src/core/lib/iomgr/resolve_address.h + - src/core/lib/iomgr/resource_quota.h - src/core/lib/iomgr/sockaddr.h - src/core/lib/iomgr/sockaddr_posix.h - src/core/lib/iomgr/sockaddr_utils.h @@ -216,6 +217,7 @@ filegroups: - src/core/lib/iomgr/socket_utils_posix.h - src/core/lib/iomgr/socket_windows.h - src/core/lib/iomgr/tcp_client.h + - src/core/lib/iomgr/tcp_client_posix.h - src/core/lib/iomgr/tcp_posix.h - src/core/lib/iomgr/tcp_server.h - src/core/lib/iomgr/tcp_uv.h @@ -303,6 +305,7 @@ filegroups: - src/core/lib/iomgr/resolve_address_posix.c - src/core/lib/iomgr/resolve_address_uv.c - src/core/lib/iomgr/resolve_address_windows.c + - src/core/lib/iomgr/resource_quota.c - src/core/lib/iomgr/sockaddr_utils.c - src/core/lib/iomgr/socket_utils_common_posix.c - src/core/lib/iomgr/socket_utils_linux.c @@ -713,6 +716,7 @@ filegroups: - include/grpc++/impl/thd.h - include/grpc++/impl/thd_cxx11.h - include/grpc++/impl/thd_no_cxx11.h + - include/grpc++/resource_quota.h - include/grpc++/security/auth_context.h - include/grpc++/security/auth_metadata_processor.h - include/grpc++/security/credentials.h @@ -751,6 +755,7 @@ filegroups: - src/cpp/common/channel_filter.cc - src/cpp/common/completion_queue_cc.cc - src/cpp/common/core_codegen.cc + - src/cpp/common/resource_quota_cc.cc - src/cpp/common/rpc_method.cc - src/cpp/server/async_generic_service.cc - src/cpp/server/create_default_thread_pool.cc @@ -2429,6 +2434,17 @@ targets: - gpr exclude_iomgrs: - uv +- name: resource_quota_test + cpu_cost: 30 + build: test + language: c + src: + - test/core/iomgr/resource_quota_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: secure_channel_create_test build: test language: c diff --git a/config.m4 b/config.m4 index 84af6bd3d0d..8f26e426789 100644 --- a/config.m4 +++ b/config.m4 @@ -128,6 +128,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/resolve_address_posix.c \ src/core/lib/iomgr/resolve_address_uv.c \ src/core/lib/iomgr/resolve_address_windows.c \ + src/core/lib/iomgr/resource_quota.c \ src/core/lib/iomgr/sockaddr_utils.c \ src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 0e4af8ae6c3..43aedecd06f 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -35,7 +35,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.0.0' + version = '1.0.1' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'http://www.grpc.io' @@ -186,7 +186,7 @@ Pod::Spec.new do |s| ss.header_mappings_dir = '.' ss.libraries = 'z' ss.dependency "#{s.name}/Interface", version - ss.dependency 'BoringSSL', '~> 6.0' + ss.dependency 'BoringSSL', '~> 7.0' # To save you from scrolling, this is the last part of the podspec. ss.source_files = 'src/core/lib/profiling/timers.h', @@ -291,6 +291,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/pollset_windows.h', 'src/core/lib/iomgr/port.h', 'src/core/lib/iomgr/resolve_address.h', + 'src/core/lib/iomgr/resource_quota.h', 'src/core/lib/iomgr/sockaddr.h', 'src/core/lib/iomgr/sockaddr_posix.h', 'src/core/lib/iomgr/sockaddr_utils.h', @@ -299,6 +300,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/socket_utils_posix.h', 'src/core/lib/iomgr/socket_windows.h', 'src/core/lib/iomgr/tcp_client.h', + 'src/core/lib/iomgr/tcp_client_posix.h', 'src/core/lib/iomgr/tcp_posix.h', 'src/core/lib/iomgr/tcp_server.h', 'src/core/lib/iomgr/tcp_uv.h', @@ -466,6 +468,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/resolve_address_posix.c', 'src/core/lib/iomgr/resolve_address_uv.c', 'src/core/lib/iomgr/resolve_address_windows.c', + 'src/core/lib/iomgr/resource_quota.c', 'src/core/lib/iomgr/sockaddr_utils.c', 'src/core/lib/iomgr/socket_utils_common_posix.c', 'src/core/lib/iomgr/socket_utils_linux.c', @@ -684,6 +687,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/pollset_windows.h', 'src/core/lib/iomgr/port.h', 'src/core/lib/iomgr/resolve_address.h', + 'src/core/lib/iomgr/resource_quota.h', 'src/core/lib/iomgr/sockaddr.h', 'src/core/lib/iomgr/sockaddr_posix.h', 'src/core/lib/iomgr/sockaddr_utils.h', @@ -692,6 +696,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/socket_utils_posix.h', 'src/core/lib/iomgr/socket_windows.h', 'src/core/lib/iomgr/tcp_client.h', + 'src/core/lib/iomgr/tcp_client_posix.h', 'src/core/lib/iomgr/tcp_posix.h', 'src/core/lib/iomgr/tcp_server.h', 'src/core/lib/iomgr/tcp_uv.h', diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index f6426fb4deb..61d4b62d391 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -30,7 +30,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.0.0' + version = '1.0.1' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'http://www.grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 316843436ba..d59385c039a 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -30,7 +30,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.0.0' + version = '1.0.1' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'http://www.grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 9a479349531..76410b17d28 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -30,7 +30,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.0.0' + version = '1.0.1' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'http://www.grpc.io' diff --git a/grpc.def b/grpc.def index 6e905a297ed..0b6db801d8d 100644 --- a/grpc.def +++ b/grpc.def @@ -94,6 +94,11 @@ EXPORTS grpc_header_nonbin_value_is_legal grpc_is_binary_header grpc_call_error_to_string + grpc_resource_quota_create + grpc_resource_quota_ref + grpc_resource_quota_unref + grpc_resource_quota_resize + grpc_resource_quota_arg_vtable grpc_insecure_channel_create_from_fd grpc_server_add_insecure_channel_from_fd grpc_use_signal diff --git a/grpc.gemspec b/grpc.gemspec index c5bb926cb50..615a962cdb6 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -211,6 +211,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/pollset_windows.h ) s.files += %w( src/core/lib/iomgr/port.h ) s.files += %w( src/core/lib/iomgr/resolve_address.h ) + s.files += %w( src/core/lib/iomgr/resource_quota.h ) s.files += %w( src/core/lib/iomgr/sockaddr.h ) s.files += %w( src/core/lib/iomgr/sockaddr_posix.h ) s.files += %w( src/core/lib/iomgr/sockaddr_utils.h ) @@ -219,6 +220,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/socket_utils_posix.h ) s.files += %w( src/core/lib/iomgr/socket_windows.h ) s.files += %w( src/core/lib/iomgr/tcp_client.h ) + s.files += %w( src/core/lib/iomgr/tcp_client_posix.h ) s.files += %w( src/core/lib/iomgr/tcp_posix.h ) s.files += %w( src/core/lib/iomgr/tcp_server.h ) s.files += %w( src/core/lib/iomgr/tcp_uv.h ) @@ -386,6 +388,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/resolve_address_posix.c ) s.files += %w( src/core/lib/iomgr/resolve_address_uv.c ) s.files += %w( src/core/lib/iomgr/resolve_address_windows.c ) + s.files += %w( src/core/lib/iomgr/resource_quota.c ) s.files += %w( src/core/lib/iomgr/sockaddr_utils.c ) s.files += %w( src/core/lib/iomgr/socket_utils_common_posix.c ) s.files += %w( src/core/lib/iomgr/socket_utils_linux.c ) diff --git a/include/grpc++/resource_quota.h b/include/grpc++/resource_quota.h new file mode 100644 index 00000000000..db5bc8e7be1 --- /dev/null +++ b/include/grpc++/resource_quota.h @@ -0,0 +1,70 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPCXX_RESOURCE_QUOTA_H +#define GRPCXX_RESOURCE_QUOTA_H + +struct grpc_resource_quota; + +#include + +namespace grpc { + +/// ResourceQuota represents a bound on memory usage by the gRPC library. +/// A ResourceQuota can be attached to a server (via ServerBuilder), or a client +/// channel (via ChannelArguments). gRPC will attempt to keep memory used by +/// all attached entities below the ResourceQuota bound. +class ResourceQuota GRPC_FINAL { + public: + explicit ResourceQuota(const grpc::string& name); + ResourceQuota(); + ~ResourceQuota(); + + /// Resize this ResourceQuota to a new size. If new_size is smaller than the + /// current size of the pool, memory usage will be monotonically decreased + /// until it falls under new_size. No time bound is given for this to occur + /// however. + ResourceQuota& Resize(size_t new_size); + + grpc_resource_quota* c_resource_quota() const { return impl_; } + + private: + ResourceQuota(const ResourceQuota& rhs); + ResourceQuota& operator=(const ResourceQuota& rhs); + + grpc_resource_quota* const impl_; +}; + +} // namespace grpc + +#endif // GRPCXX_RESOURCE_QUOTA_H diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index c6bcf8b90a1..9252c6a63a8 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -46,9 +46,12 @@ #include #include +struct grpc_resource_quota; + namespace grpc { class AsyncGenericService; +class ResourceQuota; class CompletionQueue; class RpcService; class Server; @@ -64,6 +67,7 @@ class ServerBuilderPluginTest; class ServerBuilder { public: ServerBuilder(); + ~ServerBuilder(); enum SyncServerOption { NUM_CQS, MIN_POLLERS, MAX_POLLERS, CQ_TIMEOUT_MSEC }; @@ -118,6 +122,9 @@ class ServerBuilder { ServerBuilder& SetDefaultCompressionAlgorithm( grpc_compression_algorithm algorithm); + /// Set the attached buffer pool for this server + ServerBuilder& SetResourceQuota(const ResourceQuota& resource_quota); + ServerBuilder& SetOption(std::unique_ptr option); /// Only useful if this is a Synchronous server. @@ -222,6 +229,7 @@ class ServerBuilder { std::shared_ptr creds_; std::vector> plugins_; + grpc_resource_quota* resource_quota_; AsyncGenericService* generic_service_; struct { bool is_set; diff --git a/include/grpc++/support/channel_arguments.h b/include/grpc++/support/channel_arguments.h index ae243939e9a..ba203f85bda 100644 --- a/include/grpc++/support/channel_arguments.h +++ b/include/grpc++/support/channel_arguments.h @@ -46,6 +46,8 @@ namespace testing { class ChannelArgumentsTest; } // namespace testing +class ResourceQuota; + /// Options for channel creation. The user can use generic setters to pass /// key value pairs down to c channel creation code. For grpc related options, /// concrete setters are provided. @@ -80,6 +82,9 @@ class ChannelArguments { /// The given string will be sent at the front of the user agent string. void SetUserAgentPrefix(const grpc::string& user_agent_prefix); + /// The given buffer pool will be attached to the constructed channel + void SetResourceQuota(const ResourceQuota& resource_quota); + // Generic channel argument setters. Only for advanced use cases. /// Set an integer argument \a value under \a key. void SetInt(const grpc::string& key, int value); @@ -88,6 +93,9 @@ class ChannelArguments { /// Set a pointer argument \a value under \a key. Owership is not transferred. void SetPointer(const grpc::string& key, void* value); + void SetPointerWithVtable(const grpc::string& key, void* value, + const grpc_arg_pointer_vtable* vtable); + /// Set a textual argument \a value under \a key. void SetString(const grpc::string& key, const grpc::string& value); diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index 587d86c98fd..f8e442274e6 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -401,6 +401,23 @@ GRPCAPI int grpc_is_binary_header(const char *key, size_t length); /** Convert grpc_call_error values to a string */ GRPCAPI const char *grpc_call_error_to_string(grpc_call_error error); +/** Create a buffer pool */ +GRPCAPI grpc_resource_quota *grpc_resource_quota_create(const char *trace_name); + +/** Add a reference to a buffer pool */ +GRPCAPI void grpc_resource_quota_ref(grpc_resource_quota *resource_quota); + +/** Drop a reference to a buffer pool */ +GRPCAPI void grpc_resource_quota_unref(grpc_resource_quota *resource_quota); + +/** Update the size of a buffer pool */ +GRPCAPI void grpc_resource_quota_resize(grpc_resource_quota *resource_quota, + size_t new_size); + +/** Fetch a vtable for a grpc_channel_arg that points to a grpc_resource_quota + */ +GRPCAPI const grpc_arg_pointer_vtable *grpc_resource_quota_arg_vtable(void); + #ifdef __cplusplus } #endif diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 0be7ab2ad23..66ece3c8a0f 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -201,6 +201,9 @@ typedef struct { #define GRPC_ARG_MAX_METADATA_SIZE "grpc.max_metadata_size" /** If non-zero, allow the use of SO_REUSEPORT if it's available (default 1) */ #define GRPC_ARG_ALLOW_REUSEPORT "grpc.so_reuseport" +/** If non-zero, a pointer to a buffer pool (use grpc_resource_quota_arg_vtable + to fetch an appropriate pointer arg vtable */ +#define GRPC_ARG_RESOURCE_QUOTA "grpc.resource_quota" /** Service config data, to be passed to subchannels. Not intended for external use. */ #define GRPC_ARG_SERVICE_CONFIG "grpc.service_config" @@ -467,6 +470,8 @@ typedef struct grpc_op { } data; } grpc_op; +typedef struct grpc_resource_quota grpc_resource_quota; + #ifdef __cplusplus } #endif diff --git a/package.json b/package.json index c6b2600209b..5506e9bbdf8 100644 --- a/package.json +++ b/package.json @@ -25,26 +25,28 @@ "coverage": "./node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha src/node/test", "install": "./node_modules/.bin/node-pre-gyp install --fallback-to-build" }, - "bundledDependencies": ["node-pre-gyp"], + "bundledDependencies": [ + "node-pre-gyp" + ], "dependencies": { "arguejs": "^0.2.3", - "lodash": "^3.9.3", + "lodash": "^4.15.0", "nan": "^2.0.0", - "protobufjs": "^4.0.0" + "node-pre-gyp": "^0.6.0", + "protobufjs": "^5.0.0" }, "devDependencies": { - "async": "^1.5.0", + "async": "^2.0.1", "body-parser": "^1.15.2", "express": "^4.14.0", "google-auth-library": "^0.9.2", "google-protobuf": "^3.0.0", - "istanbul": "^0.3.21", + "istanbul": "^0.4.4", "jsdoc": "^3.3.2", "jshint": "^2.5.0", "minimist": "^1.1.0", - "mocha": "^2.3.4", - "mocha-jenkins-reporter": "^0.1.9", - "mustache": "^2.0.0", + "mocha": "^3.0.2", + "mocha-jenkins-reporter": "^0.2.3", "poisson-process": "^0.2.1" }, "engines": { @@ -52,11 +54,10 @@ }, "binary": { "module_name": "grpc_node", - "module_path": "./build/Release/", + "module_path": "src/node/extension_binary", "host": "https://storage.googleapis.com/", "remote_path": "grpc-precompiled-binaries/node/{name}/v{version}", - "package_name": "{node_abi}-{platform}-{arch}.tar.gz", - "module_path": "src/node/extension_binary" + "package_name": "{node_abi}-{platform}-{arch}.tar.gz" }, "files": [ "LICENSE", @@ -77,7 +78,7 @@ ], "main": "src/node/index.js", "license": "BSD-3-Clause", - "jshintConfig" : { + "jshintConfig": { "bitwise": true, "curly": true, "eqeqeq": true, diff --git a/package.xml b/package.xml index d4a4e3c7b97..a60eac68205 100644 --- a/package.xml +++ b/package.xml @@ -218,6 +218,7 @@ + @@ -226,6 +227,7 @@ + @@ -393,6 +395,7 @@ + diff --git a/src/compiler/ruby_generator.cc b/src/compiler/ruby_generator.cc index 02202568cb4..c85babf1b86 100644 --- a/src/compiler/ruby_generator.cc +++ b/src/compiler/ruby_generator.cc @@ -119,6 +119,43 @@ void PrintService(const ServiceDescriptor *service, const grpc::string &package, } // namespace +// The following functions are copied directly from the source for the protoc +// ruby generator +// to ensure compatibility (with the exception of int and string type changes). +// See +// https://github.com/google/protobuf/blob/master/src/google/protobuf/compiler/ruby/ruby_generator.cc#L250 +// TODO: keep up to date with protoc code generation, though this behavior isn't +// expected to change +bool IsLower(char ch) { return ch >= 'a' && ch <= 'z'; } + +char ToUpper(char ch) { return IsLower(ch) ? (ch - 'a' + 'A') : ch; } + +// Package names in protobuf are snake_case by convention, but Ruby module +// names must be PascalCased. +// +// foo_bar_baz -> FooBarBaz +grpc::string PackageToModule(const grpc::string &name) { + bool next_upper = true; + grpc::string result; + result.reserve(name.size()); + + for (grpc::string::size_type i = 0; i < name.size(); i++) { + if (name[i] == '_') { + next_upper = true; + } else { + if (next_upper) { + result.push_back(ToUpper(name[i])); + } else { + result.push_back(name[i]); + } + next_upper = false; + } + } + + return result; +} +// end copying of protoc generator for ruby code + grpc::string GetServices(const FileDescriptor *file) { grpc::string output; { @@ -162,7 +199,7 @@ grpc::string GetServices(const FileDescriptor *file) { std::vector modules = Split(file->package(), '.'); for (size_t i = 0; i < modules.size(); ++i) { std::map module_vars = ListToDict({ - "module.name", CapitalizeFirst(modules[i]), + "module.name", PackageToModule(modules[i]), }); out.Print(module_vars, "module $module.name$\n"); out.Indent(); diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 80b4f048c2f..ff773ac3348 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -1022,6 +1022,10 @@ static void cc_destroy_call_elem(grpc_exec_ctx *exec_ctx, GPR_ASSERT(calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING); gpr_mu_destroy(&calld->mu); GPR_ASSERT(calld->waiting_ops_count == 0); + if (calld->connected_subchannel != NULL) { + GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, calld->connected_subchannel, + "picked"); + } gpr_free(calld->waiting_ops); gpr_free(and_free_memory); } diff --git a/src/core/ext/client_channel/subchannel.c b/src/core/ext/client_channel/subchannel.c index ab6aede23ac..789966cb693 100644 --- a/src/core/ext/client_channel/subchannel.c +++ b/src/core/ext/client_channel/subchannel.c @@ -183,9 +183,10 @@ static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_free(c); } -void grpc_connected_subchannel_ref( +grpc_connected_subchannel *grpc_connected_subchannel_ref( grpc_connected_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON); + return c; } void grpc_connected_subchannel_unref(grpc_exec_ctx *exec_ctx, diff --git a/src/core/ext/client_channel/subchannel.h b/src/core/ext/client_channel/subchannel.h index 56520740741..93bd72d20d5 100644 --- a/src/core/ext/client_channel/subchannel.h +++ b/src/core/ext/client_channel/subchannel.h @@ -97,7 +97,7 @@ grpc_subchannel *grpc_subchannel_weak_ref( void grpc_subchannel_weak_unref(grpc_exec_ctx *exec_ctx, grpc_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS); -void grpc_connected_subchannel_ref( +grpc_connected_subchannel *grpc_connected_subchannel_ref( grpc_connected_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS); void grpc_connected_subchannel_unref(grpc_exec_ctx *exec_ctx, grpc_connected_subchannel *channel diff --git a/src/core/ext/lb_policy/pick_first/pick_first.c b/src/core/ext/lb_policy/pick_first/pick_first.c index 5d3433df74a..ac3c6a305ac 100644 --- a/src/core/ext/lb_policy/pick_first/pick_first.c +++ b/src/core/ext/lb_policy/pick_first/pick_first.c @@ -209,7 +209,7 @@ static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, /* Check atomically for a selected channel */ grpc_connected_subchannel *selected = GET_SELECTED(p); if (selected != NULL) { - *target = selected; + *target = GRPC_CONNECTED_SUBCHANNEL_REF(selected, "picked"); return 1; } @@ -218,7 +218,7 @@ static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, selected = GET_SELECTED(p); if (selected) { gpr_mu_unlock(&p->mu); - *target = selected; + *target = GRPC_CONNECTED_SUBCHANNEL_REF(selected, "picked"); return 1; } else { if (!p->started_picking) { @@ -310,7 +310,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, /* update any calls that were waiting for a pick */ while ((pp = p->pending_picks)) { p->pending_picks = pp->next; - *pp->target = selected; + *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF(selected, "picked"); grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL); gpr_free(pp); } diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index c0743b00e80..37a9b18b970 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -397,7 +397,9 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, gpr_mu_lock(&p->mu); if ((selected = peek_next_connected_locked(p))) { /* readily available, report right away */ - *target = grpc_subchannel_get_connected_subchannel(selected->subchannel); + *target = GRPC_CONNECTED_SUBCHANNEL_REF( + grpc_subchannel_get_connected_subchannel(selected->subchannel), + "picked"); if (user_data != NULL) { *user_data = selected->user_data; @@ -463,8 +465,9 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, while ((pp = p->pending_picks)) { p->pending_picks = pp->next; - *pp->target = - grpc_subchannel_get_connected_subchannel(selected->subchannel); + *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF( + grpc_subchannel_get_connected_subchannel(selected->subchannel), + "picked"); if (pp->user_data != NULL) { *pp->user_data = selected->user_data; } @@ -578,7 +581,9 @@ static void rr_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, gpr_mu_lock(&p->mu); if ((selected = peek_next_connected_locked(p))) { gpr_mu_unlock(&p->mu); - target = grpc_subchannel_get_connected_subchannel(selected->subchannel); + target = GRPC_CONNECTED_SUBCHANNEL_REF( + grpc_subchannel_get_connected_subchannel(selected->subchannel), + "picked"); grpc_connected_subchannel_ping(exec_ctx, target, closure); } else { gpr_mu_unlock(&p->mu); diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c index 372fb7bf4ca..71a06e118ba 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c @@ -154,7 +154,8 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con, c->tcp = NULL; grpc_closure_init(&c->connected, connected, c); grpc_tcp_client_connect(exec_ctx, &c->connected, &c->tcp, - args->interested_parties, args->addr, args->deadline); + args->interested_parties, args->channel_args, + args->addr, args->deadline); } static const grpc_connector_vtable connector_vtable = { diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c index b2c5e5b088c..1e5b1c22e39 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c @@ -44,6 +44,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/iomgr/tcp_client_posix.h" #include "src/core/lib/iomgr/tcp_posix.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" @@ -65,9 +66,8 @@ grpc_channel *grpc_insecure_channel_create_from_fd( int flags = fcntl(fd, F_GETFL, 0); GPR_ASSERT(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == 0); - grpc_endpoint *client = - grpc_tcp_create(grpc_fd_create(fd, "client"), - GRPC_TCP_DEFAULT_READ_SLICE_SIZE, "fd-client"); + grpc_endpoint *client = grpc_tcp_client_create_from_fd( + &exec_ctx, grpc_fd_create(fd, "client"), args, "fd-client"); grpc_transport *transport = grpc_create_chttp2_transport(&exec_ctx, final_args, client, 1); diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c index 29095af2076..57e1a8ec01f 100644 --- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c +++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c @@ -212,9 +212,9 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con, GPR_ASSERT(c->connecting_endpoint == NULL); gpr_mu_unlock(&c->mu); grpc_closure_init(&c->connected_closure, connected, c); - grpc_tcp_client_connect(exec_ctx, &c->connected_closure, - &c->newly_connecting_endpoint, - args->interested_parties, args->addr, args->deadline); + grpc_tcp_client_connect( + exec_ctx, &c->connected_closure, &c->newly_connecting_endpoint, + args->interested_parties, args->channel_args, args->addr, args->deadline); } static const grpc_connector_vtable connector_vtable = { diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c index e4967c2f08b..d42611b8637 100644 --- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c +++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c @@ -139,8 +139,8 @@ int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) { goto error; } - err = - grpc_tcp_server_create(NULL, grpc_server_get_channel_args(server), &tcp); + err = grpc_tcp_server_create(&exec_ctx, NULL, + grpc_server_get_channel_args(server), &tcp); if (err != GRPC_ERROR_NONE) { goto error; } diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c index 9af17fb5ae7..aa2ecf5743d 100644 --- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c +++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c @@ -57,8 +57,12 @@ void grpc_server_add_insecure_channel_from_fd(grpc_server *server, char *name; gpr_asprintf(&name, "fd:%d", fd); - grpc_endpoint *server_endpoint = grpc_tcp_create( - grpc_fd_create(fd, name), GRPC_TCP_DEFAULT_READ_SLICE_SIZE, name); + grpc_resource_quota *resource_quota = grpc_resource_quota_from_channel_args( + grpc_server_get_channel_args(server)); + grpc_endpoint *server_endpoint = + grpc_tcp_create(grpc_fd_create(fd, name), resource_quota, + GRPC_TCP_DEFAULT_READ_SLICE_SIZE, name); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); gpr_free(name); diff --git a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c index 0fab8885418..7ad687042d3 100644 --- a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c +++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c @@ -271,7 +271,8 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr, memset(server_state, 0, sizeof(*server_state)); grpc_closure_init(&server_state->tcp_server_shutdown_complete, tcp_server_shutdown_complete, server_state); - err = grpc_tcp_server_create(&server_state->tcp_server_shutdown_complete, + err = grpc_tcp_server_create(&exec_ctx, + &server_state->tcp_server_shutdown_complete, grpc_server_get_channel_args(server), &tcp); if (err != GRPC_ERROR_NONE) { goto error; diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index ecf3aea8701..4a9f806354c 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -114,6 +114,20 @@ static void fail_pending_writes(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, grpc_error *error); +static void benign_reclaimer(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); +static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); +static void destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); +static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); + +static void post_benign_reclaimer(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); +static void post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); + static void close_transport_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_error *error); static void end_all_the_calls(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, @@ -241,6 +255,11 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_closure_init(&t->write_action_end_locked, write_action_end_locked, t); grpc_closure_init(&t->read_action_begin, read_action_begin, t); grpc_closure_init(&t->read_action_locked, read_action_locked, t); + grpc_closure_init(&t->benign_reclaimer, benign_reclaimer, t); + grpc_closure_init(&t->destructive_reclaimer, destructive_reclaimer, t); + grpc_closure_init(&t->benign_reclaimer_locked, benign_reclaimer_locked, t); + grpc_closure_init(&t->destructive_reclaimer_locked, + destructive_reclaimer_locked, t); grpc_chttp2_goaway_parser_init(&t->goaway_parser); grpc_chttp2_hpack_parser_init(&t->hpack_parser); @@ -362,6 +381,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } grpc_chttp2_initiate_write(exec_ctx, t, false, "init"); + post_benign_reclaimer(exec_ctx, t); } static void destroy_transport_locked(grpc_exec_ctx *exec_ctx, void *tp, @@ -467,6 +487,7 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; *t->accepting_stream = s; grpc_chttp2_stream_map_add(&t->stream_map, s->id, s); + post_destructive_reclaimer(exec_ctx, t); } GPR_TIMER_END("init_stream", 0); @@ -675,6 +696,13 @@ static void write_action_end_locked(grpc_exec_ctx *exec_ctx, void *tp, close_transport_locked(exec_ctx, t, GRPC_ERROR_REF(error)); } + if (t->sent_goaway_state == GRPC_CHTTP2_GOAWAY_SEND_SCHEDULED) { + t->sent_goaway_state = GRPC_CHTTP2_GOAWAY_SENT; + if (grpc_chttp2_stream_map_size(&t->stream_map) == 0) { + close_transport_locked(exec_ctx, t, GRPC_ERROR_CREATE("goaway sent")); + } + } + grpc_chttp2_end_write(exec_ctx, t, GRPC_ERROR_REF(error)); switch (t->write_state) { @@ -780,6 +808,7 @@ static void maybe_start_some_streams(grpc_exec_ctx *exec_ctx, [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; s->max_recv_bytes = GPR_MAX(stream_incoming_window, s->max_recv_bytes); grpc_chttp2_stream_map_add(&t->stream_map, s->id, s); + post_destructive_reclaimer(exec_ctx, t); grpc_chttp2_become_writable(exec_ctx, t, s, true, "new_stream"); } /* cancel out streams that will never be started */ @@ -993,9 +1022,14 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, } if (!s->write_closed) { if (t->is_client) { - GPR_ASSERT(s->id == 0); - grpc_chttp2_list_add_waiting_for_concurrency(t, s); - maybe_start_some_streams(exec_ctx, t); + if (!t->closed) { + GPR_ASSERT(s->id == 0); + grpc_chttp2_list_add_waiting_for_concurrency(t, s); + maybe_start_some_streams(exec_ctx, t); + } else { + grpc_chttp2_cancel_stream(exec_ctx, t, s, + GRPC_ERROR_CREATE("Transport closed")); + } } else { GPR_ASSERT(s->id != 0); grpc_chttp2_become_writable(exec_ctx, t, s, true, @@ -1185,6 +1219,14 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, gpr_free(msg); } +static void send_goaway(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_error_code error, gpr_slice data) { + t->sent_goaway_state = GRPC_CHTTP2_GOAWAY_SEND_SCHEDULED; + grpc_chttp2_goaway_append(t->last_new_stream_id, (uint32_t)error, data, + &t->qbuf); + grpc_chttp2_initiate_write(exec_ctx, t, false, "goaway_sent"); +} + static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, grpc_error *error_ignored) { @@ -1199,15 +1241,9 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, } if (op->send_goaway) { - t->sent_goaway = 1; - grpc_chttp2_goaway_append( - t->last_new_stream_id, - (uint32_t)grpc_chttp2_grpc_status_to_http2_error(op->goaway_status), - gpr_slice_ref(*op->goaway_message), &t->qbuf); - close_transport = grpc_chttp2_stream_map_size(&t->stream_map) == 0 - ? GRPC_ERROR_CREATE("GOAWAY sent") - : GRPC_ERROR_NONE; - grpc_chttp2_initiate_write(exec_ctx, t, false, "goaway_sent"); + send_goaway(exec_ctx, t, + grpc_chttp2_grpc_status_to_http2_error(op->goaway_status), + gpr_slice_ref(*op->goaway_message)); } if (op->set_accept_stream) { @@ -1341,10 +1377,14 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, s->data_parser.parsing_frame = NULL; } - if (grpc_chttp2_stream_map_size(&t->stream_map) == 0 && t->sent_goaway) { - close_transport_locked( - exec_ctx, t, GRPC_ERROR_CREATE_REFERENCING( - "Last stream closed after sending GOAWAY", &error, 1)); + if (grpc_chttp2_stream_map_size(&t->stream_map) == 0) { + post_benign_reclaimer(exec_ctx, t); + if (t->sent_goaway_state == GRPC_CHTTP2_GOAWAY_SENT) { + close_transport_locked( + exec_ctx, t, + GRPC_ERROR_CREATE_REFERENCING( + "Last stream closed after sending GOAWAY", &error, 1)); + } } if (grpc_chttp2_list_remove_writable_stream(t, s)) { GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:remove_stream"); @@ -2071,6 +2111,103 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( return incoming_byte_stream; } +/******************************************************************************* + * RESOURCE QUOTAS + */ + +static void post_benign_reclaimer(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + if (!t->benign_reclaimer_registered) { + t->benign_reclaimer_registered = true; + GRPC_CHTTP2_REF_TRANSPORT(t, "benign_reclaimer"); + grpc_resource_user_post_reclaimer(exec_ctx, + grpc_endpoint_get_resource_user(t->ep), + false, &t->benign_reclaimer); + } +} + +static void post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + if (!t->destructive_reclaimer_registered) { + t->destructive_reclaimer_registered = true; + GRPC_CHTTP2_REF_TRANSPORT(t, "destructive_reclaimer"); + grpc_resource_user_post_reclaimer(exec_ctx, + grpc_endpoint_get_resource_user(t->ep), + true, &t->destructive_reclaimer); + } +} + +static void benign_reclaimer(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_chttp2_transport *t = arg; + grpc_combiner_execute(exec_ctx, t->combiner, &t->benign_reclaimer_locked, + GRPC_ERROR_REF(error), false); +} + +static void destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_chttp2_transport *t = arg; + grpc_combiner_execute(exec_ctx, t->combiner, &t->destructive_reclaimer_locked, + GRPC_ERROR_REF(error), false); +} + +static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_chttp2_transport *t = arg; + if (error == GRPC_ERROR_NONE && + grpc_chttp2_stream_map_size(&t->stream_map) == 0) { + /* Channel with no active streams: send a goaway to try and make it + * disconnect cleanly */ + if (grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, "HTTP2: %s - send goaway to free memory", + t->peer_string); + } + send_goaway(exec_ctx, t, GRPC_CHTTP2_ENHANCE_YOUR_CALM, + gpr_slice_from_static_string("Buffers full")); + } else if (error == GRPC_ERROR_NONE && grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, + "HTTP2: %s - skip benign reclamation, there are still %" PRIdPTR + " streams", + t->peer_string, grpc_chttp2_stream_map_size(&t->stream_map)); + } + t->benign_reclaimer_registered = false; + if (error != GRPC_ERROR_CANCELLED) { + grpc_resource_user_finish_reclamation( + exec_ctx, grpc_endpoint_get_resource_user(t->ep)); + } + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "benign_reclaimer"); +} + +static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_chttp2_transport *t = arg; + size_t n = grpc_chttp2_stream_map_size(&t->stream_map); + t->destructive_reclaimer_registered = false; + if (error == GRPC_ERROR_NONE && n > 0) { + grpc_chttp2_stream *s = grpc_chttp2_stream_map_rand(&t->stream_map); + if (grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, "HTTP2: %s - abandon stream id %d", t->peer_string, + s->id); + } + grpc_chttp2_cancel_stream( + exec_ctx, t, s, grpc_error_set_int(GRPC_ERROR_CREATE("Buffers full"), + GRPC_ERROR_INT_HTTP2_ERROR, + GRPC_CHTTP2_ENHANCE_YOUR_CALM)); + if (n > 1) { + /* Since we cancel one stream per destructive reclamation, if + there are more streams left, we can immediately post a new + reclaimer in case the resource quota needs to free more + memory */ + post_destructive_reclaimer(exec_ctx, t); + } + } + if (error != GRPC_ERROR_CANCELLED) { + grpc_resource_user_finish_reclamation( + exec_ctx, grpc_endpoint_get_resource_user(t->ep)); + } + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "destructive_reclaimer"); +} + /******************************************************************************* * TRACING */ diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 01f521c3604..e0c4a1e925d 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -138,6 +138,12 @@ typedef enum { GRPC_NUM_SETTING_SETS } grpc_chttp2_setting_set; +typedef enum { + GRPC_CHTTP2_NO_GOAWAY_SEND, + GRPC_CHTTP2_GOAWAY_SEND_SCHEDULED, + GRPC_CHTTP2_GOAWAY_SENT, +} grpc_chttp2_sent_goaway_state; + /* Outstanding ping request data */ typedef struct grpc_chttp2_outstanding_ping { uint8_t id[8]; @@ -249,7 +255,7 @@ struct grpc_chttp2_transport { /** have we seen a goaway */ uint8_t seen_goaway; /** have we sent a goaway */ - uint8_t sent_goaway; + grpc_chttp2_sent_goaway_state sent_goaway_state; /** are the local settings dirty and need to be sent? */ uint8_t dirtied_local_settings; @@ -320,6 +326,18 @@ struct grpc_chttp2_transport { /* if non-NULL, close the transport with this error when writes are finished */ grpc_error *close_transport_on_writes_finished; + + /* buffer pool state */ + /** have we scheduled a benign cleanup? */ + bool benign_reclaimer_registered; + /** have we scheduled a destructive cleanup? */ + bool destructive_reclaimer_registered; + /** benign cleanup closure */ + grpc_closure benign_reclaimer; + grpc_closure benign_reclaimer_locked; + /** destructive cleanup closure */ + grpc_closure destructive_reclaimer; + grpc_closure destructive_reclaimer_locked; }; typedef enum { diff --git a/src/core/ext/transport/chttp2/transport/stream_map.c b/src/core/ext/transport/chttp2/transport/stream_map.c index 59b3a14e0a0..5f5a28446d2 100644 --- a/src/core/ext/transport/chttp2/transport/stream_map.c +++ b/src/core/ext/transport/chttp2/transport/stream_map.c @@ -151,6 +151,17 @@ size_t grpc_chttp2_stream_map_size(grpc_chttp2_stream_map *map) { return map->count - map->free; } +void *grpc_chttp2_stream_map_rand(grpc_chttp2_stream_map *map) { + if (map->count == map->free) { + return NULL; + } + if (map->free != 0) { + map->count = compact(map->keys, map->values, map->count); + map->free = 0; + } + return map->values[((size_t)rand()) % map->count]; +} + void grpc_chttp2_stream_map_for_each(grpc_chttp2_stream_map *map, void (*f)(void *user_data, uint32_t key, void *value), diff --git a/src/core/ext/transport/chttp2/transport/stream_map.h b/src/core/ext/transport/chttp2/transport/stream_map.h index e76312dd1a0..203f6406802 100644 --- a/src/core/ext/transport/chttp2/transport/stream_map.h +++ b/src/core/ext/transport/chttp2/transport/stream_map.h @@ -68,6 +68,9 @@ void *grpc_chttp2_stream_map_delete(grpc_chttp2_stream_map *map, uint32_t key); /* Return an existing key, or NULL if it does not exist */ void *grpc_chttp2_stream_map_find(grpc_chttp2_stream_map *map, uint32_t key); +/* Return a random entry */ +void *grpc_chttp2_stream_map_rand(grpc_chttp2_stream_map *map); + /* How many (populated) entries are in the stream map? */ size_t grpc_chttp2_stream_map_size(grpc_chttp2_stream_map *map); diff --git a/src/core/lib/http/httpcli.c b/src/core/lib/http/httpcli.c index 55de1b59905..411e669b531 100644 --- a/src/core/lib/http/httpcli.c +++ b/src/core/lib/http/httpcli.c @@ -70,6 +70,7 @@ typedef struct { grpc_closure done_write; grpc_closure connected; grpc_error *overall_error; + grpc_resource_quota *resource_quota; } internal_request; static grpc_httpcli_get_override g_get_override = NULL; @@ -117,6 +118,7 @@ static void finish(grpc_exec_ctx *exec_ctx, internal_request *req, gpr_slice_buffer_destroy(&req->incoming); gpr_slice_buffer_destroy(&req->outgoing); GRPC_ERROR_UNREF(req->overall_error); + grpc_resource_quota_internal_unref(exec_ctx, req->resource_quota); gpr_free(req); } @@ -223,8 +225,15 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req, } addr = &req->addresses->addrs[req->next_address++]; grpc_closure_init(&req->connected, on_connected, req); + grpc_arg arg; + arg.key = GRPC_ARG_RESOURCE_QUOTA; + arg.type = GRPC_ARG_POINTER; + arg.value.pointer.p = req->resource_quota; + arg.value.pointer.vtable = grpc_resource_quota_arg_vtable(); + grpc_channel_args args = {1, &arg}; grpc_tcp_client_connect(exec_ctx, &req->connected, &req->ep, - req->context->pollset_set, addr, req->deadline); + req->context->pollset_set, &args, addr, + req->deadline); } static void on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { @@ -240,6 +249,7 @@ static void on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { static void internal_request_begin(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, grpc_polling_entity *pollent, + grpc_resource_quota *resource_quota, const grpc_httpcli_request *request, gpr_timespec deadline, grpc_closure *on_done, grpc_httpcli_response *response, @@ -255,6 +265,7 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx, req->context = context; req->pollent = pollent; req->overall_error = GRPC_ERROR_NONE; + req->resource_quota = grpc_resource_quota_internal_ref(resource_quota); grpc_closure_init(&req->on_read, on_read, req); grpc_closure_init(&req->done_write, done_write, req); gpr_slice_buffer_init(&req->incoming); @@ -272,6 +283,7 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx, void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, grpc_polling_entity *pollent, + grpc_resource_quota *resource_quota, const grpc_httpcli_request *request, gpr_timespec deadline, grpc_closure *on_done, grpc_httpcli_response *response) { @@ -281,14 +293,15 @@ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, return; } gpr_asprintf(&name, "HTTP:GET:%s:%s", request->host, request->http.path); - internal_request_begin(exec_ctx, context, pollent, request, deadline, on_done, - response, name, + internal_request_begin(exec_ctx, context, pollent, resource_quota, request, + deadline, on_done, response, name, grpc_httpcli_format_get_request(request)); gpr_free(name); } void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, grpc_polling_entity *pollent, + grpc_resource_quota *resource_quota, const grpc_httpcli_request *request, const char *body_bytes, size_t body_size, gpr_timespec deadline, grpc_closure *on_done, @@ -301,7 +314,8 @@ void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, } gpr_asprintf(&name, "HTTP:POST:%s:%s", request->host, request->http.path); internal_request_begin( - exec_ctx, context, pollent, request, deadline, on_done, response, name, + exec_ctx, context, pollent, resource_quota, request, deadline, on_done, + response, name, grpc_httpcli_format_post_request(request, body_bytes, body_size)); gpr_free(name); } diff --git a/src/core/lib/http/httpcli.h b/src/core/lib/http/httpcli.h index 320c0f86c61..11e03b44dfd 100644 --- a/src/core/lib/http/httpcli.h +++ b/src/core/lib/http/httpcli.h @@ -96,6 +96,7 @@ void grpc_httpcli_context_destroy(grpc_httpcli_context *context); 'on_response' is a callback to report results to */ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, grpc_polling_entity *pollent, + grpc_resource_quota *resource_quota, const grpc_httpcli_request *request, gpr_timespec deadline, grpc_closure *on_complete, grpc_httpcli_response *response); @@ -116,6 +117,7 @@ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, Does not support ?var1=val1&var2=val2 in the path. */ void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, grpc_polling_entity *pollent, + grpc_resource_quota *resource_quota, const grpc_httpcli_request *request, const char *body_bytes, size_t body_size, gpr_timespec deadline, grpc_closure *on_complete, diff --git a/src/core/lib/iomgr/endpoint.c b/src/core/lib/iomgr/endpoint.c index f901fcf9622..74fa9c45dff 100644 --- a/src/core/lib/iomgr/endpoint.c +++ b/src/core/lib/iomgr/endpoint.c @@ -69,3 +69,7 @@ char* grpc_endpoint_get_peer(grpc_endpoint* ep) { grpc_workqueue* grpc_endpoint_get_workqueue(grpc_endpoint* ep) { return ep->vtable->get_workqueue(ep); } + +grpc_resource_user* grpc_endpoint_get_resource_user(grpc_endpoint* ep) { + return ep->vtable->get_resource_user(ep); +} diff --git a/src/core/lib/iomgr/endpoint.h b/src/core/lib/iomgr/endpoint.h index 910a6f6532a..0ac5486ff57 100644 --- a/src/core/lib/iomgr/endpoint.h +++ b/src/core/lib/iomgr/endpoint.h @@ -39,6 +39,7 @@ #include #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" +#include "src/core/lib/iomgr/resource_quota.h" /* An endpoint caps a streaming channel between two communicating processes. Examples may be: a tcp socket, , or some shared memory. */ @@ -58,6 +59,7 @@ struct grpc_endpoint_vtable { grpc_pollset_set *pollset); void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep); void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep); + grpc_resource_user *(*get_resource_user)(grpc_endpoint *ep); char *(*get_peer)(grpc_endpoint *ep); }; @@ -100,6 +102,8 @@ void grpc_endpoint_add_to_pollset_set(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_pollset_set *pollset_set); +grpc_resource_user *grpc_endpoint_get_resource_user(grpc_endpoint *endpoint); + struct grpc_endpoint { const grpc_endpoint_vtable *vtable; }; diff --git a/src/core/lib/iomgr/endpoint_pair.h b/src/core/lib/iomgr/endpoint_pair.h index 5cd78051bdf..f9de0c715ec 100644 --- a/src/core/lib/iomgr/endpoint_pair.h +++ b/src/core/lib/iomgr/endpoint_pair.h @@ -41,7 +41,8 @@ typedef struct { grpc_endpoint *server; } grpc_endpoint_pair; -grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name, - size_t read_slice_size); +grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( + const char *name, grpc_resource_quota *resource_quota, + size_t read_slice_size); #endif /* GRPC_CORE_LIB_IOMGR_ENDPOINT_PAIR_H */ diff --git a/src/core/lib/iomgr/endpoint_pair_posix.c b/src/core/lib/iomgr/endpoint_pair_posix.c index ec2cd782b14..b9ff969e810 100644 --- a/src/core/lib/iomgr/endpoint_pair_posix.c +++ b/src/core/lib/iomgr/endpoint_pair_posix.c @@ -62,20 +62,21 @@ static void create_sockets(int sv[2]) { GPR_ASSERT(grpc_set_socket_no_sigpipe_if_possible(sv[1]) == GRPC_ERROR_NONE); } -grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name, - size_t read_slice_size) { +grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( + const char *name, grpc_resource_quota *resource_quota, + size_t read_slice_size) { int sv[2]; grpc_endpoint_pair p; char *final_name; create_sockets(sv); gpr_asprintf(&final_name, "%s:client", name); - p.client = grpc_tcp_create(grpc_fd_create(sv[1], final_name), read_slice_size, - "socketpair-server"); + p.client = grpc_tcp_create(grpc_fd_create(sv[1], final_name), resource_quota, + read_slice_size, "socketpair-server"); gpr_free(final_name); gpr_asprintf(&final_name, "%s:server", name); - p.server = grpc_tcp_create(grpc_fd_create(sv[0], final_name), read_slice_size, - "socketpair-client"); + p.server = grpc_tcp_create(grpc_fd_create(sv[0], final_name), resource_quota, + read_slice_size, "socketpair-client"); gpr_free(final_name); return p; } diff --git a/src/core/lib/iomgr/endpoint_pair_windows.c b/src/core/lib/iomgr/endpoint_pair_windows.c index 5c78c95492b..93f71b745c6 100644 --- a/src/core/lib/iomgr/endpoint_pair_windows.c +++ b/src/core/lib/iomgr/endpoint_pair_windows.c @@ -82,15 +82,16 @@ static void create_sockets(SOCKET sv[2]) { sv[0] = svr_sock; } -grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name, - size_t read_slice_size) { +grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( + const char *name, grpc_resource_quota *resource_quota, + size_t read_slice_size) { SOCKET sv[2]; grpc_endpoint_pair p; create_sockets(sv); p.client = grpc_tcp_create(grpc_winsocket_create(sv[1], "endpoint:client"), - "endpoint:server"); + resource_quota, "endpoint:server"); p.server = grpc_tcp_create(grpc_winsocket_create(sv[0], "endpoint:server"), - "endpoint:client"); + resource_quota, "endpoint:client"); return p; } diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c new file mode 100644 index 00000000000..bfc905845d6 --- /dev/null +++ b/src/core/lib/iomgr/resource_quota.c @@ -0,0 +1,714 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/resource_quota.h" + +#include + +#include +#include +#include +#include + +#include "src/core/lib/iomgr/combiner.h" + +int grpc_resource_quota_trace = 0; + +struct grpc_resource_quota { + /* refcount */ + gpr_refcount refs; + + /* Master combiner lock: all activity on a quota executes under this combiner + * (so no mutex is needed for this data structure) + */ + grpc_combiner *combiner; + /* Size of the resource quota */ + int64_t size; + /* Amount of free memory in the resource quota */ + int64_t free_pool; + + /* Has rq_step been scheduled to occur? */ + bool step_scheduled; + /* Are we currently reclaiming memory */ + bool reclaiming; + /* Closure around rq_step */ + grpc_closure rq_step_closure; + /* Closure around rq_reclamation_done */ + grpc_closure rq_reclamation_done_closure; + + /* Roots of all resource user lists */ + grpc_resource_user *roots[GRPC_RULIST_COUNT]; + + char *name; +}; + +/******************************************************************************* + * list management + */ + +static void rulist_add_head(grpc_resource_user *resource_user, + grpc_rulist list) { + grpc_resource_quota *resource_quota = resource_user->resource_quota; + grpc_resource_user **root = &resource_quota->roots[list]; + if (*root == NULL) { + *root = resource_user; + resource_user->links[list].next = resource_user->links[list].prev = + resource_user; + } else { + resource_user->links[list].next = *root; + resource_user->links[list].prev = (*root)->links[list].prev; + resource_user->links[list].next->links[list].prev = + resource_user->links[list].prev->links[list].next = resource_user; + *root = resource_user; + } +} + +static void rulist_add_tail(grpc_resource_user *resource_user, + grpc_rulist list) { + grpc_resource_quota *resource_quota = resource_user->resource_quota; + grpc_resource_user **root = &resource_quota->roots[list]; + if (*root == NULL) { + *root = resource_user; + resource_user->links[list].next = resource_user->links[list].prev = + resource_user; + } else { + resource_user->links[list].next = (*root)->links[list].next; + resource_user->links[list].prev = *root; + resource_user->links[list].next->links[list].prev = + resource_user->links[list].prev->links[list].next = resource_user; + } +} + +static bool rulist_empty(grpc_resource_quota *resource_quota, + grpc_rulist list) { + return resource_quota->roots[list] == NULL; +} + +static grpc_resource_user *rulist_pop_head(grpc_resource_quota *resource_quota, + grpc_rulist list) { + grpc_resource_user **root = &resource_quota->roots[list]; + grpc_resource_user *resource_user = *root; + if (resource_user == NULL) { + return NULL; + } + if (resource_user->links[list].next == resource_user) { + *root = NULL; + } else { + resource_user->links[list].next->links[list].prev = + resource_user->links[list].prev; + resource_user->links[list].prev->links[list].next = + resource_user->links[list].next; + *root = resource_user->links[list].next; + } + resource_user->links[list].next = resource_user->links[list].prev = NULL; + return resource_user; +} + +static void rulist_remove(grpc_resource_user *resource_user, grpc_rulist list) { + if (resource_user->links[list].next == NULL) return; + grpc_resource_quota *resource_quota = resource_user->resource_quota; + if (resource_quota->roots[list] == resource_user) { + resource_quota->roots[list] = resource_user->links[list].next; + if (resource_quota->roots[list] == resource_user) { + resource_quota->roots[list] = NULL; + } + } + resource_user->links[list].next->links[list].prev = + resource_user->links[list].prev; + resource_user->links[list].prev->links[list].next = + resource_user->links[list].next; +} + +/******************************************************************************* + * resource quota state machine + */ + +static bool rq_alloc(grpc_exec_ctx *exec_ctx, + grpc_resource_quota *resource_quota); +static bool rq_reclaim_from_per_user_free_pool( + grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota); +static bool rq_reclaim(grpc_exec_ctx *exec_ctx, + grpc_resource_quota *resource_quota, bool destructive); + +static void rq_step(grpc_exec_ctx *exec_ctx, void *rq, grpc_error *error) { + grpc_resource_quota *resource_quota = rq; + resource_quota->step_scheduled = false; + do { + if (rq_alloc(exec_ctx, resource_quota)) goto done; + } while (rq_reclaim_from_per_user_free_pool(exec_ctx, resource_quota)); + rq_reclaim(exec_ctx, resource_quota, false) || + rq_reclaim(exec_ctx, resource_quota, true); +done: + grpc_resource_quota_internal_unref(exec_ctx, resource_quota); +} + +static void rq_step_sched(grpc_exec_ctx *exec_ctx, + grpc_resource_quota *resource_quota) { + if (resource_quota->step_scheduled) return; + resource_quota->step_scheduled = true; + grpc_resource_quota_internal_ref(resource_quota); + grpc_combiner_execute_finally(exec_ctx, resource_quota->combiner, + &resource_quota->rq_step_closure, + GRPC_ERROR_NONE, false); +} + +/* returns true if all allocations are completed */ +static bool rq_alloc(grpc_exec_ctx *exec_ctx, + grpc_resource_quota *resource_quota) { + grpc_resource_user *resource_user; + while ((resource_user = rulist_pop_head(resource_quota, + GRPC_RULIST_AWAITING_ALLOCATION))) { + gpr_mu_lock(&resource_user->mu); + if (resource_user->free_pool < 0 && + -resource_user->free_pool <= resource_quota->free_pool) { + int64_t amt = -resource_user->free_pool; + resource_user->free_pool = 0; + resource_quota->free_pool -= amt; + if (grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, "RQ %s %s: grant alloc %" PRId64 + " bytes; rq_free_pool -> %" PRId64, + resource_quota->name, resource_user->name, amt, + resource_quota->free_pool); + } + } else if (grpc_resource_quota_trace && resource_user->free_pool >= 0) { + gpr_log(GPR_DEBUG, "RQ %s %s: discard already satisfied alloc request", + resource_quota->name, resource_user->name); + } + if (resource_user->free_pool >= 0) { + resource_user->allocating = false; + grpc_exec_ctx_enqueue_list(exec_ctx, &resource_user->on_allocated, NULL); + gpr_mu_unlock(&resource_user->mu); + } else { + rulist_add_head(resource_user, GRPC_RULIST_AWAITING_ALLOCATION); + gpr_mu_unlock(&resource_user->mu); + return false; + } + } + return true; +} + +/* returns true if any memory could be reclaimed from buffers */ +static bool rq_reclaim_from_per_user_free_pool( + grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota) { + grpc_resource_user *resource_user; + while ((resource_user = rulist_pop_head(resource_quota, + GRPC_RULIST_NON_EMPTY_FREE_POOL))) { + gpr_mu_lock(&resource_user->mu); + if (resource_user->free_pool > 0) { + int64_t amt = resource_user->free_pool; + resource_user->free_pool = 0; + resource_quota->free_pool += amt; + if (grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, "RQ %s %s: reclaim_from_per_user_free_pool %" PRId64 + " bytes; rq_free_pool -> %" PRId64, + resource_quota->name, resource_user->name, amt, + resource_quota->free_pool); + } + gpr_mu_unlock(&resource_user->mu); + return true; + } else { + gpr_mu_unlock(&resource_user->mu); + } + } + return false; +} + +/* returns true if reclamation is proceeding */ +static bool rq_reclaim(grpc_exec_ctx *exec_ctx, + grpc_resource_quota *resource_quota, bool destructive) { + if (resource_quota->reclaiming) return true; + grpc_rulist list = destructive ? GRPC_RULIST_RECLAIMER_DESTRUCTIVE + : GRPC_RULIST_RECLAIMER_BENIGN; + grpc_resource_user *resource_user = rulist_pop_head(resource_quota, list); + if (resource_user == NULL) return false; + if (grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, "RQ %s %s: initiate %s reclamation", + resource_quota->name, resource_user->name, + destructive ? "destructive" : "benign"); + } + resource_quota->reclaiming = true; + grpc_resource_quota_internal_ref(resource_quota); + grpc_closure *c = resource_user->reclaimers[destructive]; + resource_user->reclaimers[destructive] = NULL; + grpc_closure_run(exec_ctx, c, GRPC_ERROR_NONE); + return true; +} + +/******************************************************************************* + * ru_slice: a slice implementation that is backed by a grpc_resource_user + */ + +typedef struct { + gpr_slice_refcount base; + gpr_refcount refs; + grpc_resource_user *resource_user; + size_t size; +} ru_slice_refcount; + +static void ru_slice_ref(void *p) { + ru_slice_refcount *rc = p; + gpr_ref(&rc->refs); +} + +static void ru_slice_unref(void *p) { + ru_slice_refcount *rc = p; + if (gpr_unref(&rc->refs)) { + /* TODO(ctiller): this is dangerous, but I think safe for now: + we have no guarantee here that we're at a safe point for creating an + execution context, but we have no way of writing this code otherwise. + In the future: consider lifting gpr_slice to grpc, and offering an + internal_{ref,unref} pair that is execution context aware. + Alternatively, + make exec_ctx be thread local and 'do the right thing' (whatever that + is) + if NULL */ + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, rc->resource_user, rc->size); + grpc_exec_ctx_finish(&exec_ctx); + gpr_free(rc); + } +} + +static gpr_slice ru_slice_create(grpc_resource_user *resource_user, + size_t size) { + ru_slice_refcount *rc = gpr_malloc(sizeof(ru_slice_refcount) + size); + rc->base.ref = ru_slice_ref; + rc->base.unref = ru_slice_unref; + gpr_ref_init(&rc->refs, 1); + rc->resource_user = resource_user; + rc->size = size; + gpr_slice slice; + slice.refcount = &rc->base; + slice.data.refcounted.bytes = (uint8_t *)(rc + 1); + slice.data.refcounted.length = size; + return slice; +} + +/******************************************************************************* + * grpc_resource_quota internal implementation: resource user manipulation under + * the combiner + */ + +static void ru_allocate(grpc_exec_ctx *exec_ctx, void *ru, grpc_error *error) { + grpc_resource_user *resource_user = ru; + if (rulist_empty(resource_user->resource_quota, + GRPC_RULIST_AWAITING_ALLOCATION)) { + rq_step_sched(exec_ctx, resource_user->resource_quota); + } + rulist_add_tail(resource_user, GRPC_RULIST_AWAITING_ALLOCATION); +} + +static void ru_add_to_free_pool(grpc_exec_ctx *exec_ctx, void *ru, + grpc_error *error) { + grpc_resource_user *resource_user = ru; + if (!rulist_empty(resource_user->resource_quota, + GRPC_RULIST_AWAITING_ALLOCATION) && + rulist_empty(resource_user->resource_quota, + GRPC_RULIST_NON_EMPTY_FREE_POOL)) { + rq_step_sched(exec_ctx, resource_user->resource_quota); + } + rulist_add_tail(resource_user, GRPC_RULIST_NON_EMPTY_FREE_POOL); +} + +static void ru_post_benign_reclaimer(grpc_exec_ctx *exec_ctx, void *ru, + grpc_error *error) { + grpc_resource_user *resource_user = ru; + if (!rulist_empty(resource_user->resource_quota, + GRPC_RULIST_AWAITING_ALLOCATION) && + rulist_empty(resource_user->resource_quota, + GRPC_RULIST_NON_EMPTY_FREE_POOL) && + rulist_empty(resource_user->resource_quota, + GRPC_RULIST_RECLAIMER_BENIGN)) { + rq_step_sched(exec_ctx, resource_user->resource_quota); + } + rulist_add_tail(resource_user, GRPC_RULIST_RECLAIMER_BENIGN); +} + +static void ru_post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *ru, + grpc_error *error) { + grpc_resource_user *resource_user = ru; + if (!rulist_empty(resource_user->resource_quota, + GRPC_RULIST_AWAITING_ALLOCATION) && + rulist_empty(resource_user->resource_quota, + GRPC_RULIST_NON_EMPTY_FREE_POOL) && + rulist_empty(resource_user->resource_quota, + GRPC_RULIST_RECLAIMER_BENIGN) && + rulist_empty(resource_user->resource_quota, + GRPC_RULIST_RECLAIMER_DESTRUCTIVE)) { + rq_step_sched(exec_ctx, resource_user->resource_quota); + } + rulist_add_tail(resource_user, GRPC_RULIST_RECLAIMER_DESTRUCTIVE); +} + +static void ru_destroy(grpc_exec_ctx *exec_ctx, void *ru, grpc_error *error) { + grpc_resource_user *resource_user = ru; + GPR_ASSERT(resource_user->allocated == 0); + for (int i = 0; i < GRPC_RULIST_COUNT; i++) { + rulist_remove(resource_user, (grpc_rulist)i); + } + grpc_exec_ctx_sched(exec_ctx, resource_user->reclaimers[0], + GRPC_ERROR_CANCELLED, NULL); + grpc_exec_ctx_sched(exec_ctx, resource_user->reclaimers[1], + GRPC_ERROR_CANCELLED, NULL); + grpc_exec_ctx_sched(exec_ctx, (grpc_closure *)gpr_atm_no_barrier_load( + &resource_user->on_done_destroy_closure), + GRPC_ERROR_NONE, NULL); + if (resource_user->free_pool != 0) { + resource_user->resource_quota->free_pool += resource_user->free_pool; + rq_step_sched(exec_ctx, resource_user->resource_quota); + } +} + +static void ru_allocated_slices(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_resource_user_slice_allocator *slice_allocator = arg; + if (error == GRPC_ERROR_NONE) { + for (size_t i = 0; i < slice_allocator->count; i++) { + gpr_slice_buffer_add_indexed( + slice_allocator->dest, ru_slice_create(slice_allocator->resource_user, + slice_allocator->length)); + } + } + grpc_closure_run(exec_ctx, &slice_allocator->on_done, GRPC_ERROR_REF(error)); +} + +/******************************************************************************* + * grpc_resource_quota internal implementation: quota manipulation under the + * combiner + */ + +typedef struct { + int64_t size; + grpc_resource_quota *resource_quota; + grpc_closure closure; +} rq_resize_args; + +static void rq_resize(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) { + rq_resize_args *a = args; + int64_t delta = a->size - a->resource_quota->size; + a->resource_quota->size += delta; + a->resource_quota->free_pool += delta; + rq_step_sched(exec_ctx, a->resource_quota); + grpc_resource_quota_internal_unref(exec_ctx, a->resource_quota); + gpr_free(a); +} + +static void rq_reclamation_done(grpc_exec_ctx *exec_ctx, void *rq, + grpc_error *error) { + grpc_resource_quota *resource_quota = rq; + resource_quota->reclaiming = false; + rq_step_sched(exec_ctx, resource_quota); + grpc_resource_quota_internal_unref(exec_ctx, resource_quota); +} + +/******************************************************************************* + * grpc_resource_quota api + */ + +/* Public API */ +grpc_resource_quota *grpc_resource_quota_create(const char *name) { + grpc_resource_quota *resource_quota = gpr_malloc(sizeof(*resource_quota)); + gpr_ref_init(&resource_quota->refs, 1); + resource_quota->combiner = grpc_combiner_create(NULL); + resource_quota->free_pool = INT64_MAX; + resource_quota->size = INT64_MAX; + resource_quota->step_scheduled = false; + resource_quota->reclaiming = false; + if (name != NULL) { + resource_quota->name = gpr_strdup(name); + } else { + gpr_asprintf(&resource_quota->name, "anonymous_pool_%" PRIxPTR, + (intptr_t)resource_quota); + } + grpc_closure_init(&resource_quota->rq_step_closure, rq_step, resource_quota); + grpc_closure_init(&resource_quota->rq_reclamation_done_closure, + rq_reclamation_done, resource_quota); + for (int i = 0; i < GRPC_RULIST_COUNT; i++) { + resource_quota->roots[i] = NULL; + } + return resource_quota; +} + +void grpc_resource_quota_internal_unref(grpc_exec_ctx *exec_ctx, + grpc_resource_quota *resource_quota) { + if (gpr_unref(&resource_quota->refs)) { + grpc_combiner_destroy(exec_ctx, resource_quota->combiner); + gpr_free(resource_quota->name); + gpr_free(resource_quota); + } +} + +/* Public API */ +void grpc_resource_quota_unref(grpc_resource_quota *resource_quota) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); + grpc_exec_ctx_finish(&exec_ctx); +} + +grpc_resource_quota *grpc_resource_quota_internal_ref( + grpc_resource_quota *resource_quota) { + gpr_ref(&resource_quota->refs); + return resource_quota; +} + +/* Public API */ +void grpc_resource_quota_ref(grpc_resource_quota *resource_quota) { + grpc_resource_quota_internal_ref(resource_quota); +} + +/* Public API */ +void grpc_resource_quota_resize(grpc_resource_quota *resource_quota, + size_t size) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + rq_resize_args *a = gpr_malloc(sizeof(*a)); + a->resource_quota = grpc_resource_quota_internal_ref(resource_quota); + a->size = (int64_t)size; + grpc_closure_init(&a->closure, rq_resize, a); + grpc_combiner_execute(&exec_ctx, resource_quota->combiner, &a->closure, + GRPC_ERROR_NONE, false); + grpc_exec_ctx_finish(&exec_ctx); +} + +/******************************************************************************* + * grpc_resource_user channel args api + */ + +grpc_resource_quota *grpc_resource_quota_from_channel_args( + const grpc_channel_args *channel_args) { + for (size_t i = 0; i < channel_args->num_args; i++) { + if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { + if (channel_args->args[i].type == GRPC_ARG_POINTER) { + return grpc_resource_quota_internal_ref( + channel_args->args[i].value.pointer.p); + } else { + gpr_log(GPR_DEBUG, GRPC_ARG_RESOURCE_QUOTA " should be a pointer"); + } + } + } + return grpc_resource_quota_create(NULL); +} + +static void *rq_copy(void *rq) { + grpc_resource_quota_ref(rq); + return rq; +} + +static void rq_destroy(void *rq) { grpc_resource_quota_unref(rq); } + +static int rq_cmp(void *a, void *b) { return GPR_ICMP(a, b); } + +const grpc_arg_pointer_vtable *grpc_resource_quota_arg_vtable(void) { + static const grpc_arg_pointer_vtable vtable = {rq_copy, rq_destroy, rq_cmp}; + return &vtable; +} + +/******************************************************************************* + * grpc_resource_user api + */ + +void grpc_resource_user_init(grpc_resource_user *resource_user, + grpc_resource_quota *resource_quota, + const char *name) { + resource_user->resource_quota = + grpc_resource_quota_internal_ref(resource_quota); + grpc_closure_init(&resource_user->allocate_closure, &ru_allocate, + resource_user); + grpc_closure_init(&resource_user->add_to_free_pool_closure, + &ru_add_to_free_pool, resource_user); + grpc_closure_init(&resource_user->post_reclaimer_closure[0], + &ru_post_benign_reclaimer, resource_user); + grpc_closure_init(&resource_user->post_reclaimer_closure[1], + &ru_post_destructive_reclaimer, resource_user); + grpc_closure_init(&resource_user->destroy_closure, &ru_destroy, + resource_user); + gpr_mu_init(&resource_user->mu); + resource_user->allocated = 0; + resource_user->free_pool = 0; + grpc_closure_list_init(&resource_user->on_allocated); + resource_user->allocating = false; + resource_user->added_to_free_pool = false; + gpr_atm_no_barrier_store(&resource_user->on_done_destroy_closure, 0); + resource_user->reclaimers[0] = NULL; + resource_user->reclaimers[1] = NULL; + for (int i = 0; i < GRPC_RULIST_COUNT; i++) { + resource_user->links[i].next = resource_user->links[i].prev = NULL; + } + if (name != NULL) { + resource_user->name = gpr_strdup(name); + } else { + gpr_asprintf(&resource_user->name, "anonymous_resource_user_%" PRIxPTR, + (intptr_t)resource_user); + } +} + +void grpc_resource_user_shutdown(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user, + grpc_closure *on_done) { + gpr_mu_lock(&resource_user->mu); + GPR_ASSERT(gpr_atm_no_barrier_load(&resource_user->on_done_destroy_closure) == + 0); + gpr_atm_no_barrier_store(&resource_user->on_done_destroy_closure, + (gpr_atm)on_done); + if (resource_user->allocated == 0) { + grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, + &resource_user->destroy_closure, GRPC_ERROR_NONE, + false); + } + gpr_mu_unlock(&resource_user->mu); +} + +void grpc_resource_user_destroy(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user) { + grpc_resource_quota_internal_unref(exec_ctx, resource_user->resource_quota); + gpr_mu_destroy(&resource_user->mu); + gpr_free(resource_user->name); +} + +void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user, size_t size, + grpc_closure *optional_on_done) { + gpr_mu_lock(&resource_user->mu); + grpc_closure *on_done_destroy = (grpc_closure *)gpr_atm_no_barrier_load( + &resource_user->on_done_destroy_closure); + if (on_done_destroy != NULL) { + /* already shutdown */ + if (grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR " after shutdown", + resource_user->resource_quota->name, resource_user->name, size); + } + grpc_exec_ctx_sched( + exec_ctx, optional_on_done, + GRPC_ERROR_CREATE("Buffer pool user is already shutdown"), NULL); + gpr_mu_unlock(&resource_user->mu); + return; + } + resource_user->allocated += (int64_t)size; + resource_user->free_pool -= (int64_t)size; + if (grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR "; allocated -> %" PRId64 + ", free_pool -> %" PRId64, + resource_user->resource_quota->name, resource_user->name, size, + resource_user->allocated, resource_user->free_pool); + } + if (resource_user->free_pool < 0) { + grpc_closure_list_append(&resource_user->on_allocated, optional_on_done, + GRPC_ERROR_NONE); + if (!resource_user->allocating) { + resource_user->allocating = true; + grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, + &resource_user->allocate_closure, GRPC_ERROR_NONE, + false); + } + } else { + grpc_exec_ctx_sched(exec_ctx, optional_on_done, GRPC_ERROR_NONE, NULL); + } + gpr_mu_unlock(&resource_user->mu); +} + +void grpc_resource_user_free(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user, size_t size) { + gpr_mu_lock(&resource_user->mu); + GPR_ASSERT(resource_user->allocated >= (int64_t)size); + bool was_zero_or_negative = resource_user->free_pool <= 0; + resource_user->free_pool += (int64_t)size; + resource_user->allocated -= (int64_t)size; + if (grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, "RQ %s %s: free %" PRIdPTR "; allocated -> %" PRId64 + ", free_pool -> %" PRId64, + resource_user->resource_quota->name, resource_user->name, size, + resource_user->allocated, resource_user->free_pool); + } + bool is_bigger_than_zero = resource_user->free_pool > 0; + if (is_bigger_than_zero && was_zero_or_negative && + !resource_user->added_to_free_pool) { + resource_user->added_to_free_pool = true; + grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, + &resource_user->add_to_free_pool_closure, + GRPC_ERROR_NONE, false); + } + grpc_closure *on_done_destroy = (grpc_closure *)gpr_atm_no_barrier_load( + &resource_user->on_done_destroy_closure); + if (on_done_destroy != NULL && resource_user->allocated == 0) { + grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, + &resource_user->destroy_closure, GRPC_ERROR_NONE, + false); + } + gpr_mu_unlock(&resource_user->mu); +} + +void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user, + bool destructive, + grpc_closure *closure) { + if (gpr_atm_acq_load(&resource_user->on_done_destroy_closure) == 0) { + GPR_ASSERT(resource_user->reclaimers[destructive] == NULL); + resource_user->reclaimers[destructive] = closure; + grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, + &resource_user->post_reclaimer_closure[destructive], + GRPC_ERROR_NONE, false); + } else { + grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CANCELLED, NULL); + } +} + +void grpc_resource_user_finish_reclamation(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user) { + if (grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, "RQ %s %s: reclamation complete", + resource_user->resource_quota->name, resource_user->name); + } + grpc_combiner_execute( + exec_ctx, resource_user->resource_quota->combiner, + &resource_user->resource_quota->rq_reclamation_done_closure, + GRPC_ERROR_NONE, false); +} + +void grpc_resource_user_slice_allocator_init( + grpc_resource_user_slice_allocator *slice_allocator, + grpc_resource_user *resource_user, grpc_iomgr_cb_func cb, void *p) { + grpc_closure_init(&slice_allocator->on_allocated, ru_allocated_slices, + slice_allocator); + grpc_closure_init(&slice_allocator->on_done, cb, p); + slice_allocator->resource_user = resource_user; +} + +void grpc_resource_user_alloc_slices( + grpc_exec_ctx *exec_ctx, + grpc_resource_user_slice_allocator *slice_allocator, size_t length, + size_t count, gpr_slice_buffer *dest) { + slice_allocator->length = length; + slice_allocator->count = count; + slice_allocator->dest = dest; + grpc_resource_user_alloc(exec_ctx, slice_allocator->resource_user, + count * length, &slice_allocator->on_allocated); +} diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h new file mode 100644 index 00000000000..6dfac55f88c --- /dev/null +++ b/src/core/lib/iomgr/resource_quota.h @@ -0,0 +1,224 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_RESOURCE_QUOTA_H +#define GRPC_CORE_LIB_IOMGR_RESOURCE_QUOTA_H + +#include + +#include "src/core/lib/iomgr/exec_ctx.h" + +/** \file Tracks resource usage against a pool. + + The current implementation tracks only memory usage, but in the future + this may be extended to (for example) threads and file descriptors. + + A grpc_resource_quota represents the pooled resources, and + grpc_resource_user instances attach to the quota and consume those + resources. They also offer a vector for reclamation: if we become + resource constrained, grpc_resource_user instances are asked (in turn) to + free up whatever they can so that the system as a whole can make progress. + + There are three kinds of reclamation that take place, in order of increasing + invasiveness: + - an internal reclamation, where cached resource at the resource user level + is returned to the quota + - a benign reclamation phase, whereby resources that are in use but are not + helping anything make progress are reclaimed + - a destructive reclamation, whereby resources that are helping something + make progress may be enacted so that at least one part of the system can + complete. + + Only one reclamation will be outstanding for a given quota at a given time. + On each reclamation attempt, the kinds of reclamation are tried in order of + increasing invasiveness, stopping at the first one that succeeds. Thus, on a + given reclamation attempt, if internal and benign reclamation both fail, it + will wind up doing a destructive reclamation. However, the next reclamation + attempt may then be able to get what it needs via internal or benign + reclamation, due to resources that may have been freed up by the destructive + reclamation in the previous attempt. + + Future work will be to expose the current resource pressure so that back + pressure can be applied to avoid reclamation phases starting. + + Resource users own references to resource quotas, and resource quotas + maintain lists of users (which users arrange to leave before they are + destroyed) */ + +extern int grpc_resource_quota_trace; + +grpc_resource_quota *grpc_resource_quota_internal_ref( + grpc_resource_quota *resource_quota); +void grpc_resource_quota_internal_unref(grpc_exec_ctx *exec_ctx, + grpc_resource_quota *resource_quota); +grpc_resource_quota *grpc_resource_quota_from_channel_args( + const grpc_channel_args *channel_args); + +/* Resource users are kept in (potentially) several intrusive linked lists + at once. These are the list names. */ +typedef enum { + /* Resource users that are waiting for an allocation */ + GRPC_RULIST_AWAITING_ALLOCATION, + /* Resource users that have free memory available for internal reclamation */ + GRPC_RULIST_NON_EMPTY_FREE_POOL, + /* Resource users that have published a benign reclamation is available */ + GRPC_RULIST_RECLAIMER_BENIGN, + /* Resource users that have published a destructive reclamation is + available */ + GRPC_RULIST_RECLAIMER_DESTRUCTIVE, + /* Number of lists: must be last */ + GRPC_RULIST_COUNT +} grpc_rulist; + +typedef struct grpc_resource_user grpc_resource_user; + +/* Internal linked list pointers for a resource user */ +typedef struct { + grpc_resource_user *next; + grpc_resource_user *prev; +} grpc_resource_user_link; + +struct grpc_resource_user { + /* The quota this resource user consumes from */ + grpc_resource_quota *resource_quota; + + /* Closure to schedule an allocation under the resource quota combiner lock */ + grpc_closure allocate_closure; + /* Closure to publish a non empty free pool under the resource quota combiner + lock */ + grpc_closure add_to_free_pool_closure; + + gpr_mu mu; + /* Total allocated memory outstanding by this resource user in bytes; + always positive */ + int64_t allocated; + /* The amount of memory (in bytes) this user has cached for its own use: to + avoid quota contention, each resource user can keep some memory in + addition to what it is immediately using (e.g., for caching), and the quota + can pull it back under memory pressure. + This value can become negative if more memory has been requested than + existed in the free pool, at which point the quota is consulted to bring + this value non-negative (asynchronously). */ + int64_t free_pool; + /* A list of closures to call once free_pool becomes non-negative - ie when + all outstanding allocations have been granted. */ + grpc_closure_list on_allocated; + /* True if we are currently trying to allocate from the quota, false if not */ + bool allocating; + /* True if we are currently trying to add ourselves to the non-free quota + list, false otherwise */ + bool added_to_free_pool; + + /* Reclaimers: index 0 is the benign reclaimer, 1 is the destructive reclaimer + */ + grpc_closure *reclaimers[2]; + /* Trampoline closures to finish reclamation and re-enter the quota combiner + lock */ + grpc_closure post_reclaimer_closure[2]; + + /* Closure to execute under the quota combiner to de-register and shutdown the + resource user */ + grpc_closure destroy_closure; + /* User supplied closure to call once the user has finished shutting down AND + all outstanding allocations have been freed. Real type is grpc_closure*, + but it's stored as an atomic to avoid a mutex on some fast paths. */ + gpr_atm on_done_destroy_closure; + + /* Links in the various grpc_rulist lists */ + grpc_resource_user_link links[GRPC_RULIST_COUNT]; + + /* The name of this resource user, for debugging/tracing */ + char *name; +}; + +void grpc_resource_user_init(grpc_resource_user *resource_user, + grpc_resource_quota *resource_quota, + const char *name); +void grpc_resource_user_shutdown(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user, + grpc_closure *on_done); +void grpc_resource_user_destroy(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user); + +/* Allocate from the resource user (and its quota). + If optional_on_done is NULL, then allocate immediately. This may push the + quota over-limit, at which point reclamation will kick in. + If optional_on_done is non-NULL, it will be scheduled when the allocation has + been granted by the quota. */ +void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user, size_t size, + grpc_closure *optional_on_done); +/* Release memory back to the quota */ +void grpc_resource_user_free(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user, size_t size); +/* Post a memory reclaimer to the resource user. Only one benign and one + destructive reclaimer can be posted at once. When executed, the reclaimer + MUST call grpc_resource_user_finish_reclamation before it completes, to + return control to the resource quota. */ +void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user, + bool destructive, grpc_closure *closure); +/* Finish a reclamation step */ +void grpc_resource_user_finish_reclamation(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user); + +/* Helper to allocate slices from a resource user */ +typedef struct grpc_resource_user_slice_allocator { + /* Closure for when a resource user allocation completes */ + grpc_closure on_allocated; + /* Closure to call when slices have been allocated */ + grpc_closure on_done; + /* Length of slices to allocate on the current request */ + size_t length; + /* Number of slices to allocate on the current request */ + size_t count; + /* Destination for slices to allocate on the current request */ + gpr_slice_buffer *dest; + /* Parent resource user */ + grpc_resource_user *resource_user; +} grpc_resource_user_slice_allocator; + +/* Initialize a slice allocator. + When an allocation is completed, calls \a cb with arg \p. */ +void grpc_resource_user_slice_allocator_init( + grpc_resource_user_slice_allocator *slice_allocator, + grpc_resource_user *resource_user, grpc_iomgr_cb_func cb, void *p); + +/* Allocate \a count slices of length \a length into \a dest. Only one request + can be outstanding at a time. */ +void grpc_resource_user_alloc_slices( + grpc_exec_ctx *exec_ctx, + grpc_resource_user_slice_allocator *slice_allocator, size_t length, + size_t count, gpr_slice_buffer *dest); + +#endif /* GRPC_CORE_LIB_IOMGR_RESOURCE_QUOTA_H */ diff --git a/src/core/lib/iomgr/tcp_client.h b/src/core/lib/iomgr/tcp_client.h index fe1e7f4d3ec..18e6e60ebce 100644 --- a/src/core/lib/iomgr/tcp_client.h +++ b/src/core/lib/iomgr/tcp_client.h @@ -39,6 +39,10 @@ #include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/iomgr/resolve_address.h" +/* Channel arg (integer) setting how large a slice to try and read from the wire + each time recvmsg (or equivalent) is called */ +#define GRPC_ARG_TCP_READ_CHUNK_SIZE "grpc.experimental.tcp_read_chunk_size" + /* Asynchronously connect to an address (specified as (addr, len)), and call cb with arg and the completed connection when done (or call cb with arg and NULL on failure). @@ -47,6 +51,7 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_connect, grpc_endpoint **endpoint, grpc_pollset_set *interested_parties, + const grpc_channel_args *channel_args, const grpc_resolved_address *addr, gpr_timespec deadline); diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c index e7ae1ef6957..bc08c94ee00 100644 --- a/src/core/lib/iomgr/tcp_client_posix.c +++ b/src/core/lib/iomgr/tcp_client_posix.c @@ -35,7 +35,7 @@ #ifdef GRPC_POSIX_SOCKET -#include "src/core/lib/iomgr/tcp_client.h" +#include "src/core/lib/iomgr/tcp_client_posix.h" #include #include @@ -47,6 +47,7 @@ #include #include +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr_posix.h" #include "src/core/lib/iomgr/sockaddr_utils.h" @@ -69,6 +70,7 @@ typedef struct { char *addr_str; grpc_endpoint **ep; grpc_closure *closure; + grpc_channel_args *channel_args; } async_connect; static grpc_error *prepare_socket(const grpc_resolved_address *addr, int fd) { @@ -114,10 +116,39 @@ static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { if (done) { gpr_mu_destroy(&ac->mu); gpr_free(ac->addr_str); + grpc_channel_args_destroy(ac->channel_args); gpr_free(ac); } } +grpc_endpoint *grpc_tcp_client_create_from_fd( + grpc_exec_ctx *exec_ctx, grpc_fd *fd, const grpc_channel_args *channel_args, + const char *addr_str) { + size_t tcp_read_chunk_size = GRPC_TCP_DEFAULT_READ_SLICE_SIZE; + grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); + if (channel_args != NULL) { + for (size_t i = 0; i < channel_args->num_args; i++) { + if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_TCP_READ_CHUNK_SIZE)) { + grpc_integer_options options = {(int)tcp_read_chunk_size, 1, + 8 * 1024 * 1024}; + tcp_read_chunk_size = (size_t)grpc_channel_arg_get_integer( + &channel_args->args[i], options); + } else if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { + grpc_resource_quota_internal_unref(exec_ctx, resource_quota); + resource_quota = grpc_resource_quota_internal_ref( + channel_args->args[i].value.pointer.p); + } + } + } + + grpc_endpoint *ep = + grpc_tcp_create(fd, resource_quota, tcp_read_chunk_size, addr_str); + grpc_resource_quota_internal_unref(exec_ctx, resource_quota); + return ep; +} + static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { async_connect *ac = acp; int so_error = 0; @@ -165,7 +196,8 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { switch (so_error) { case 0: grpc_pollset_set_del_fd(exec_ctx, ac->interested_parties, fd); - *ep = grpc_tcp_create(fd, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, ac->addr_str); + *ep = grpc_tcp_client_create_from_fd(exec_ctx, fd, ac->channel_args, + ac->addr_str); fd = NULL; break; case ENOBUFS: @@ -215,6 +247,7 @@ finish: if (done) { gpr_mu_destroy(&ac->mu); gpr_free(ac->addr_str); + grpc_channel_args_destroy(ac->channel_args); gpr_free(ac); } grpc_exec_ctx_sched(exec_ctx, closure, error, NULL); @@ -223,6 +256,7 @@ finish: static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, grpc_pollset_set *interested_parties, + const grpc_channel_args *channel_args, const grpc_resolved_address *addr, gpr_timespec deadline) { int fd; @@ -270,7 +304,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, fdobj = grpc_fd_create(fd, name); if (err >= 0) { - *ep = grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str); + *ep = + grpc_tcp_client_create_from_fd(exec_ctx, fdobj, channel_args, addr_str); grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_NONE, NULL); goto done; } @@ -295,6 +330,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, ac->refs = 2; ac->write_closure.cb = on_writable; ac->write_closure.cb_arg = ac; + ac->channel_args = grpc_channel_args_copy(channel_args); if (grpc_tcp_trace) { gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", @@ -316,16 +352,18 @@ done: // overridden by api_fuzzer.c void (*grpc_tcp_client_connect_impl)( grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, - grpc_pollset_set *interested_parties, const grpc_resolved_address *addr, + grpc_pollset_set *interested_parties, const grpc_channel_args *channel_args, + const grpc_resolved_address *addr, gpr_timespec deadline) = tcp_client_connect_impl; void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, grpc_pollset_set *interested_parties, + const grpc_channel_args *channel_args, const grpc_resolved_address *addr, gpr_timespec deadline) { - grpc_tcp_client_connect_impl(exec_ctx, closure, ep, interested_parties, addr, - deadline); + grpc_tcp_client_connect_impl(exec_ctx, closure, ep, interested_parties, + channel_args, addr, deadline); } #endif diff --git a/src/core/lib/iomgr/tcp_client_posix.h b/src/core/lib/iomgr/tcp_client_posix.h new file mode 100644 index 00000000000..efc5fcd5bba --- /dev/null +++ b/src/core/lib/iomgr/tcp_client_posix.h @@ -0,0 +1,45 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_TCP_CLIENT_POSIX_H +#define GRPC_CORE_LIB_IOMGR_TCP_CLIENT_POSIX_H + +#include "src/core/lib/iomgr/endpoint.h" +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/tcp_client.h" + +grpc_endpoint *grpc_tcp_client_create_from_fd( + grpc_exec_ctx *exec_ctx, grpc_fd *fd, const grpc_channel_args *channel_args, + const char *addr_str); + +#endif /* GRPC_CORE_LIB_IOMGR_TCP_CLIENT_POSIX_H */ diff --git a/src/core/lib/iomgr/tcp_client_windows.c b/src/core/lib/iomgr/tcp_client_windows.c index fdd8c1a1f81..30f7c66f159 100644 --- a/src/core/lib/iomgr/tcp_client_windows.c +++ b/src/core/lib/iomgr/tcp_client_windows.c @@ -43,6 +43,7 @@ #include #include +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/iocp_windows.h" #include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/iomgr/sockaddr_utils.h" @@ -61,13 +62,16 @@ typedef struct { int refs; grpc_closure on_connect; grpc_endpoint **endpoint; + grpc_resource_quota *resource_quota; } async_connect; -static void async_connect_unlock_and_cleanup(async_connect *ac, +static void async_connect_unlock_and_cleanup(grpc_exec_ctx *exec_ctx, + async_connect *ac, grpc_winsocket *socket) { int done = (--ac->refs == 0); gpr_mu_unlock(&ac->mu); if (done) { + grpc_resource_quota_internal_unref(exec_ctx, ac->resource_quota); gpr_mu_destroy(&ac->mu); gpr_free(ac->addr_name); gpr_free(ac); @@ -83,7 +87,7 @@ static void on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { if (socket != NULL) { grpc_winsocket_shutdown(socket); } - async_connect_unlock_and_cleanup(ac, socket); + async_connect_unlock_and_cleanup(exec_ctx, ac, socket); } static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { @@ -113,12 +117,12 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { if (!wsa_success) { error = GRPC_WSA_ERROR(WSAGetLastError(), "ConnectEx"); } else { - *ep = grpc_tcp_create(socket, ac->addr_name); + *ep = grpc_tcp_create(socket, ac->resource_quota, ac->addr_name); socket = NULL; } } - async_connect_unlock_and_cleanup(ac, socket); + async_connect_unlock_and_cleanup(exec_ctx, ac, socket); /* If the connection was aborted, the callback was already called when the deadline was met. */ grpc_exec_ctx_sched(exec_ctx, on_done, error, NULL); @@ -129,6 +133,7 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, grpc_endpoint **endpoint, grpc_pollset_set *interested_parties, + const grpc_channel_args *channel_args, const grpc_resolved_address *addr, gpr_timespec deadline) { SOCKET sock = INVALID_SOCKET; @@ -144,6 +149,17 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, grpc_winsocket_callback_info *info; grpc_error *error = GRPC_ERROR_NONE; + grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); + if (channel_args != NULL) { + for (size_t i = 0; i < channel_args->num_args; i++) { + if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { + grpc_resource_quota_internal_unref(exec_ctx, resource_quota); + resource_quota = grpc_resource_quota_internal_ref( + channel_args->args[i].value.pointer.p); + } + } + } + *endpoint = NULL; /* Use dualstack sockets where available. */ @@ -177,8 +193,8 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, grpc_sockaddr_make_wildcard6(0, &local_address); - status = - bind(sock, (struct sockaddr *)&local_address.addr, local_address.len); + status = bind(sock, (struct sockaddr *)&local_address.addr, + (int)local_address.len); if (status != 0) { error = GRPC_WSA_ERROR(WSAGetLastError(), "bind"); goto failure; @@ -206,6 +222,7 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, ac->refs = 2; ac->addr_name = grpc_sockaddr_to_uri(addr); ac->endpoint = endpoint; + ac->resource_quota = resource_quota; grpc_closure_init(&ac->on_connect, on_connect, ac); grpc_timer_init(exec_ctx, &ac->alarm, deadline, on_alarm, ac, @@ -225,6 +242,7 @@ failure: } else if (sock != INVALID_SOCKET) { closesocket(sock); } + grpc_resource_quota_internal_unref(exec_ctx, resource_quota); grpc_exec_ctx_sched(exec_ctx, on_done, final_error, NULL); } diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index 18c63cfbaec..880af93ee1a 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -80,6 +80,7 @@ typedef struct { msg_iovlen_type iov_size; /* Number of slices to allocate per read attempt */ size_t slice_size; gpr_refcount refcount; + gpr_atm shutdown_count; /* garbage after the last read */ gpr_slice_buffer last_read_buffer; @@ -100,15 +101,29 @@ typedef struct { grpc_closure write_closure; char *peer_string; + + grpc_resource_user resource_user; + grpc_resource_user_slice_allocator slice_allocator; } grpc_tcp; static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, grpc_error *error); static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, grpc_error *error); +static void tcp_unref_closure(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, + grpc_error *error); + +static void tcp_maybe_shutdown_resource_user(grpc_exec_ctx *exec_ctx, + grpc_tcp *tcp) { + if (gpr_atm_full_fetch_add(&tcp->shutdown_count, 1) == 0) { + grpc_resource_user_shutdown(exec_ctx, &tcp->resource_user, + grpc_closure_create(tcp_unref_closure, tcp)); + } +} static void tcp_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; + tcp_maybe_shutdown_resource_user(exec_ctx, tcp); grpc_fd_shutdown(exec_ctx, tcp->em_fd); } @@ -116,6 +131,7 @@ static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { grpc_fd_orphan(exec_ctx, tcp->em_fd, tcp->release_fd_cb, tcp->release_fd, "tcp_unref_orphan"); gpr_slice_buffer_destroy(&tcp->last_read_buffer); + grpc_resource_user_destroy(exec_ctx, &tcp->resource_user); gpr_free(tcp->peer_string); gpr_free(tcp); } @@ -152,9 +168,16 @@ static void tcp_unref(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { static void tcp_ref(grpc_tcp *tcp) { gpr_ref(&tcp->refcount); } #endif +static void tcp_unref_closure(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + TCP_UNREF(exec_ctx, arg, "resource_user"); +} + static void tcp_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_network_status_unregister_endpoint(ep); grpc_tcp *tcp = (grpc_tcp *)ep; + tcp_maybe_shutdown_resource_user(exec_ctx, tcp); + gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer); TCP_UNREF(exec_ctx, tcp, "destroy"); } @@ -181,7 +204,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, } #define MAX_READ_IOVEC 4 -static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { +static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { struct msghdr msg; struct iovec iov[MAX_READ_IOVEC]; ssize_t read_bytes; @@ -192,10 +215,6 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { GPR_ASSERT(tcp->incoming_buffer->count <= MAX_READ_IOVEC); GPR_TIMER_BEGIN("tcp_continue_read", 0); - while (tcp->incoming_buffer->count < (size_t)tcp->iov_size) { - gpr_slice_buffer_add_indexed(tcp->incoming_buffer, - gpr_slice_malloc(tcp->slice_size)); - } for (i = 0; i < tcp->incoming_buffer->count; i++) { iov[i].iov_base = GPR_SLICE_START_PTR(tcp->incoming_buffer->slices[i]); iov[i].iov_len = GPR_SLICE_LENGTH(tcp->incoming_buffer->slices[i]); @@ -232,7 +251,7 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { } else if (read_bytes == 0) { /* 0 read size ==> end of stream */ gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer); - call_read_cb(exec_ctx, tcp, GRPC_ERROR_CREATE("EOF")); + call_read_cb(exec_ctx, tcp, GRPC_ERROR_CREATE("Socket closed")); TCP_UNREF(exec_ctx, tcp, "read"); } else { GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length); @@ -252,6 +271,30 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { GPR_TIMER_END("tcp_continue_read", 0); } +static void tcp_read_allocation_done(grpc_exec_ctx *exec_ctx, void *tcpp, + grpc_error *error) { + grpc_tcp *tcp = tcpp; + if (error != GRPC_ERROR_NONE) { + gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer); + gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer); + call_read_cb(exec_ctx, tcp, GRPC_ERROR_REF(error)); + TCP_UNREF(exec_ctx, tcp, "read"); + } else { + tcp_do_read(exec_ctx, tcp); + } +} + +static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { + if (tcp->incoming_buffer->count < (size_t)tcp->iov_size) { + grpc_resource_user_alloc_slices( + exec_ctx, &tcp->slice_allocator, tcp->slice_size, + (size_t)tcp->iov_size - tcp->incoming_buffer->count, + tcp->incoming_buffer); + } else { + tcp_do_read(exec_ctx, tcp); + } +} + static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, grpc_error *error) { grpc_tcp *tcp = (grpc_tcp *)arg; @@ -259,6 +302,7 @@ static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, if (error != GRPC_ERROR_NONE) { gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer); + gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer); call_read_cb(exec_ctx, tcp, GRPC_ERROR_REF(error)); TCP_UNREF(exec_ctx, tcp, "read"); } else { @@ -469,6 +513,11 @@ static grpc_workqueue *tcp_get_workqueue(grpc_endpoint *ep) { return grpc_fd_get_workqueue(tcp->em_fd); } +static grpc_resource_user *tcp_get_resource_user(grpc_endpoint *ep) { + grpc_tcp *tcp = (grpc_tcp *)ep; + return &tcp->resource_user; +} + static const grpc_endpoint_vtable vtable = {tcp_read, tcp_write, tcp_get_workqueue, @@ -476,10 +525,12 @@ static const grpc_endpoint_vtable vtable = {tcp_read, tcp_add_to_pollset_set, tcp_shutdown, tcp_destroy, + tcp_get_resource_user, tcp_get_peer}; -grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size, - const char *peer_string) { +grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, + grpc_resource_quota *resource_quota, + size_t slice_size, const char *peer_string) { grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); tcp->base.vtable = &vtable; tcp->peer_string = gpr_strdup(peer_string); @@ -492,14 +543,20 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size, tcp->slice_size = slice_size; tcp->iov_size = 1; tcp->finished_edge = true; - /* paired with unref in grpc_tcp_destroy */ - gpr_ref_init(&tcp->refcount, 1); + /* paired with unref in grpc_tcp_destroy, and with the shutdown for our + * resource_user */ + gpr_ref_init(&tcp->refcount, 2); + gpr_atm_no_barrier_store(&tcp->shutdown_count, 0); tcp->em_fd = em_fd; tcp->read_closure.cb = tcp_handle_read; tcp->read_closure.cb_arg = tcp; tcp->write_closure.cb = tcp_handle_write; tcp->write_closure.cb_arg = tcp; gpr_slice_buffer_init(&tcp->last_read_buffer); + grpc_resource_user_init(&tcp->resource_user, resource_quota, peer_string); + grpc_resource_user_slice_allocator_init(&tcp->slice_allocator, + &tcp->resource_user, + tcp_read_allocation_done, tcp); /* Tell network status tracker about new endpoint */ grpc_network_status_register_endpoint(&tcp->base); @@ -514,10 +571,13 @@ int grpc_tcp_fd(grpc_endpoint *ep) { void grpc_tcp_destroy_and_release_fd(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, int *fd, grpc_closure *done) { + grpc_network_status_unregister_endpoint(ep); grpc_tcp *tcp = (grpc_tcp *)ep; GPR_ASSERT(ep->vtable == &vtable); tcp->release_fd = fd; tcp->release_fd_cb = done; + tcp_maybe_shutdown_resource_user(exec_ctx, tcp); + gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer); TCP_UNREF(exec_ctx, tcp, "destroy"); } diff --git a/src/core/lib/iomgr/tcp_posix.h b/src/core/lib/iomgr/tcp_posix.h index 99125836d69..1c0d13f96e2 100644 --- a/src/core/lib/iomgr/tcp_posix.h +++ b/src/core/lib/iomgr/tcp_posix.h @@ -53,8 +53,8 @@ extern int grpc_tcp_trace; /* Create a tcp endpoint given a file desciptor and a read slice size. Takes ownership of fd. */ -grpc_endpoint *grpc_tcp_create(grpc_fd *fd, size_t read_slice_size, - const char *peer_string); +grpc_endpoint *grpc_tcp_create(grpc_fd *fd, grpc_resource_quota *resource_quota, + size_t read_slice_size, const char *peer_string); /* Return the tcp endpoint's fd, or -1 if this is not available. Does not release the fd. diff --git a/src/core/lib/iomgr/tcp_server.h b/src/core/lib/iomgr/tcp_server.h index 0c2a4f41da7..6eba8c40571 100644 --- a/src/core/lib/iomgr/tcp_server.h +++ b/src/core/lib/iomgr/tcp_server.h @@ -61,7 +61,8 @@ typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg, /* Create a server, initially not bound to any ports. The caller owns one ref. If shutdown_complete is not NULL, it will be used by grpc_tcp_server_unref() when the ref count reaches zero. */ -grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete, +grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, + grpc_closure *shutdown_complete, const grpc_channel_args *args, grpc_tcp_server **server); diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index 127f8595d1b..b6fc1e4ca29 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -134,6 +134,8 @@ struct grpc_tcp_server { /* next pollset to assign a channel to */ gpr_atm next_pollset_to_assign; + + grpc_resource_quota *resource_quota; }; static gpr_once check_init = GPR_ONCE_INIT; @@ -150,23 +152,37 @@ static void init(void) { #endif } -grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete, +grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, + grpc_closure *shutdown_complete, const grpc_channel_args *args, grpc_tcp_server **server) { gpr_once_init(&check_init, init); grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server)); s->so_reuseport = has_so_reuseport; + s->resource_quota = grpc_resource_quota_create(NULL); for (size_t i = 0; i < (args == NULL ? 0 : args->num_args); i++) { if (0 == strcmp(GRPC_ARG_ALLOW_REUSEPORT, args->args[i].key)) { if (args->args[i].type == GRPC_ARG_INTEGER) { s->so_reuseport = has_so_reuseport && (args->args[i].value.integer != 0); } else { + grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota); gpr_free(s); return GRPC_ERROR_CREATE(GRPC_ARG_ALLOW_REUSEPORT " must be an integer"); } + } else if (0 == strcmp(GRPC_ARG_RESOURCE_QUOTA, args->args[i].key)) { + if (args->args[i].type == GRPC_ARG_POINTER) { + grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota); + s->resource_quota = + grpc_resource_quota_internal_ref(args->args[i].value.pointer.p); + } else { + grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota); + gpr_free(s); + return GRPC_ERROR_CREATE(GRPC_ARG_RESOURCE_QUOTA + " must be a pointer to a buffer pool"); + } } } gpr_ref_init(&s->refs, 1); @@ -203,6 +219,8 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { gpr_free(sp); } + grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota); + gpr_free(s); } @@ -419,7 +437,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { sp->server->on_accept_cb( exec_ctx, sp->server->on_accept_cb_arg, - grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str), + grpc_tcp_create(fdobj, sp->server->resource_quota, + GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str), read_notifier_pollset, &acceptor); gpr_free(name); diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c index ad6769a6ba8..ae54c70d2d0 100644 --- a/src/core/lib/iomgr/tcp_server_windows.c +++ b/src/core/lib/iomgr/tcp_server_windows.c @@ -100,14 +100,32 @@ struct grpc_tcp_server { /* shutdown callback */ grpc_closure *shutdown_complete; + + grpc_resource_quota *resource_quota; }; /* Public function. Allocates the proper data structures to hold a grpc_tcp_server. */ -grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete, +grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, + grpc_closure *shutdown_complete, const grpc_channel_args *args, grpc_tcp_server **server) { grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server)); + s->resource_quota = grpc_resource_quota_create(NULL); + for (size_t i = 0; i < (args == NULL ? 0 : args->num_args); i++) { + if (0 == strcmp(GRPC_ARG_RESOURCE_QUOTA, args->args[i].key)) { + if (args->args[i].type == GRPC_ARG_POINTER) { + grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota); + s->resource_quota = + grpc_resource_quota_internal_ref(args->args[i].value.pointer.p); + } else { + grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota); + gpr_free(s); + return GRPC_ERROR_CREATE(GRPC_ARG_RESOURCE_QUOTA + " must be a pointer to a buffer pool"); + } + } + } gpr_ref_init(&s->refs, 1); gpr_mu_init(&s->mu); s->active_ports = 0; @@ -137,6 +155,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { grpc_winsocket_destroy(sp->socket); gpr_free(sp); } + grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota); gpr_free(s); } @@ -207,12 +226,13 @@ static grpc_error *prepare_socket(SOCKET sock, goto failure; } - sockname_temp.len = sizeof(struct sockaddr_storage); + int sockname_temp_len = sizeof(struct sockaddr_storage); if (getsockname(sock, (struct sockaddr *)sockname_temp.addr, - &sockname_temp.len) == SOCKET_ERROR) { + &sockname_temp_len) == SOCKET_ERROR) { error = GRPC_WSA_ERROR(WSAGetLastError(), "getsockname"); goto failure; } + sockname_temp.len = sockname_temp_len; *port = grpc_sockaddr_get_port(&sockname_temp); return GRPC_ERROR_NONE; @@ -357,8 +377,10 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { gpr_log(GPR_ERROR, "setsockopt error: %s", utf8_message); gpr_free(utf8_message); } + int peer_name_len = (int)peer_name.len; err = - getpeername(sock, (struct sockaddr *)peer_name.addr, &peer_name.len); + getpeername(sock, (struct sockaddr *)peer_name.addr, &peer_name_len); + peer_name.len = peer_name_len; if (!err) { peer_name_string = grpc_sockaddr_to_uri(&peer_name); } else { @@ -368,7 +390,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { } gpr_asprintf(&fd_name, "tcp_server:%s", peer_name_string); ep = grpc_tcp_create(grpc_winsocket_create(sock, fd_name), - peer_name_string); + sp->server->resource_quota, peer_name_string); gpr_free(fd_name); gpr_free(peer_name_string); } else { @@ -466,10 +488,11 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s, as some previously created listener. */ if (grpc_sockaddr_get_port(addr) == 0) { for (sp = s->head; sp; sp = sp->next) { - sockname_temp.len = sizeof(struct sockaddr_storage); + int sockname_temp_len = sizeof(struct sockaddr_storage); if (0 == getsockname(sp->socket->socket, (struct sockaddr *)sockname_temp.addr, - &sockname_temp.len)) { + &sockname_temp_len)) { + sockname_temp.len = sockname_temp_len; *port = grpc_sockaddr_get_port(&sockname_temp); if (*port > 0) { allocated_addr = gpr_malloc(sizeof(grpc_resolved_address)); diff --git a/src/core/lib/iomgr/tcp_windows.c b/src/core/lib/iomgr/tcp_windows.c index a5f508a2c35..46f0491d10a 100644 --- a/src/core/lib/iomgr/tcp_windows.c +++ b/src/core/lib/iomgr/tcp_windows.c @@ -109,14 +109,29 @@ typedef struct grpc_tcp { gpr_slice_buffer *write_slices; gpr_slice_buffer *read_slices; + grpc_resource_user resource_user; + /* The IO Completion Port runs from another thread. We need some mechanism to protect ourselves when requesting a shutdown. */ gpr_mu mu; int shutting_down; + gpr_atm resource_user_shutdown_count; + char *peer_string; } grpc_tcp; +static void win_unref_closure(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, + grpc_error *error); + +static void win_maybe_shutdown_resource_user(grpc_exec_ctx *exec_ctx, + grpc_tcp *tcp) { + if (gpr_atm_full_fetch_add(&tcp->resource_user_shutdown_count, 1) == 0) { + grpc_resource_user_shutdown(exec_ctx, &tcp->resource_user, + grpc_closure_create(win_unref_closure, tcp)); + } +} + static void tcp_free(grpc_tcp *tcp) { grpc_winsocket_destroy(tcp->socket); gpr_mu_destroy(&tcp->mu); @@ -155,6 +170,11 @@ static void tcp_unref(grpc_tcp *tcp) { static void tcp_ref(grpc_tcp *tcp) { gpr_ref(&tcp->refcount); } #endif +static void win_unref_closure(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + TCP_UNREF(arg, "resource_user"); +} + /* Asynchronous callback from the IOCP, or the background thread. */ static void on_read(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) { grpc_tcp *tcp = tcpp; @@ -376,12 +396,14 @@ static void win_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { callback. See the comments in on_read and on_write. */ tcp->shutting_down = 1; grpc_winsocket_shutdown(tcp->socket); + win_maybe_shutdown_resource_user(exec_ctx, tcp); gpr_mu_unlock(&tcp->mu); } static void win_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_network_status_unregister_endpoint(ep); grpc_tcp *tcp = (grpc_tcp *)ep; + win_maybe_shutdown_resource_user(exec_ctx, tcp); TCP_UNREF(tcp, "destroy"); } @@ -392,6 +414,11 @@ static char *win_get_peer(grpc_endpoint *ep) { static grpc_workqueue *win_get_workqueue(grpc_endpoint *ep) { return NULL; } +static grpc_resource_user *win_get_resource_user(grpc_endpoint *ep) { + grpc_tcp *tcp = (grpc_tcp *)ep; + return &tcp->resource_user; +} + static grpc_endpoint_vtable vtable = {win_read, win_write, win_get_workqueue, @@ -399,18 +426,22 @@ static grpc_endpoint_vtable vtable = {win_read, win_add_to_pollset_set, win_shutdown, win_destroy, + win_get_resource_user, win_get_peer}; -grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string) { +grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, + grpc_resource_quota *resource_quota, + char *peer_string) { grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); memset(tcp, 0, sizeof(grpc_tcp)); tcp->base.vtable = &vtable; tcp->socket = socket; gpr_mu_init(&tcp->mu); - gpr_ref_init(&tcp->refcount, 1); + gpr_ref_init(&tcp->refcount, 2); grpc_closure_init(&tcp->on_read, on_read, tcp); grpc_closure_init(&tcp->on_write, on_write, tcp); tcp->peer_string = gpr_strdup(peer_string); + grpc_resource_user_init(&tcp->resource_user, resource_quota, peer_string); /* Tell network status tracking code about the new endpoint */ grpc_network_status_register_endpoint(&tcp->base); diff --git a/src/core/lib/iomgr/tcp_windows.h b/src/core/lib/iomgr/tcp_windows.h index 86d777235ef..4402de1c385 100644 --- a/src/core/lib/iomgr/tcp_windows.h +++ b/src/core/lib/iomgr/tcp_windows.h @@ -50,7 +50,9 @@ /* Create a tcp endpoint given a winsock handle. * Takes ownership of the handle. */ -grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string); +grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, + grpc_resource_quota *resource_quota, + char *peer_string); grpc_error *grpc_tcp_prepare_socket(SOCKET sock); diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c index 312a3d4f90b..cb5ba554b07 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.c +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c @@ -124,11 +124,14 @@ static int is_stack_running_on_compute_engine(void) { grpc_httpcli_context_init(&context); + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("google_default_credentials"); grpc_httpcli_get( - &exec_ctx, &context, &detector.pollent, &request, + &exec_ctx, &context, &detector.pollent, resource_quota, &request, gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), max_detection_delay), grpc_closure_create(on_compute_engine_detection_http_response, &detector), &detector.response); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); grpc_exec_ctx_flush(&exec_ctx); diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c index 73eb2e32580..43eb6425153 100644 --- a/src/core/lib/security/credentials/jwt/jwt_verifier.c +++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c @@ -657,11 +657,17 @@ static void on_openid_config_retrieved(grpc_exec_ctx *exec_ctx, void *user_data, *(req.host + (req.http.path - jwks_uri)) = '\0'; } + /* TODO(ctiller): Carry the resource_quota in ctx and share it with the host + channel. This would allow us to cancel an authentication query when under + extreme memory pressure. */ + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("jwt_verifier"); grpc_httpcli_get( - exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, &req, + exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, resource_quota, &req, gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), grpc_jwt_verifier_max_delay), grpc_closure_create(on_keys_retrieved, ctx), &ctx->responses[HTTP_RESPONSE_KEYS]); + grpc_resource_quota_internal_unref(exec_ctx, resource_quota); grpc_json_destroy(json); gpr_free(req.host); return; @@ -764,10 +770,16 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx, rsp_idx = HTTP_RESPONSE_OPENID; } + /* TODO(ctiller): Carry the resource_quota in ctx and share it with the host + channel. This would allow us to cancel an authentication query when under + extreme memory pressure. */ + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("jwt_verifier"); grpc_httpcli_get( - exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, &req, + exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, resource_quota, &req, gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), grpc_jwt_verifier_max_delay), http_cb, &ctx->responses[rsp_idx]); + grpc_resource_quota_internal_unref(exec_ctx, resource_quota); gpr_free(req.host); gpr_free(req.http.path); return; diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c index c22ea5c468b..d980577c46a 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c @@ -307,9 +307,15 @@ static void compute_engine_fetch_oauth2( request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH; request.http.hdr_count = 1; request.http.hdrs = &header; - grpc_httpcli_get(exec_ctx, httpcli_context, pollent, &request, deadline, - grpc_closure_create(response_cb, metadata_req), + /* TODO(ctiller): Carry the resource_quota in ctx and share it with the host + channel. This would allow us to cancel an authentication query when under + extreme memory pressure. */ + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("oauth2_credentials"); + grpc_httpcli_get(exec_ctx, httpcli_context, pollent, resource_quota, &request, + deadline, grpc_closure_create(response_cb, metadata_req), &metadata_req->response); + grpc_resource_quota_internal_unref(exec_ctx, resource_quota); } grpc_call_credentials *grpc_google_compute_engine_credentials_create( @@ -357,10 +363,16 @@ static void refresh_token_fetch_oauth2( request.http.hdr_count = 1; request.http.hdrs = &header; request.handshaker = &grpc_httpcli_ssl; - grpc_httpcli_post(exec_ctx, httpcli_context, pollent, &request, body, - strlen(body), deadline, + /* TODO(ctiller): Carry the resource_quota in ctx and share it with the host + channel. This would allow us to cancel an authentication query when under + extreme memory pressure. */ + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("oauth2_credentials_refresh"); + grpc_httpcli_post(exec_ctx, httpcli_context, pollent, resource_quota, + &request, body, strlen(body), deadline, grpc_closure_create(response_cb, metadata_req), &metadata_req->response); + grpc_resource_quota_internal_unref(exec_ctx, resource_quota); gpr_free(body); } diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c index acb0113ea8c..3924997d31f 100644 --- a/src/core/lib/security/transport/secure_endpoint.c +++ b/src/core/lib/security/transport/secure_endpoint.c @@ -370,6 +370,12 @@ static grpc_workqueue *endpoint_get_workqueue(grpc_endpoint *secure_ep) { return grpc_endpoint_get_workqueue(ep->wrapped_ep); } +static grpc_resource_user *endpoint_get_resource_user( + grpc_endpoint *secure_ep) { + secure_endpoint *ep = (secure_endpoint *)secure_ep; + return grpc_endpoint_get_resource_user(ep->wrapped_ep); +} + static const grpc_endpoint_vtable vtable = {endpoint_read, endpoint_write, endpoint_get_workqueue, @@ -377,6 +383,7 @@ static const grpc_endpoint_vtable vtable = {endpoint_read, endpoint_add_to_pollset_set, endpoint_shutdown, endpoint_destroy, + endpoint_get_resource_user, endpoint_get_peer}; grpc_endpoint *grpc_secure_endpoint_create( diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 02a5de0857c..6c25952c0a9 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -1516,8 +1516,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, call, STATUS_FROM_API_OVERRIDE, GRPC_MDSTR_REF(call->send_extra_metadata[1].md->value)); } - set_status_code(call, STATUS_FROM_API_OVERRIDE, - (uint32_t)op->data.send_status_from_server.status); + if (op->data.send_status_from_server.status != GRPC_STATUS_OK) { + set_status_code(call, STATUS_FROM_API_OVERRIDE, + (uint32_t)op->data.send_status_from_server.status); + } if (!prepare_application_metadata( call, (int)op->data.send_status_from_server.trailing_metadata_count, diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c index 8ca0643ba7e..7903f57a68c 100644 --- a/src/core/lib/surface/init.c +++ b/src/core/lib/surface/init.c @@ -52,6 +52,7 @@ #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/iomgr/iomgr.h" +#include "src/core/lib/iomgr/resource_quota.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/call.h" @@ -191,6 +192,7 @@ void grpc_init(void) { // Default timeout trace to 1 grpc_cq_event_timeout_trace = 1; grpc_register_tracer("op_failure", &grpc_trace_operation_failures); + grpc_register_tracer("resource_quota", &grpc_resource_quota_trace); #ifndef NDEBUG grpc_register_tracer("pending_tags", &grpc_trace_pending_tags); #endif diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc index f297ae85870..d136d49c89f 100644 --- a/src/cpp/common/channel_arguments.cc +++ b/src/cpp/common/channel_arguments.cc @@ -34,6 +34,7 @@ #include +#include #include #include #include "src/core/lib/channel/channel_args.h" @@ -113,6 +114,13 @@ void ChannelArguments::SetUserAgentPrefix( } } +void ChannelArguments::SetResourceQuota( + const grpc::ResourceQuota& resource_quota) { + SetPointerWithVtable(GRPC_ARG_RESOURCE_QUOTA, + resource_quota.c_resource_quota(), + grpc_resource_quota_arg_vtable()); +} + void ChannelArguments::SetInt(const grpc::string& key, int value) { grpc_arg arg; arg.type = GRPC_ARG_INTEGER; @@ -127,12 +135,18 @@ void ChannelArguments::SetPointer(const grpc::string& key, void* value) { static const grpc_arg_pointer_vtable vtable = { &PointerVtableMembers::Copy, &PointerVtableMembers::Destroy, &PointerVtableMembers::Compare}; + SetPointerWithVtable(key, value, &vtable); +} + +void ChannelArguments::SetPointerWithVtable( + const grpc::string& key, void* value, + const grpc_arg_pointer_vtable* vtable) { grpc_arg arg; arg.type = GRPC_ARG_POINTER; strings_.push_back(key); arg.key = const_cast(strings_.back().c_str()); arg.value.pointer.p = value; - arg.value.pointer.vtable = &vtable; + arg.value.pointer.vtable = vtable; args_.push_back(arg); } diff --git a/src/cpp/common/resource_quota_cc.cc b/src/cpp/common/resource_quota_cc.cc new file mode 100644 index 00000000000..335896ab917 --- /dev/null +++ b/src/cpp/common/resource_quota_cc.cc @@ -0,0 +1,51 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include + +namespace grpc { + +ResourceQuota::ResourceQuota() : impl_(grpc_resource_quota_create(nullptr)) {} + +ResourceQuota::ResourceQuota(const grpc::string& name) + : impl_(grpc_resource_quota_create(name.c_str())) {} + +ResourceQuota::~ResourceQuota() { grpc_resource_quota_unref(impl_); } + +ResourceQuota& ResourceQuota::Resize(size_t new_size) { + grpc_resource_quota_resize(impl_, new_size); + return *this; +} + +} // namespace grpc diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc index 8ca29ee58c3..00a90bb184c 100644 --- a/src/cpp/server/server_builder.cc +++ b/src/cpp/server/server_builder.cc @@ -34,6 +34,7 @@ #include #include +#include #include #include #include @@ -56,6 +57,7 @@ ServerBuilder::ServerBuilder() : max_receive_message_size_(-1), max_send_message_size_(-1), sync_server_settings_(SyncServerSettings()), + resource_quota_(nullptr), generic_service_(nullptr) { gpr_once_init(&once_init_plugin_list, do_plugin_list_init); for (auto it = g_plugin_factory_list->begin(); @@ -73,6 +75,12 @@ ServerBuilder::ServerBuilder() sizeof(maybe_default_compression_algorithm_)); } +ServerBuilder::~ServerBuilder() { + if (resource_quota_ != nullptr) { + grpc_resource_quota_unref(resource_quota_); + } +} + std::unique_ptr ServerBuilder::AddCompletionQueue( bool is_frequently_polled) { ServerCompletionQueue* cq = new ServerCompletionQueue(is_frequently_polled); @@ -152,6 +160,16 @@ ServerBuilder& ServerBuilder::SetDefaultCompressionAlgorithm( return *this; } +ServerBuilder& ServerBuilder::SetResourceQuota( + const grpc::ResourceQuota& resource_quota) { + if (resource_quota_ != nullptr) { + grpc_resource_quota_unref(resource_quota_); + } + resource_quota_ = resource_quota.c_resource_quota(); + grpc_resource_quota_ref(resource_quota_); + return *this; +} + ServerBuilder& ServerBuilder::AddListeningPort( const grpc::string& addr, std::shared_ptr creds, int* selected_port) { @@ -190,6 +208,11 @@ std::unique_ptr ServerBuilder::BuildAndStart() { maybe_default_compression_algorithm_.algorithm); } + if (resource_quota_ != nullptr) { + args.SetPointerWithVtable(GRPC_ARG_RESOURCE_QUOTA, resource_quota_, + grpc_resource_quota_arg_vtable()); + } + // == Determine if the server has any syncrhonous methods == bool has_sync_methods = false; for (auto it = services_.begin(); it != services_.end(); ++it) { diff --git a/src/csharp/Grpc.Core/Internal/AsyncCall.cs b/src/csharp/Grpc.Core/Internal/AsyncCall.cs index 49d0a111efa..5e61e9ec129 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCall.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCall.cs @@ -52,9 +52,8 @@ namespace Grpc.Core.Internal // Completion of a pending unary response if not null. TaskCompletionSource unaryResponseTcs; - // TODO(jtattermusch): this field doesn't need to be initialized for unary response calls. - // Indicates that response streaming call has finished. - TaskCompletionSource streamingCallFinishedTcs = new TaskCompletionSource(); + // Completion of a streaming response call if not null. + TaskCompletionSource streamingResponseCallFinishedTcs; // TODO(jtattermusch): this field could be lazy-initialized (only if someone requests the response headers). // Response headers set here once received. @@ -198,6 +197,7 @@ namespace Grpc.Core.Internal byte[] payload = UnsafeSerialize(msg); + streamingResponseCallFinishedTcs = new TaskCompletionSource(); using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) { call.StartServerStreaming(HandleFinished, payload, metadataArray, GetWriteFlagsForCall()); @@ -219,6 +219,7 @@ namespace Grpc.Core.Internal Initialize(details.Channel.CompletionQueue); + streamingResponseCallFinishedTcs = new TaskCompletionSource(); using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) { call.StartDuplexStreaming(HandleFinished, metadataArray); @@ -276,13 +277,13 @@ namespace Grpc.Core.Internal } /// - /// Get the task that completes once if streaming call finishes with ok status and throws RpcException with given status otherwise. + /// Get the task that completes once if streaming response call finishes with ok status and throws RpcException with given status otherwise. /// - public Task StreamingCallFinishedTask + public Task StreamingResponseCallFinishedTask { get { - return streamingCallFinishedTcs.Task; + return streamingResponseCallFinishedTcs.Task; } } @@ -529,11 +530,11 @@ namespace Grpc.Core.Internal var status = receivedStatus.Status; if (status.StatusCode != StatusCode.OK) { - streamingCallFinishedTcs.SetException(new RpcException(status)); + streamingResponseCallFinishedTcs.SetException(new RpcException(status)); return; } - streamingCallFinishedTcs.SetResult(null); + streamingResponseCallFinishedTcs.SetResult(null); } } } diff --git a/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs b/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs index ad9423ff58c..65bf60269a5 100644 --- a/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs +++ b/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs @@ -73,7 +73,7 @@ namespace Grpc.Core.Internal if (result == null) { - await call.StreamingCallFinishedTask.ConfigureAwait(false); + await call.StreamingResponseCallFinishedTask.ConfigureAwait(false); return false; } return true; diff --git a/src/node/ext/byte_buffer.cc b/src/node/ext/byte_buffer.cc index a3f678f32c6..ad7d0ec8c89 100644 --- a/src/node/ext/byte_buffer.cc +++ b/src/node/ext/byte_buffer.cc @@ -44,8 +44,8 @@ namespace grpc { namespace node { +using Nan::MaybeLocal; -using v8::Context; using v8::Function; using v8::Local; using v8::Object; @@ -89,15 +89,19 @@ Local ByteBufferToBuffer(grpc_byte_buffer *buffer) { Local MakeFastBuffer(Local slowBuffer) { Nan::EscapableHandleScope scope; Local globalObj = Nan::GetCurrentContext()->Global(); + MaybeLocal constructorValue = Nan::Get( + globalObj, Nan::New("Buffer").ToLocalChecked()); Local bufferConstructor = Local::Cast( - globalObj->Get(Nan::New("Buffer").ToLocalChecked())); - Local consArgs[3] = { + constructorValue.ToLocalChecked()); + const int argc = 3; + Local consArgs[argc] = { slowBuffer, Nan::New(::node::Buffer::Length(slowBuffer)), Nan::New(0) }; - Local fastBuffer = bufferConstructor->NewInstance(3, consArgs); - return scope.Escape(fastBuffer); + MaybeLocal fastBuffer = Nan::NewInstance(bufferConstructor, + argc, consArgs); + return scope.Escape(fastBuffer.ToLocalChecked()); } } // namespace node } // namespace grpc diff --git a/src/node/ext/call.cc b/src/node/ext/call.cc index b48a7bd6987..191e763e0e7 100644 --- a/src/node/ext/call.cc +++ b/src/node/ext/call.cc @@ -669,16 +669,16 @@ NAN_METHOD(Call::New) { return Nan::ThrowTypeError("Call's fourth argument must be a string"); } call = new Call(wrapped_call); - info.This()->SetHiddenValue(Nan::New("channel_").ToLocalChecked(), - channel_object); + Nan::Set(info.This(), Nan::New("channel_").ToLocalChecked(), + channel_object); } call->Wrap(info.This()); info.GetReturnValue().Set(info.This()); } else { const int argc = 4; Local argv[argc] = {info[0], info[1], info[2], info[3]}; - MaybeLocal maybe_instance = constructor->GetFunction()->NewInstance( - argc, argv); + MaybeLocal maybe_instance = Nan::NewInstance( + constructor->GetFunction(), argc, argv); if (maybe_instance.IsEmpty()) { // There's probably a pending exception return; diff --git a/src/node/ext/channel.cc b/src/node/ext/channel.cc index c4028170e73..5bc58b9b324 100644 --- a/src/node/ext/channel.cc +++ b/src/node/ext/channel.cc @@ -208,8 +208,8 @@ NAN_METHOD(Channel::New) { } else { const int argc = 3; Local argv[argc] = {info[0], info[1], info[2]}; - MaybeLocal maybe_instance = constructor->GetFunction()->NewInstance( - argc, argv); + MaybeLocal maybe_instance = Nan::NewInstance( + constructor->GetFunction(), argc, argv); if (maybe_instance.IsEmpty()) { // There's probably a pending exception return; diff --git a/src/node/ext/server.cc b/src/node/ext/server.cc index 29f31ff15ee..70d5b96f397 100644 --- a/src/node/ext/server.cc +++ b/src/node/ext/server.cc @@ -222,7 +222,7 @@ NAN_METHOD(Server::New) { const int argc = 1; Local argv[argc] = {info[0]}; MaybeLocal maybe_instance = - constructor->GetFunction()->NewInstance(argc, argv); + Nan::NewInstance(constructor->GetFunction(), argc, argv); if (maybe_instance.IsEmpty()) { // There's probably a pending exception return; diff --git a/src/node/src/common.js b/src/node/src/common.js index 22159dd39f7..c6c6d597a88 100644 --- a/src/node/src/common.js +++ b/src/node/src/common.js @@ -141,7 +141,7 @@ exports.getProtobufServiceAttrs = function getProtobufServiceAttrs(service, binaryAsBase64 = options.binaryAsBase64; longsAsStrings = options.longsAsStrings; } - return _.object(_.map(service.children, function(method) { + return _.fromPairs(_.map(service.children, function(method) { return [_.camelCase(method.name), { path: prefix + method.name, requestStream: method.requestStream, diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 0c3c3216abe..6e594fd3edc 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.0.0' + v = '1.0.1' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC @@ -95,7 +95,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.0.0' + s.dependency '!ProtoCompiler', '3.0.2' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.1' s.osx.deployment_target = '10.9' diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index 5018dedc066..b55f6c93c62 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.0.0' + v = '3.0.2' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC diff --git a/src/objective-c/BoringSSL.podspec b/src/objective-c/BoringSSL.podspec index e14f39b898c..47b5b1a2e7b 100644 --- a/src/objective-c/BoringSSL.podspec +++ b/src/objective-c/BoringSSL.podspec @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.name = 'BoringSSL' - version = '6.0' + version = '7.0' s.version = version s.summary = 'BoringSSL is a fork of OpenSSL that is designed to meet Google’s needs.' # Adapted from the homepage: @@ -70,7 +70,7 @@ Pod::Spec.new do |s| s.source = { :git => 'https://boringssl.googlesource.com/boringssl', :tag => "version_for_cocoapods_#{version}", - # :commit => '4ac2dc4c0d48ca45da4f66c40e60d6b425fa94a3', + # :commit => '4fec04b48406111cb88fdd8d196253adc54f7a31', } name = 'openssl' @@ -388,42 +388,42 @@ Pod::Spec.new do |s| 0x28340c19, 0x283480ac, 0x283500ea, - 0x2c322843, - 0x2c32a851, - 0x2c332863, - 0x2c33a875, - 0x2c342889, - 0x2c34a89b, - 0x2c3528b6, - 0x2c35a8c8, - 0x2c3628db, + 0x2c322910, + 0x2c32a91e, + 0x2c332930, + 0x2c33a942, + 0x2c342956, + 0x2c34a968, + 0x2c352983, + 0x2c35a995, + 0x2c3629a8, 0x2c36832d, - 0x2c3728e8, - 0x2c37a8fa, - 0x2c38290d, - 0x2c38a924, - 0x2c392932, - 0x2c39a942, - 0x2c3a2954, - 0x2c3aa968, - 0x2c3b2979, - 0x2c3ba998, - 0x2c3c29ac, - 0x2c3ca9c2, - 0x2c3d29db, - 0x2c3da9f8, - 0x2c3e2a09, - 0x2c3eaa17, - 0x2c3f2a2f, - 0x2c3faa47, - 0x2c402a54, + 0x2c3729b5, + 0x2c37a9c7, + 0x2c3829da, + 0x2c38a9f1, + 0x2c3929ff, + 0x2c39aa0f, + 0x2c3a2a21, + 0x2c3aaa35, + 0x2c3b2a46, + 0x2c3baa65, + 0x2c3c2a79, + 0x2c3caa8f, + 0x2c3d2aa8, + 0x2c3daac5, + 0x2c3e2ad6, + 0x2c3eaae4, + 0x2c3f2afc, + 0x2c3fab14, + 0x2c402b21, 0x2c4090e7, - 0x2c412a65, - 0x2c41aa78, + 0x2c412b32, + 0x2c41ab45, 0x2c4210c0, - 0x2c42aa89, + 0x2c42ab56, 0x2c430720, - 0x2c43a98a, + 0x2c43aa57, 0x30320000, 0x30328015, 0x3033001f, @@ -576,174 +576,183 @@ Pod::Spec.new do |s| 0x403b9861, 0x403c0064, 0x403c8083, - 0x403d1890, - 0x403d98a6, - 0x403e18b5, - 0x403e98c8, - 0x403f18e2, - 0x403f98f0, - 0x40401905, - 0x40409919, - 0x40411936, - 0x40419951, - 0x4042196a, - 0x4042997d, - 0x40431991, - 0x404399a9, - 0x404419c0, + 0x403d18aa, + 0x403d98c0, + 0x403e18cf, + 0x403e98e2, + 0x403f18fc, + 0x403f990a, + 0x4040191f, + 0x40409933, + 0x40411950, + 0x4041996b, + 0x40421984, + 0x40429997, + 0x404319ab, + 0x404399c3, + 0x404419da, 0x404480ac, - 0x404519d5, - 0x404599e7, - 0x40461a0b, - 0x40469a2b, - 0x40471a39, - 0x40479a60, - 0x40481a89, - 0x40489aa2, - 0x40491ab9, - 0x40499ad3, - 0x404a1aea, - 0x404a9b08, - 0x404b1b20, - 0x404b9b37, - 0x404c1b4d, - 0x404c9b5f, - 0x404d1b80, - 0x404d9ba2, - 0x404e1bb6, - 0x404e9bc3, - 0x404f1bf0, - 0x404f9c19, - 0x40501c43, - 0x40509c57, - 0x40511c72, - 0x40519c82, - 0x40521c99, - 0x40529cbd, - 0x40531cd5, - 0x40539ce8, - 0x40541cfd, - 0x40549d20, - 0x40551d2e, - 0x40559d4b, - 0x40561d58, - 0x40569d71, - 0x40571d89, - 0x40579d9c, - 0x40581db1, - 0x40589dc3, - 0x40591df2, - 0x40599e0b, - 0x405a1e1f, - 0x405a9e2f, - 0x405b1e47, - 0x405b9e58, - 0x405c1e6b, - 0x405c9e7c, - 0x405d1e89, - 0x405d9ea0, - 0x405e1ec0, + 0x404519ef, + 0x40459a01, + 0x40461a25, + 0x40469a45, + 0x40471a53, + 0x40479a7a, + 0x40481ab7, + 0x40489ad0, + 0x40491ae7, + 0x40499b01, + 0x404a1b18, + 0x404a9b36, + 0x404b1b4e, + 0x404b9b65, + 0x404c1b7b, + 0x404c9b8d, + 0x404d1bae, + 0x404d9bd0, + 0x404e1be4, + 0x404e9bf1, + 0x404f1c1e, + 0x404f9c47, + 0x40501c71, + 0x40509c85, + 0x40511ca0, + 0x40519cb0, + 0x40521cc7, + 0x40529ceb, + 0x40531d03, + 0x40539d16, + 0x40541d2b, + 0x40549d4e, + 0x40551d5c, + 0x40559d79, + 0x40561d86, + 0x40569d9f, + 0x40571db7, + 0x40579dca, + 0x40581ddf, + 0x40589e06, + 0x40591e35, + 0x40599e62, + 0x405a1e76, + 0x405a9e86, + 0x405b1e9e, + 0x405b9eaf, + 0x405c1ec2, + 0x405c9ee3, + 0x405d1ef0, + 0x405d9f07, + 0x405e1f27, 0x405e8a95, - 0x405f1ee1, - 0x405f9eee, - 0x40601efc, - 0x40609f1e, - 0x40611f46, - 0x40619f5b, - 0x40621f72, - 0x40629f83, - 0x40631f94, - 0x40639fa9, - 0x40641fc0, - 0x40649fd1, - 0x40651fec, - 0x4065a003, - 0x4066201b, - 0x4066a045, - 0x40672070, - 0x4067a091, - 0x406820a4, - 0x4068a0c5, - 0x406920f7, - 0x4069a125, - 0x406a2146, - 0x406aa166, - 0x406b22ee, - 0x406ba311, - 0x406c2327, - 0x406ca553, - 0x406d2582, - 0x406da5aa, - 0x406e25c3, - 0x406ea5db, - 0x406f25fa, - 0x406fa60f, - 0x40702622, - 0x4070a63f, + 0x405f1f48, + 0x405f9f55, + 0x40601f63, + 0x40609f85, + 0x40611fad, + 0x40619fc2, + 0x40621fd9, + 0x40629fea, + 0x40631ffb, + 0x4063a010, + 0x40642027, + 0x4064a053, + 0x4065206e, + 0x4065a085, + 0x4066209d, + 0x4066a0c7, + 0x406720f2, + 0x4067a113, + 0x40682126, + 0x4068a147, + 0x40692179, + 0x4069a1a7, + 0x406a21c8, + 0x406aa1e8, + 0x406b2370, + 0x406ba393, + 0x406c23a9, + 0x406ca60b, + 0x406d263a, + 0x406da662, + 0x406e2690, + 0x406ea6a8, + 0x406f26c7, + 0x406fa6dc, + 0x407026ef, + 0x4070a70c, 0x40710800, - 0x4071a651, - 0x40722664, - 0x4072a67d, - 0x40732695, + 0x4071a71e, + 0x40722731, + 0x4072a74a, + 0x40732762, 0x4073936d, - 0x407426a9, - 0x4074a6c3, - 0x407526d4, - 0x4075a6e8, - 0x407626f6, + 0x40742776, + 0x4074a790, + 0x407527a1, + 0x4075a7b5, + 0x407627c3, 0x407691aa, - 0x4077271b, - 0x4077a73d, - 0x40782758, - 0x4078a791, - 0x407927a8, - 0x4079a7be, - 0x407a27ca, - 0x407aa7dd, - 0x407b27f2, - 0x407ba804, - 0x407c2819, - 0x407ca822, - 0x407d20e0, - 0x407d9c29, - 0x407e276d, - 0x407e9dd3, - 0x407f1a4d, - 0x407f986d, - 0x40801c00, - 0x40809a75, - 0x40811cab, - 0x40819bda, - 0x41f42219, - 0x41f922ab, - 0x41fe219e, - 0x41fea37a, - 0x41ff246b, - 0x42032232, - 0x42082254, - 0x4208a290, - 0x42092182, - 0x4209a2ca, - 0x420a21d9, - 0x420aa1b9, - 0x420b21f9, - 0x420ba272, - 0x420c2487, - 0x420ca347, - 0x420d2361, - 0x420da398, - 0x421223b2, - 0x4217244e, - 0x4217a3f4, - 0x421c2416, - 0x421f23d1, - 0x4221249e, - 0x42262431, - 0x422b2537, - 0x422ba500, - 0x422c251f, - 0x422ca4da, - 0x422d24b9, + 0x407727e8, + 0x4077a80a, + 0x40782825, + 0x4078a85e, + 0x40792875, + 0x4079a88b, + 0x407a2897, + 0x407aa8aa, + 0x407b28bf, + 0x407ba8d1, + 0x407c28e6, + 0x407ca8ef, + 0x407d2162, + 0x407d9c57, + 0x407e283a, + 0x407e9e16, + 0x407f1a67, + 0x407f9887, + 0x40801c2e, + 0x40809a8f, + 0x40811cd9, + 0x40819c08, + 0x4082267b, + 0x4082986d, + 0x40831df1, + 0x4083a038, + 0x40841aa3, + 0x40849e4e, + 0x40851ed3, + 0x41f4229b, + 0x41f9232d, + 0x41fe2220, + 0x41fea3fc, + 0x41ff24ed, + 0x420322b4, + 0x420822d6, + 0x4208a312, + 0x42092204, + 0x4209a34c, + 0x420a225b, + 0x420aa23b, + 0x420b227b, + 0x420ba2f4, + 0x420c2509, + 0x420ca3c9, + 0x420d23e3, + 0x420da41a, + 0x42122434, + 0x421724d0, + 0x4217a476, + 0x421c2498, + 0x421f2453, + 0x42212520, + 0x422624b3, + 0x422b25ef, + 0x422ba59d, + 0x422c25d7, + 0x422ca55c, + 0x422d253b, + 0x422da5bc, + 0x422e2582, 0x4432072b, 0x4432873a, 0x44330746, @@ -786,69 +795,69 @@ Pod::Spec.new do |s| 0x4c3d136d, 0x4c3d937c, 0x4c3e1389, - 0x50322a9b, - 0x5032aaaa, - 0x50332ab5, - 0x5033aac5, - 0x50342ade, - 0x5034aaf8, - 0x50352b06, - 0x5035ab1c, - 0x50362b2e, - 0x5036ab44, - 0x50372b5d, - 0x5037ab70, - 0x50382b88, - 0x5038ab99, - 0x50392bae, - 0x5039abc2, - 0x503a2be2, - 0x503aabf8, - 0x503b2c10, - 0x503bac22, - 0x503c2c3e, - 0x503cac55, - 0x503d2c6e, - 0x503dac84, - 0x503e2c91, - 0x503eaca7, - 0x503f2cb9, + 0x50322b68, + 0x5032ab77, + 0x50332b82, + 0x5033ab92, + 0x50342bab, + 0x5034abc5, + 0x50352bd3, + 0x5035abe9, + 0x50362bfb, + 0x5036ac11, + 0x50372c2a, + 0x5037ac3d, + 0x50382c55, + 0x5038ac66, + 0x50392c7b, + 0x5039ac8f, + 0x503a2caf, + 0x503aacc5, + 0x503b2cdd, + 0x503bacef, + 0x503c2d0b, + 0x503cad22, + 0x503d2d3b, + 0x503dad51, + 0x503e2d5e, + 0x503ead74, + 0x503f2d86, 0x503f8382, - 0x50402ccc, - 0x5040acdc, - 0x50412cf6, - 0x5041ad05, - 0x50422d1f, - 0x5042ad3c, - 0x50432d4c, - 0x5043ad5c, - 0x50442d6b, + 0x50402d99, + 0x5040ada9, + 0x50412dc3, + 0x5041add2, + 0x50422dec, + 0x5042ae09, + 0x50432e19, + 0x5043ae29, + 0x50442e38, 0x5044843f, - 0x50452d7f, - 0x5045ad9d, - 0x50462db0, - 0x5046adc6, - 0x50472dd8, - 0x5047aded, - 0x50482e13, - 0x5048ae21, - 0x50492e34, - 0x5049ae49, - 0x504a2e5f, - 0x504aae6f, - 0x504b2e8f, - 0x504baea2, - 0x504c2ec5, - 0x504caef3, - 0x504d2f05, - 0x504daf22, - 0x504e2f3d, - 0x504eaf59, - 0x504f2f6b, - 0x504faf82, - 0x50502f91, + 0x50452e4c, + 0x5045ae6a, + 0x50462e7d, + 0x5046ae93, + 0x50472ea5, + 0x5047aeba, + 0x50482ee0, + 0x5048aeee, + 0x50492f01, + 0x5049af16, + 0x504a2f2c, + 0x504aaf3c, + 0x504b2f5c, + 0x504baf6f, + 0x504c2f92, + 0x504cafc0, + 0x504d2fd2, + 0x504dafef, + 0x504e300a, + 0x504eb026, + 0x504f3038, + 0x504fb04f, + 0x5050305e, 0x505086ef, - 0x50512fa4, + 0x50513071, 0x58320ec9, 0x68320e8b, 0x68328c25, @@ -1209,6 +1218,7 @@ Pod::Spec.new do |s| "BAD_SSL_FILETYPE\\0" "BAD_WRITE_RETRY\\0" "BIO_NOT_SET\\0" + "BLOCK_CIPHER_PAD_IS_WRONG\\0" "BUFFERED_MESSAGES_ON_CIPHER_CHANGE\\0" "CA_DN_LENGTH_MISMATCH\\0" "CA_DN_TOO_LONG\\0" @@ -1233,6 +1243,7 @@ Pod::Spec.new do |s| "DOWNGRADE_DETECTED\\0" "DTLS_MESSAGE_TOO_BIG\\0" "DUPLICATE_EXTENSION\\0" + "DUPLICATE_KEY_SHARE\\0" "ECC_CERT_NOT_FOR_SIGNING\\0" "EMS_STATE_INCONSISTENT\\0" "ENCRYPTED_LENGTH_TOO_LONG\\0" @@ -1270,15 +1281,18 @@ Pod::Spec.new do |s| "NO_CERTIFICATE_SET\\0" "NO_CIPHERS_AVAILABLE\\0" "NO_CIPHERS_PASSED\\0" + "NO_CIPHERS_SPECIFIED\\0" "NO_CIPHER_MATCH\\0" "NO_COMMON_SIGNATURE_ALGORITHMS\\0" "NO_COMPRESSION_SPECIFIED\\0" + "NO_GROUPS_SPECIFIED\\0" "NO_METHOD_SPECIFIED\\0" "NO_P256_SUPPORT\\0" "NO_PRIVATE_KEY_ASSIGNED\\0" "NO_RENEGOTIATION\\0" "NO_REQUIRED_DIGEST\\0" "NO_SHARED_CIPHER\\0" + "NO_SHARED_GROUP\\0" "NULL_SSL_CTX\\0" "NULL_SSL_METHOD_PASSED\\0" "OLD_SESSION_CIPHER_NOT_RETURNED\\0" @@ -1294,6 +1308,7 @@ Pod::Spec.new do |s| "READ_TIMEOUT_EXPIRED\\0" "RECORD_LENGTH_MISMATCH\\0" "RECORD_TOO_LARGE\\0" + "RENEGOTIATION_EMS_MISMATCH\\0" "RENEGOTIATION_ENCODING_ERR\\0" "RENEGOTIATION_MISMATCH\\0" "REQUIRED_CIPHER_MISSING\\0" @@ -1338,12 +1353,15 @@ Pod::Spec.new do |s| "TLSV1_ALERT_USER_CANCELLED\\0" "TLSV1_BAD_CERTIFICATE_HASH_VALUE\\0" "TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE\\0" + "TLSV1_CERTIFICATE_REQUIRED\\0" "TLSV1_CERTIFICATE_UNOBTAINABLE\\0" + "TLSV1_UNKNOWN_PSK_IDENTITY\\0" "TLSV1_UNRECOGNIZED_NAME\\0" "TLSV1_UNSUPPORTED_EXTENSION\\0" "TLS_PEER_DID_NOT_RESPOND_WITH_CERTIFICATE_LIST\\0" "TLS_RSA_ENCRYPTED_VALUE_LENGTH_IS_WRONG\\0" "TOO_MANY_EMPTY_FRAGMENTS\\0" + "TOO_MANY_KEY_UPDATES\\0" "TOO_MANY_WARNING_ALERTS\\0" "UNABLE_TO_FIND_ECDH_PARAMETERS\\0" "UNEXPECTED_EXTENSION\\0" diff --git a/src/objective-c/CronetFramework.podspec b/src/objective-c/CronetFramework.podspec index 3ebcacf0554..2f47b02c0c0 100644 --- a/src/objective-c/CronetFramework.podspec +++ b/src/objective-c/CronetFramework.podspec @@ -30,14 +30,47 @@ Pod::Spec.new do |s| s.name = "CronetFramework" - s.version = "0.0.2" + s.version = "0.0.3" s.summary = "Cronet, precompiled and used as a framework." s.homepage = "http://chromium.org" - s.license = { :type => 'BSD' } + s.license = { + :type => 'BSD', + :text => <<-LICENSE + Copyright 2015, Google Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + LICENSE + } s.vendored_framework = "Cronet.framework" s.author = "The Chromium Authors" - s.ios.deployment_target = "7.1" + s.ios.deployment_target = "8.0" s.source = { :http => 'https://storage.googleapis.com/grpc-precompiled-binaries/cronet/Cronet.framework.zip' } s.preserve_paths = "Cronet.framework" s.public_header_files = "Cronet.framework/Headers/**/*{.h}" + s.source_files = "Cronet.framework/Headers/**/*{.h}" end diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index 0524472f530..dd0479083de 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -51,7 +51,7 @@ NS_ASSUME_NONNULL_BEGIN // TODO(jcanizales): Generate the version in a standalone header, from // templates. Like // templates/src/core/surface/version.c.template . -#define GRPC_OBJC_VERSION_STRING @"1.0.0" +#define GRPC_OBJC_VERSION_STRING @"1.0.1" static NSMutableDictionary *kHostCache; diff --git a/src/php/lib/Grpc/BaseStub.php b/src/php/lib/Grpc/BaseStub.php index 8a7f6572a62..36d94cae2c9 100644 --- a/src/php/lib/Grpc/BaseStub.php +++ b/src/php/lib/Grpc/BaseStub.php @@ -116,7 +116,7 @@ class BaseStub } /** - * @param $timeout in microseconds + * @param int $timeout in microseconds * * @return bool true if channel is ready * @throw Exception if channel is in FATAL_ERROR state @@ -189,7 +189,7 @@ class BaseStub /** * validate and normalize the metadata array. * - * @param $metadata The metadata map + * @param array $metadata The metadata map * * @return $metadata Validated and key-normalized metadata map * @throw InvalidArgumentException if key contains invalid characters @@ -216,8 +216,8 @@ class BaseStub * Call a remote method that takes a single argument and has a * single output. * - * @param string $method The name of the method to call - * @param $argument The argument to the method + * @param string $method The name of the method to call + * @param mixed $argument The argument to the method * @param callable $deserialize A function that deserializes the response * @param array $metadata A metadata map to send to the server * @@ -250,8 +250,8 @@ class BaseStub * Call a remote method that takes a stream of arguments and has a single * output. * - * @param string $method The name of the method to call - * @param $arguments An array or Traversable of arguments to stream to the + * @param string $method The name of the method to call + * @param array $arguments An array or Traversable of arguments to stream to the * server * @param callable $deserialize A function that deserializes the response * @param array $metadata A metadata map to send to the server @@ -284,8 +284,8 @@ class BaseStub * Call a remote method that takes a single argument and returns a stream of * responses. * - * @param string $method The name of the method to call - * @param $argument The argument to the method + * @param string $method The name of the method to call + * @param mixed $argument The argument to the method * @param callable $deserialize A function that deserializes the responses * @param array $metadata A metadata map to send to the server * diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto index ece69108158..918f5fa3e3a 100644 --- a/src/proto/grpc/testing/control.proto +++ b/src/proto/grpc/testing/control.proto @@ -137,6 +137,11 @@ message ServerConfig { // If we use an OTHER_SERVER client_type, this string gives more detail string other_server_api = 11; + + // c++-only options (for now) -------------------------------- + + // Buffer pool size (no buffer pool specified if unset) + int32 resource_quota_size = 1001; } message ServerArgs { @@ -213,6 +218,10 @@ message ScenarioResultSummary double latency_95 = 9; double latency_99 = 10; double latency_999 = 11; + + // Number of requests that succeeded/failed + double successful_requests_per_second = 12; + double failed_requests_per_second = 13; } // Results of a single benchmark scenario. @@ -232,4 +241,6 @@ message ScenarioResult { // Information on success or failure of each worker repeated bool client_success = 7; repeated bool server_success = 8; + // Number of failed requests (one row per status code seen) + repeated RequestResultCount request_results = 9; } diff --git a/src/proto/grpc/testing/proto2/empty2.proto b/src/proto/grpc/testing/proto2/empty2.proto new file mode 100644 index 00000000000..51f0fe28b1a --- /dev/null +++ b/src/proto/grpc/testing/proto2/empty2.proto @@ -0,0 +1,37 @@ + +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package grpc.testing.proto2; + +message EmptyWithExtensions { + extensions 100 to 999; +} diff --git a/src/proto/grpc/testing/proto2/empty2_extensions.proto b/src/proto/grpc/testing/proto2/empty2_extensions.proto new file mode 100644 index 00000000000..0229fe3fbd5 --- /dev/null +++ b/src/proto/grpc/testing/proto2/empty2_extensions.proto @@ -0,0 +1,43 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +import "src/proto/grpc/testing/proto2/empty2.proto"; + +package grpc.testing.proto2; + +// Fill emptiness with music. +extend grpc.testing.proto2.EmptyWithExtensions { + optional int64 Deadmau5 = 124; + optional float Madeon = 125; + optional string AboveAndBeyond = 126; + optional bool Tycho = 127; + optional fixed64 Pendulum = 128; +} diff --git a/src/proto/grpc/testing/stats.proto b/src/proto/grpc/testing/stats.proto index f9d116110b0..d40d7148016 100644 --- a/src/proto/grpc/testing/stats.proto +++ b/src/proto/grpc/testing/stats.proto @@ -59,6 +59,11 @@ message HistogramData { double count = 6; } +message RequestResultCount { + int32 status_code = 1; + int64 count = 2; +} + message ClientStats { // Latency histogram. Data points are in nanoseconds. HistogramData latencies = 1; @@ -67,4 +72,7 @@ message ClientStats { double time_elapsed = 2; double time_user = 3; double time_system = 4; + + // Number of failed requests (one row per status code seen) + repeated RequestResultCount request_results = 5; } diff --git a/src/python/.gitignore b/src/python/.gitignore index f158efa4bf1..7b520579a08 100644 --- a/src/python/.gitignore +++ b/src/python/.gitignore @@ -1 +1,3 @@ gens/ +*_pb2.py +*_pb2_grpc.py diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 7ffe0f558a6..1ad423909f5 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -122,6 +122,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/resolve_address_posix.c', 'src/core/lib/iomgr/resolve_address_uv.c', 'src/core/lib/iomgr/resolve_address_windows.c', + 'src/core/lib/iomgr/resource_quota.c', 'src/core/lib/iomgr/sockaddr_utils.c', 'src/core/lib/iomgr/socket_utils_common_posix.c', 'src/core/lib/iomgr/socket_utils_linux.c', diff --git a/src/python/grpcio_reflection/.gitignore b/src/python/grpcio_reflection/.gitignore new file mode 100644 index 00000000000..c0befdc8ead --- /dev/null +++ b/src/python/grpcio_reflection/.gitignore @@ -0,0 +1,5 @@ +*.proto +*_pb2.py +build/ +grpcio_reflection.egg-info/ +dist/ diff --git a/src/python/grpcio_reflection/grpc/__init__.py b/src/python/grpcio_reflection/grpc/__init__.py new file mode 100644 index 00000000000..70ac5edd483 --- /dev/null +++ b/src/python/grpcio_reflection/grpc/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +__import__('pkg_resources').declare_namespace(__name__) diff --git a/src/python/grpcio_reflection/grpc/reflection/__init__.py b/src/python/grpcio_reflection/grpc/reflection/__init__.py new file mode 100644 index 00000000000..d5ad73a74ab --- /dev/null +++ b/src/python/grpcio_reflection/grpc/reflection/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/python/grpcio_reflection/grpc/reflection/v1alpha/__init__.py b/src/python/grpcio_reflection/grpc/reflection/v1alpha/__init__.py new file mode 100644 index 00000000000..d5ad73a74ab --- /dev/null +++ b/src/python/grpcio_reflection/grpc/reflection/v1alpha/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/python/grpcio_reflection/grpc/reflection/v1alpha/reflection.py b/src/python/grpcio_reflection/grpc/reflection/v1alpha/reflection.py new file mode 100644 index 00000000000..3c399b0d799 --- /dev/null +++ b/src/python/grpcio_reflection/grpc/reflection/v1alpha/reflection.py @@ -0,0 +1,143 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Reference implementation for reflection in gRPC Python.""" + +import threading + +import grpc +from google.protobuf import descriptor_pb2 +from google.protobuf import descriptor_pool + +from grpc.reflection.v1alpha import reflection_pb2 + +_POOL = descriptor_pool.Default() + +def _not_found_error(): + return reflection_pb2.ServerReflectionResponse( + error_response=reflection_pb2.ErrorResponse( + error_code=grpc.StatusCode.NOT_FOUND.value[0], + error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), + ) + ) + +def _file_descriptor_response(descriptor): + proto = descriptor_pb2.FileDescriptorProto() + descriptor.CopyToProto(proto) + serialized_proto = proto.SerializeToString() + return reflection_pb2.ServerReflectionResponse( + file_descriptor_response=reflection_pb2.FileDescriptorResponse( + file_descriptor_proto=(serialized_proto,) + ), + ) + + +class ReflectionServicer(reflection_pb2.ServerReflectionServicer): + """Servicer handling RPCs for service statuses.""" + + def __init__(self, service_names, pool=None): + """Constructor. + + Args: + service_names: Iterable of fully-qualified service names available. + """ + self._service_names = list(service_names) + self._pool = _POOL if pool is None else pool + + def _file_by_filename(self, filename): + try: + descriptor = self._pool.FindFileByName(filename) + except KeyError: + return _not_found_error() + else: + return _file_descriptor_response(descriptor) + + def _file_containing_symbol(self, fully_qualified_name): + try: + descriptor = self._pool.FindFileContainingSymbol(fully_qualified_name) + except KeyError: + return _not_found_error() + else: + return _file_descriptor_response(descriptor) + + def _file_containing_extension(containing_type, extension_number): + # TODO(atash) Python protobuf currently doesn't support querying extensions. + # https://github.com/google/protobuf/issues/2248 + return reflection_pb2.ServerReflectionResponse( + error_response=reflection_pb2.ErrorResponse( + error_code=grpc.StatusCode.UNIMPLEMENTED.value[0], + error_message=grpc.StatusCode.UNIMPLMENTED.value[1].encode(), + ) + ) + + def _extension_numbers_of_type(fully_qualified_name): + # TODO(atash) We're allowed to leave this unsupported according to the + # protocol, but we should still eventually implement it. Hits the same issue + # as `_file_containing_extension`, however. + # https://github.com/google/protobuf/issues/2248 + return reflection_pb2.ServerReflectionResponse( + error_response=reflection_pb2.ErrorResponse( + error_code=grpc.StatusCode.UNIMPLEMENTED.value[0], + error_message=grpc.StatusCode.UNIMPLMENTED.value[1].encode(), + ) + ) + + def _list_services(self): + return reflection_pb2.ServerReflectionResponse( + list_services_response=reflection_pb2.ListServiceResponse( + service=[ + reflection_pb2.ServiceResponse(name=service_name) + for service_name in self._service_names + ] + ) + ) + + def ServerReflectionInfo(self, request_iterator, context): + for request in request_iterator: + if request.HasField('file_by_filename'): + yield self._file_by_filename(request.file_by_filename) + elif request.HasField('file_containing_symbol'): + yield self._file_containing_symbol(request.file_containing_symbol) + elif request.HasField('file_containing_extension'): + yield self._file_containing_extension( + request.file_containing_extension.containing_type, + request.file_containing_extension.extension_number) + elif request.HasField('all_extension_numbers_of_type'): + yield _all_extension_numbers_of_type( + request.all_extension_numbers_of_type) + elif request.HasField('list_services'): + yield self._list_services() + else: + yield reflection_pb2.ServerReflectionResponse( + error_response=reflection_pb2.ErrorResponse( + error_code=grpc.StatusCode.INVALID_ARGUMENT.value[0], + error_message=grpc.StatusCode.INVALID_ARGUMENT.value[1].encode(), + ) + ) + diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py new file mode 100644 index 00000000000..9b3c44c0221 --- /dev/null +++ b/src/python/grpcio_reflection/grpc_version.py @@ -0,0 +1,32 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! + +VERSION='1.1.0.dev0' diff --git a/src/python/grpcio_reflection/reflection_commands.py b/src/python/grpcio_reflection/reflection_commands.py new file mode 100644 index 00000000000..d189aee5774 --- /dev/null +++ b/src/python/grpcio_reflection/reflection_commands.py @@ -0,0 +1,78 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Provides distutils command classes for the GRPC Python setup process.""" + +import os +import shutil + +import setuptools + +ROOT_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) +HEALTH_PROTO = os.path.join(ROOT_DIR, '../../proto/grpc/reflection/v1alpha/reflection.proto') + + +class CopyProtoModules(setuptools.Command): + """Command to copy proto modules from grpc/src/proto.""" + + description = '' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + if os.path.isfile(HEALTH_PROTO): + shutil.copyfile( + HEALTH_PROTO, + os.path.join(ROOT_DIR, 'grpc/reflection/v1alpha/reflection.proto')) + + +class BuildPackageProtos(setuptools.Command): + """Command to generate project *_pb2.py modules from proto files.""" + + description = 'build grpc protobuf modules' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + # due to limitations of the proto generator, we require that only *one* + # directory is provided as an 'include' directory. We assume it's the '' key + # to `self.distribution.package_dir` (and get a key error if it's not + # there). + from grpc.tools import command + command.build_package_protos(self.distribution.package_dir['']) diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py new file mode 100644 index 00000000000..df95af4de16 --- /dev/null +++ b/src/python/grpcio_reflection/setup.py @@ -0,0 +1,73 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Setup module for the GRPC Python package's optional reflection.""" + +import os +import sys + +import setuptools + +# Ensure we're in the proper directory whether or not we're being used by pip. +os.chdir(os.path.dirname(os.path.abspath(__file__))) + +# Break import-style to ensure we can actually find our commands module. +import reflection_commands +import grpc_version + +PACKAGE_DIRECTORIES = { + '': '.', +} + +SETUP_REQUIRES = ( + 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), +) + +INSTALL_REQUIRES = ( + 'protobuf>=3.0.0', + 'grpcio>={version}'.format(version=grpc_version.VERSION), +) + +COMMAND_CLASS = { + # Run preprocess from the repository *before* doing any packaging! + 'preprocess': reflection_commands.CopyProtoModules, + 'build_package_protos': reflection_commands.BuildPackageProtos, +} + +setuptools.setup( + name='grpcio-reflection', + version=grpc_version.VERSION, + license='3-clause BSD', + package_dir=PACKAGE_DIRECTORIES, + packages=setuptools.find_packages('.'), + namespace_packages=['grpc'], + install_requires=INSTALL_REQUIRES, + setup_requires=SETUP_REQUIRES, + cmdclass=COMMAND_CLASS +) diff --git a/src/python/grpcio_tests/tests/reflection/__init__.py b/src/python/grpcio_tests/tests/reflection/__init__.py new file mode 100644 index 00000000000..100a624dc9c --- /dev/null +++ b/src/python/grpcio_tests/tests/reflection/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py new file mode 100644 index 00000000000..87264cf9ba9 --- /dev/null +++ b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py @@ -0,0 +1,185 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Tests of grpc.reflection.v1alpha.reflection.""" + +import unittest + +import grpc +from grpc.framework.foundation import logging_pool +from grpc.reflection.v1alpha import reflection +from grpc.reflection.v1alpha import reflection_pb2 + +from google.protobuf import descriptor_pool +from google.protobuf import descriptor_pb2 + +from src.proto.grpc.testing.proto2 import empty2_extensions_pb2 +from src.proto.grpc.testing import empty_pb2 +from tests.unit.framework.common import test_constants + +_EMPTY_PROTO_FILE_NAME = 'src/proto/grpc/testing/empty.proto' +_EMPTY_PROTO_SYMBOL_NAME = 'grpc.testing.Empty' +_SERVICE_NAMES = ( + 'Angstrom', 'Bohr', 'Curie', 'Dyson', 'Einstein', 'Feynman', 'Galilei') + +def _file_descriptor_to_proto(descriptor): + proto = descriptor_pb2.FileDescriptorProto() + descriptor.CopyToProto(proto) + return proto.SerializeToString() + +class ReflectionServicerTest(unittest.TestCase): + + def setUp(self): + servicer = reflection.ReflectionServicer(service_names=_SERVICE_NAMES) + server_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY) + self._server = grpc.server(server_pool) + port = self._server.add_insecure_port('[::]:0') + reflection_pb2.add_ServerReflectionServicer_to_server(servicer, self._server) + self._server.start() + + channel = grpc.insecure_channel('localhost:%d' % port) + self._stub = reflection_pb2.ServerReflectionStub(channel) + + def testFileByName(self): + requests = ( + reflection_pb2.ServerReflectionRequest( + file_by_filename=_EMPTY_PROTO_FILE_NAME + ), + reflection_pb2.ServerReflectionRequest( + file_by_filename='i-donut-exist' + ), + ) + responses = tuple(self._stub.ServerReflectionInfo(requests)) + expected_responses = ( + reflection_pb2.ServerReflectionResponse( + valid_host='', + file_descriptor_response=reflection_pb2.FileDescriptorResponse( + file_descriptor_proto=( + _file_descriptor_to_proto(empty_pb2.DESCRIPTOR), + ) + ) + ), + reflection_pb2.ServerReflectionResponse( + valid_host='', + error_response=reflection_pb2.ErrorResponse( + error_code=grpc.StatusCode.NOT_FOUND.value[0], + error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), + ) + ), + ) + self.assertEqual(expected_responses, responses) + + def testFileBySymbol(self): + requests = ( + reflection_pb2.ServerReflectionRequest( + file_containing_symbol=_EMPTY_PROTO_SYMBOL_NAME + ), + reflection_pb2.ServerReflectionRequest( + file_containing_symbol='i.donut.exist.co.uk.org.net.me.name.foo' + ), + ) + responses = tuple(self._stub.ServerReflectionInfo(requests)) + expected_responses = ( + reflection_pb2.ServerReflectionResponse( + valid_host='', + file_descriptor_response=reflection_pb2.FileDescriptorResponse( + file_descriptor_proto=( + _file_descriptor_to_proto(empty_pb2.DESCRIPTOR), + ) + ) + ), + reflection_pb2.ServerReflectionResponse( + valid_host='', + error_response=reflection_pb2.ErrorResponse( + error_code=grpc.StatusCode.NOT_FOUND.value[0], + error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), + ) + ), + ) + self.assertEqual(expected_responses, responses) + + @unittest.skip('TODO(atash): implement file-containing-extension reflection ' + '(see https://github.com/google/protobuf/issues/2248)') + def testFileContainingExtension(self): + requests = ( + reflection_pb2.ServerReflectionRequest( + file_containing_extension=reflection_pb2.ExtensionRequest( + containing_type='grpc.testing.proto2.Empty', + extension_number=125, + ), + ), + reflection_pb2.ServerReflectionRequest( + file_containing_extension=reflection_pb2.ExtensionRequest( + containing_type='i.donut.exist.co.uk.org.net.me.name.foo', + extension_number=55, + ), + ), + ) + responses = tuple(self._stub.ServerReflectionInfo(requests)) + expected_responses = ( + reflection_pb2.ServerReflectionResponse( + valid_host='', + file_descriptor_response=reflection_pb2.FileDescriptorResponse( + file_descriptor_proto=( + _file_descriptor_to_proto(empty_extensions_pb2.DESCRIPTOR), + ) + ) + ), + reflection_pb2.ServerReflectionResponse( + valid_host='', + error_response=reflection_pb2.ErrorResponse( + error_code=grpc.StatusCode.NOT_FOUND.value[0], + error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), + ) + ), + ) + self.assertEqual(expected_responses, responses) + + def testListServices(self): + requests = ( + reflection_pb2.ServerReflectionRequest( + list_services='', + ), + ) + responses = tuple(self._stub.ServerReflectionInfo(requests)) + expected_responses = ( + reflection_pb2.ServerReflectionResponse( + valid_host='', + list_services_response=reflection_pb2.ListServiceResponse( + service=tuple( + reflection_pb2.ServiceResponse(name=name) + for name in _SERVICE_NAMES + ) + ) + ), + ) + self.assertEqual(expected_responses, responses) + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index 2071a33e13a..2e85f1f397b 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -4,41 +4,42 @@ "_api_test.ChannelTest", "_auth_test.AccessTokenCallCredentialsTest", "_auth_test.GoogleCallCredentialsTest", - "_beta_features_test.BetaFeaturesTest", - "_beta_features_test.ContextManagementAndLifecycleTest", + "_beta_features_test.BetaFeaturesTest", + "_beta_features_test.ContextManagementAndLifecycleTest", "_cancel_many_calls_test.CancelManyCallsTest", "_channel_args_test.ChannelArgsTest", "_channel_connectivity_test.ChannelConnectivityTest", "_channel_ready_future_test.ChannelReadyFutureTest", - "_channel_test.ChannelTest", + "_channel_test.ChannelTest", "_compression_test.CompressionTest", "_connectivity_channel_test.ConnectivityStatesTest", "_credentials_test.CredentialsTest", "_empty_message_test.EmptyMessageTest", "_exit_test.ExitTest", - "_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", - "_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", - "_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", - "_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", - "_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", + "_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", + "_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", + "_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", + "_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", + "_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", "_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", "_health_servicer_test.HealthServicerTest", "_implementations_test.CallCredentialsTest", - "_implementations_test.ChannelCredentialsTest", - "_insecure_interop_test.InsecureInteropTest", - "_logging_pool_test.LoggingPoolTest", + "_implementations_test.ChannelCredentialsTest", + "_insecure_interop_test.InsecureInteropTest", + "_logging_pool_test.LoggingPoolTest", "_metadata_code_details_test.MetadataCodeDetailsTest", "_metadata_test.MetadataTest", - "_not_found_test.NotFoundTest", + "_not_found_test.NotFoundTest", "_python_plugin_test.PythonPluginTest", "_read_some_but_not_all_responses_test.ReadSomeButNotAllResponsesTest", + "_reflection_servicer_test.ReflectionServicerTest", "_rpc_test.RPCTest", - "_sanity_test.Sanity", - "_secure_interop_test.SecureInteropTest", + "_sanity_test.Sanity", + "_secure_interop_test.SecureInteropTest", "_thread_cleanup_test.CleanupThreadTest", - "_utilities_test.ChannelConnectivityTest", - "beta_python_plugin_test.PythonPluginTest", - "cygrpc_test.InsecureServerInsecureClient", - "cygrpc_test.SecureServerSecureClient", + "_utilities_test.ChannelConnectivityTest", + "beta_python_plugin_test.PythonPluginTest", + "cygrpc_test.InsecureServerInsecureClient", + "cygrpc_test.SecureServerSecureClient", "cygrpc_test.TypeSmokeTest" ] diff --git a/src/ruby/ext/grpc/rb_call_credentials.c b/src/ruby/ext/grpc/rb_call_credentials.c index 9b6675da846..280f21c9731 100644 --- a/src/ruby/ext/grpc/rb_call_credentials.c +++ b/src/ruby/ext/grpc/rb_call_credentials.c @@ -86,19 +86,16 @@ static VALUE grpc_rb_call_credentials_callback_rescue(VALUE args, rb_funcall(exception_object, rb_intern("backtrace"), 0), rb_intern("join"), 1, rb_str_new2("\n\tfrom ")); - VALUE rb_exception_info = rb_funcall(exception_object, rb_intern("to_s"), 0); - const char *exception_classname = rb_obj_classname(exception_object); + VALUE rb_exception_info = rb_funcall(exception_object, rb_intern("inspect"), 0); (void)args; - gpr_log(GPR_INFO, "Call credentials callback failed: %s: %s\n%s", - exception_classname, StringValueCStr(rb_exception_info), + gpr_log(GPR_INFO, "Call credentials callback failed: %s\n%s", + StringValueCStr(rb_exception_info), StringValueCStr(backtrace)); rb_hash_aset(result, rb_str_new2("metadata"), Qnil); - /* Currently only gives the exception class name. It should be possible get - more details */ rb_hash_aset(result, rb_str_new2("status"), - INT2NUM(GRPC_STATUS_PERMISSION_DENIED)); + INT2NUM(GRPC_STATUS_UNAUTHENTICATED)); rb_hash_aset(result, rb_str_new2("details"), - rb_str_new2(exception_classname)); + rb_exception_info); return result; } diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index a6cad0db1ac..fd73cc79704 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -132,6 +132,11 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import; grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import; grpc_is_binary_header_type grpc_is_binary_header_import; grpc_call_error_to_string_type grpc_call_error_to_string_import; +grpc_resource_quota_create_type grpc_resource_quota_create_import; +grpc_resource_quota_ref_type grpc_resource_quota_ref_import; +grpc_resource_quota_unref_type grpc_resource_quota_unref_import; +grpc_resource_quota_resize_type grpc_resource_quota_resize_import; +grpc_resource_quota_arg_vtable_type grpc_resource_quota_arg_vtable_import; grpc_insecure_channel_create_from_fd_type grpc_insecure_channel_create_from_fd_import; grpc_server_add_insecure_channel_from_fd_type grpc_server_add_insecure_channel_from_fd_import; grpc_use_signal_type grpc_use_signal_import; @@ -401,6 +406,11 @@ void grpc_rb_load_imports(HMODULE library) { grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal"); grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header"); grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string"); + grpc_resource_quota_create_import = (grpc_resource_quota_create_type) GetProcAddress(library, "grpc_resource_quota_create"); + grpc_resource_quota_ref_import = (grpc_resource_quota_ref_type) GetProcAddress(library, "grpc_resource_quota_ref"); + grpc_resource_quota_unref_import = (grpc_resource_quota_unref_type) GetProcAddress(library, "grpc_resource_quota_unref"); + grpc_resource_quota_resize_import = (grpc_resource_quota_resize_type) GetProcAddress(library, "grpc_resource_quota_resize"); + grpc_resource_quota_arg_vtable_import = (grpc_resource_quota_arg_vtable_type) GetProcAddress(library, "grpc_resource_quota_arg_vtable"); grpc_insecure_channel_create_from_fd_import = (grpc_insecure_channel_create_from_fd_type) GetProcAddress(library, "grpc_insecure_channel_create_from_fd"); grpc_server_add_insecure_channel_from_fd_import = (grpc_server_add_insecure_channel_from_fd_type) GetProcAddress(library, "grpc_server_add_insecure_channel_from_fd"); grpc_use_signal_import = (grpc_use_signal_type) GetProcAddress(library, "grpc_use_signal"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 00a67b0b2cd..c2244150f2c 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -347,6 +347,21 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import; typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error); extern grpc_call_error_to_string_type grpc_call_error_to_string_import; #define grpc_call_error_to_string grpc_call_error_to_string_import +typedef grpc_resource_quota *(*grpc_resource_quota_create_type)(const char *trace_name); +extern grpc_resource_quota_create_type grpc_resource_quota_create_import; +#define grpc_resource_quota_create grpc_resource_quota_create_import +typedef void(*grpc_resource_quota_ref_type)(grpc_resource_quota *resource_quota); +extern grpc_resource_quota_ref_type grpc_resource_quota_ref_import; +#define grpc_resource_quota_ref grpc_resource_quota_ref_import +typedef void(*grpc_resource_quota_unref_type)(grpc_resource_quota *resource_quota); +extern grpc_resource_quota_unref_type grpc_resource_quota_unref_import; +#define grpc_resource_quota_unref grpc_resource_quota_unref_import +typedef void(*grpc_resource_quota_resize_type)(grpc_resource_quota *resource_quota, size_t new_size); +extern grpc_resource_quota_resize_type grpc_resource_quota_resize_import; +#define grpc_resource_quota_resize grpc_resource_quota_resize_import +typedef const grpc_arg_pointer_vtable *(*grpc_resource_quota_arg_vtable_type)(void); +extern grpc_resource_quota_arg_vtable_type grpc_resource_quota_arg_vtable_import; +#define grpc_resource_quota_arg_vtable grpc_resource_quota_arg_vtable_import typedef grpc_channel *(*grpc_insecure_channel_create_from_fd_type)(const char *target, int fd, const grpc_channel_args *args); extern grpc_insecure_channel_create_from_fd_type grpc_insecure_channel_create_from_fd_import; #define grpc_insecure_channel_create_from_fd grpc_insecure_channel_create_from_fd_import diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index 7dbcb7d4796..57f99c8ce69 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -54,6 +54,7 @@ module GRPC DEFAULT_MAX_WAITING_REQUESTS = 60 # Default poll period is 1s + # Used for grpc server shutdown and thread pool shutdown timeouts DEFAULT_POLL_PERIOD = 1 # Signal check period is 0.25s @@ -127,7 +128,7 @@ module GRPC deadline = from_relative_time(@poll_period) @server.close(deadline) @pool.shutdown - @pool.wait_for_termination + @pool.wait_for_termination(@poll_period) end def running_state diff --git a/src/ruby/qps/server.rb b/src/ruby/qps/server.rb index d0c2073dd1e..6175855cd94 100644 --- a/src/ruby/qps/server.rb +++ b/src/ruby/qps/server.rb @@ -63,7 +63,9 @@ class BenchmarkServer cred = :this_port_is_insecure end # Make sure server can handle the large number of calls in benchmarks - @server = GRPC::RpcServer.new(pool_size: 100, max_waiting_requests: 100) + # TODO: @apolcyn, if scenario config increases total outstanding + # calls then will need to increase the pool size too + @server = GRPC::RpcServer.new(pool_size: 1024, max_waiting_requests: 1024) @port = @server.add_http2_port("0.0.0.0:" + port.to_s, cred) @server.handle(BenchmarkServiceImpl.new) @start_time = Time.now diff --git a/src/ruby/spec/generic/client_stub_spec.rb b/src/ruby/spec/generic/client_stub_spec.rb index e68b8db7ab7..607a4a3c5da 100644 --- a/src/ruby/spec/generic/client_stub_spec.rb +++ b/src/ruby/spec/generic/client_stub_spec.rb @@ -168,23 +168,61 @@ describe 'ClientStub' do expect(&blk).to raise_error(GRPC::BadStatus) th.join end + + it 'should receive UNAUTHENTICATED if call credentials plugin fails' do + server_port = create_secure_test_server + th = run_request_response(@sent_msg, @resp, @pass) + + certs = load_test_certs + secure_channel_creds = GRPC::Core::ChannelCredentials.new( + certs[0], nil, nil) + secure_stub_opts = { + channel_args: { + GRPC::Core::Channel::SSL_TARGET => 'foo.test.google.fr' + } + } + stub = GRPC::ClientStub.new("localhost:#{server_port}", + secure_channel_creds, **secure_stub_opts) + + error_message = 'Failing call credentials callback' + failing_auth = proc do + fail error_message + end + creds = GRPC::Core::CallCredentials.new(failing_auth) + + error_occured = false + begin + get_response(stub, credentials: creds) + rescue GRPC::BadStatus => e + error_occured = true + expect(e.code).to eq(GRPC::Core::StatusCodes::UNAUTHENTICATED) + expect(e.details.include?(error_message)).to be true + end + expect(error_occured).to eq(true) + + # Kill the server thread so tests can complete + th.kill + end end describe 'without a call operation' do - def get_response(stub) + def get_response(stub, credentials: nil) + puts credentials.inspect stub.request_response(@method, @sent_msg, noop, noop, - metadata: { k1: 'v1', k2: 'v2' }) + metadata: { k1: 'v1', k2: 'v2' }, + credentials: credentials) end it_behaves_like 'request response' end describe 'via a call operation' do - def get_response(stub, run_start_call_first: false) + def get_response(stub, run_start_call_first: false, credentials: nil) op = stub.request_response(@method, @sent_msg, noop, noop, return_op: true, metadata: { k1: 'v1', k2: 'v2' }, - deadline: from_relative_time(2)) + deadline: from_relative_time(2), + credentials: credentials) expect(op).to be_a(GRPC::ActiveCall::Operation) op.start_call if run_start_call_first result = op.execute @@ -492,6 +530,15 @@ describe 'ClientStub' do end end + def create_secure_test_server + certs = load_test_certs + secure_credentials = GRPC::Core::ServerCredentials.new( + nil, [{ private_key: certs[1], cert_chain: certs[2] }], false) + + @server = GRPC::Core::Server.new(nil) + @server.add_http2_port('0.0.0.0:0', secure_credentials) + end + def create_test_server @server = GRPC::Core::Server.new(nil) @server.add_http2_port('0.0.0.0:0', :this_port_is_insecure) diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index 1d5a47336c5..d6928a297cb 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -62,7 +62,7 @@ %> Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.0.0' + version = '1.0.1' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'http://www.grpc.io' @@ -149,7 +149,7 @@ ss.header_mappings_dir = '.' ss.libraries = 'z' ss.dependency "#{s.name}/Interface", version - ss.dependency 'BoringSSL', '~> 6.0' + ss.dependency 'BoringSSL', '~> 7.0' # To save you from scrolling, this is the last part of the podspec. ss.source_files = ${ruby_multiline_list(grpc_private_files(libs), 22)} diff --git a/templates/package.json.template b/templates/package.json.template index 2b3d32ec994..81f39d27f51 100644 --- a/templates/package.json.template +++ b/templates/package.json.template @@ -27,26 +27,28 @@ "coverage": "./node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha src/node/test", "install": "./node_modules/.bin/node-pre-gyp install --fallback-to-build" }, - "bundledDependencies": ["node-pre-gyp"], + "bundledDependencies": [ + "node-pre-gyp" + ], "dependencies": { "arguejs": "^0.2.3", - "lodash": "^3.9.3", + "lodash": "^4.15.0", "nan": "^2.0.0", - "protobufjs": "^4.0.0" + "node-pre-gyp": "^0.6.0", + "protobufjs": "^5.0.0" }, "devDependencies": { - "async": "^1.5.0", + "async": "^2.0.1", "body-parser": "^1.15.2", "express": "^4.14.0", "google-auth-library": "^0.9.2", "google-protobuf": "^3.0.0", - "istanbul": "^0.3.21", + "istanbul": "^0.4.4", "jsdoc": "^3.3.2", "jshint": "^2.5.0", "minimist": "^1.1.0", - "mocha": "^2.3.4", - "mocha-jenkins-reporter": "^0.1.9", - "mustache": "^2.0.0", + "mocha": "^3.0.2", + "mocha-jenkins-reporter": "^0.2.3", "poisson-process": "^0.2.1" }, "engines": { @@ -54,11 +56,10 @@ }, "binary": { "module_name": "grpc_node", - "module_path": "./build/Release/", + "module_path": "src/node/extension_binary", "host": "https://storage.googleapis.com/", "remote_path": "grpc-precompiled-binaries/node/{name}/v{version}", - "package_name": "{node_abi}-{platform}-{arch}.tar.gz", - "module_path": "src/node/extension_binary" + "package_name": "{node_abi}-{platform}-{arch}.tar.gz" }, "files": [ "LICENSE", @@ -79,7 +80,7 @@ ], "main": "src/node/index.js", "license": "BSD-3-Clause", - "jshintConfig" : { + "jshintConfig": { "bitwise": true, "curly": true, "eqeqeq": true, diff --git a/templates/src/python/grpcio_reflection/grpc_version.py.template b/templates/src/python/grpcio_reflection/grpc_version.py.template new file mode 100644 index 00000000000..3e84201f5bc --- /dev/null +++ b/templates/src/python/grpcio_reflection/grpc_version.py.template @@ -0,0 +1,34 @@ +%YAML 1.2 +--- | + # Copyright 2016, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! + + VERSION='${settings.python_version.pep440()}' diff --git a/test/core/bad_client/bad_client.c b/test/core/bad_client/bad_client.c index be88d4a69a9..a9638500b75 100644 --- a/test/core/bad_client/bad_client.c +++ b/test/core/bad_client/bad_client.c @@ -114,7 +114,10 @@ void grpc_run_bad_client_test( grpc_init(); /* Create endpoints */ - sfd = grpc_iomgr_create_endpoint_pair("fixture", 65536); + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("bad_client_test"); + sfd = grpc_iomgr_create_endpoint_pair("fixture", resource_quota, 65536); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); /* Create server, completion events */ a.server = grpc_server_create(NULL, NULL); diff --git a/test/core/end2end/end2end_nosec_tests.c b/test/core/end2end/end2end_nosec_tests.c index a6302621970..caaa97c215b 100644 --- a/test/core/end2end/end2end_nosec_tests.c +++ b/test/core/end2end/end2end_nosec_tests.c @@ -111,6 +111,8 @@ extern void request_with_flags(grpc_end2end_test_config config); extern void request_with_flags_pre_init(void); extern void request_with_payload(grpc_end2end_test_config config); extern void request_with_payload_pre_init(void); +extern void resource_quota_server(grpc_end2end_test_config config); +extern void resource_quota_server_pre_init(void); extern void server_finishes_request(grpc_end2end_test_config config); extern void server_finishes_request_pre_init(void); extern void shutdown_finishes_calls(grpc_end2end_test_config config); @@ -167,6 +169,7 @@ void grpc_end2end_tests_pre_init(void) { registered_call_pre_init(); request_with_flags_pre_init(); request_with_payload_pre_init(); + resource_quota_server_pre_init(); server_finishes_request_pre_init(); shutdown_finishes_calls_pre_init(); shutdown_finishes_tags_pre_init(); @@ -219,6 +222,7 @@ void grpc_end2end_tests(int argc, char **argv, registered_call(config); request_with_flags(config); request_with_payload(config); + resource_quota_server(config); server_finishes_request(config); shutdown_finishes_calls(config); shutdown_finishes_tags(config); @@ -368,6 +372,10 @@ void grpc_end2end_tests(int argc, char **argv, request_with_payload(config); continue; } + if (0 == strcmp("resource_quota_server", argv[i])) { + resource_quota_server(config); + continue; + } if (0 == strcmp("server_finishes_request", argv[i])) { server_finishes_request(config); continue; diff --git a/test/core/end2end/end2end_tests.c b/test/core/end2end/end2end_tests.c index 925872a71ff..6d17e686e61 100644 --- a/test/core/end2end/end2end_tests.c +++ b/test/core/end2end/end2end_tests.c @@ -113,6 +113,8 @@ extern void request_with_flags(grpc_end2end_test_config config); extern void request_with_flags_pre_init(void); extern void request_with_payload(grpc_end2end_test_config config); extern void request_with_payload_pre_init(void); +extern void resource_quota_server(grpc_end2end_test_config config); +extern void resource_quota_server_pre_init(void); extern void server_finishes_request(grpc_end2end_test_config config); extern void server_finishes_request_pre_init(void); extern void shutdown_finishes_calls(grpc_end2end_test_config config); @@ -170,6 +172,7 @@ void grpc_end2end_tests_pre_init(void) { registered_call_pre_init(); request_with_flags_pre_init(); request_with_payload_pre_init(); + resource_quota_server_pre_init(); server_finishes_request_pre_init(); shutdown_finishes_calls_pre_init(); shutdown_finishes_tags_pre_init(); @@ -223,6 +226,7 @@ void grpc_end2end_tests(int argc, char **argv, registered_call(config); request_with_flags(config); request_with_payload(config); + resource_quota_server(config); server_finishes_request(config); shutdown_finishes_calls(config); shutdown_finishes_tags(config); @@ -376,6 +380,10 @@ void grpc_end2end_tests(int argc, char **argv, request_with_payload(config); continue; } + if (0 == strcmp("resource_quota_server", argv[i])) { + resource_quota_server(config); + continue; + } if (0 == strcmp("server_finishes_request", argv[i])) { server_finishes_request(config); continue; diff --git a/test/core/end2end/fixtures/h2_sockpair+trace.c b/test/core/end2end/fixtures/h2_sockpair+trace.c index 15b045c33c4..a358edf3212 100644 --- a/test/core/end2end/fixtures/h2_sockpair+trace.c +++ b/test/core/end2end/fixtures/h2_sockpair+trace.c @@ -96,7 +96,9 @@ static grpc_end2end_test_fixture chttp2_create_fixture_socketpair( f.fixture_data = sfd; f.cq = grpc_completion_queue_create(NULL); - *sfd = grpc_iomgr_create_endpoint_pair("fixture", 65536); + grpc_resource_quota *resource_quota = grpc_resource_quota_create("fixture"); + *sfd = grpc_iomgr_create_endpoint_pair("fixture", resource_quota, 65536); + grpc_resource_quota_unref(resource_quota); return f; } diff --git a/test/core/end2end/fixtures/h2_sockpair.c b/test/core/end2end/fixtures/h2_sockpair.c index 87091f77a3b..8f7083af59a 100644 --- a/test/core/end2end/fixtures/h2_sockpair.c +++ b/test/core/end2end/fixtures/h2_sockpair.c @@ -90,7 +90,9 @@ static grpc_end2end_test_fixture chttp2_create_fixture_socketpair( f.fixture_data = sfd; f.cq = grpc_completion_queue_create(NULL); - *sfd = grpc_iomgr_create_endpoint_pair("fixture", 65536); + grpc_resource_quota *resource_quota = grpc_resource_quota_create("fixture"); + *sfd = grpc_iomgr_create_endpoint_pair("fixture", resource_quota, 65536); + grpc_resource_quota_unref(resource_quota); return f; } diff --git a/test/core/end2end/fixtures/h2_sockpair_1byte.c b/test/core/end2end/fixtures/h2_sockpair_1byte.c index 7d0a14cd634..b3d9924c122 100644 --- a/test/core/end2end/fixtures/h2_sockpair_1byte.c +++ b/test/core/end2end/fixtures/h2_sockpair_1byte.c @@ -90,7 +90,9 @@ static grpc_end2end_test_fixture chttp2_create_fixture_socketpair( f.fixture_data = sfd; f.cq = grpc_completion_queue_create(NULL); - *sfd = grpc_iomgr_create_endpoint_pair("fixture", 1); + grpc_resource_quota *resource_quota = grpc_resource_quota_create("fixture"); + *sfd = grpc_iomgr_create_endpoint_pair("fixture", resource_quota, 1); + grpc_resource_quota_unref(resource_quota); return f; } diff --git a/test/core/end2end/fixtures/http_proxy.c b/test/core/end2end/fixtures/http_proxy.c index b8f046c7dfc..630f47ef0ec 100644 --- a/test/core/end2end/fixtures/http_proxy.c +++ b/test/core/end2end/fixtures/http_proxy.c @@ -359,7 +359,7 @@ static void on_read_request_done(grpc_exec_ctx* exec_ctx, void* arg, const gpr_timespec deadline = gpr_time_add( gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(10, GPR_TIMESPAN)); grpc_tcp_client_connect(exec_ctx, &conn->on_server_connect_done, - &conn->server_endpoint, conn->pollset_set, + &conn->server_endpoint, conn->pollset_set, NULL, &resolved_addresses->addrs[0], deadline); grpc_resolved_addresses_destroy(resolved_addresses); } @@ -418,7 +418,8 @@ static void thread_main(void* arg) { grpc_exec_ctx_finish(&exec_ctx); } -grpc_end2end_http_proxy* grpc_end2end_http_proxy_create() { +grpc_end2end_http_proxy* grpc_end2end_http_proxy_create(void) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_end2end_http_proxy* proxy = gpr_malloc(sizeof(*proxy)); memset(proxy, 0, sizeof(*proxy)); // Construct proxy address. @@ -427,8 +428,8 @@ grpc_end2end_http_proxy* grpc_end2end_http_proxy_create() { gpr_log(GPR_INFO, "Proxy address: %s", proxy->proxy_name); // Create TCP server. proxy->channel_args = grpc_channel_args_copy(NULL); - grpc_error* error = - grpc_tcp_server_create(NULL, proxy->channel_args, &proxy->server); + grpc_error* error = grpc_tcp_server_create( + &exec_ctx, NULL, proxy->channel_args, &proxy->server); GPR_ASSERT(error == GRPC_ERROR_NONE); // Bind to port. grpc_resolved_address resolved_addr; @@ -443,7 +444,6 @@ grpc_end2end_http_proxy* grpc_end2end_http_proxy_create() { // Start server. proxy->pollset = gpr_malloc(grpc_pollset_size()); grpc_pollset_init(proxy->pollset, &proxy->mu); - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_tcp_server_start(&exec_ctx, proxy->server, &proxy->pollset, 1, on_accept, proxy); grpc_exec_ctx_finish(&exec_ctx); diff --git a/test/core/end2end/fuzzers/api_fuzzer.c b/test/core/end2end/fuzzers/api_fuzzer.c index 4e19b760141..659f5cf5cbb 100644 --- a/test/core/end2end/fuzzers/api_fuzzer.c +++ b/test/core/end2end/fuzzers/api_fuzzer.c @@ -173,6 +173,7 @@ static bool is_eof(input_stream *inp) { return inp->cur == inp->end; } static gpr_timespec g_now; static grpc_server *g_server; static grpc_channel *g_channel; +static grpc_resource_quota *g_resource_quota; extern gpr_timespec (*gpr_now_impl)(gpr_clock_type clock_type); @@ -231,8 +232,8 @@ void my_resolve_address(grpc_exec_ctx *exec_ctx, const char *addr, // defined in tcp_client_posix.c extern void (*grpc_tcp_client_connect_impl)( grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, - grpc_pollset_set *interested_parties, const grpc_resolved_address *addr, - gpr_timespec deadline); + grpc_pollset_set *interested_parties, const grpc_channel_args *channel_args, + const grpc_resolved_address *addr, gpr_timespec deadline); static void sched_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, gpr_timespec deadline); @@ -252,7 +253,7 @@ static void do_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { } else if (g_server != NULL) { grpc_endpoint *client; grpc_endpoint *server; - grpc_passthru_endpoint_create(&client, &server); + grpc_passthru_endpoint_create(&client, &server, g_resource_quota); *fc->ep = client; grpc_transport *transport = @@ -289,6 +290,7 @@ static void sched_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure, static void my_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, grpc_pollset_set *interested_parties, + const grpc_channel_args *channel_args, const grpc_resolved_address *addr, gpr_timespec deadline) { sched_connect(exec_ctx, closure, ep, deadline); @@ -520,6 +522,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { int pending_pings = 0; g_active_call = new_call(NULL, ROOT); + g_resource_quota = grpc_resource_quota_create("api_fuzzer"); grpc_completion_queue *cq = grpc_completion_queue_create(NULL); @@ -939,6 +942,11 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { } break; } + // resize the buffer pool + case 21: { + grpc_resource_quota_resize(g_resource_quota, read_uint22(&inp)); + break; + } } } @@ -954,6 +962,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { .type == GRPC_QUEUE_SHUTDOWN); grpc_completion_queue_destroy(cq); + grpc_resource_quota_unref(g_resource_quota); + grpc_shutdown(); return 0; } diff --git a/test/core/end2end/fuzzers/client_fuzzer.c b/test/core/end2end/fuzzers/client_fuzzer.c index 00e650a30b9..d104fe55e59 100644 --- a/test/core/end2end/fuzzers/client_fuzzer.c +++ b/test/core/end2end/fuzzers/client_fuzzer.c @@ -58,7 +58,11 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { grpc_init(); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_endpoint *mock_endpoint = grpc_mock_endpoint_create(discard_write); + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("client_fuzzer"); + grpc_endpoint *mock_endpoint = + grpc_mock_endpoint_create(discard_write, resource_quota); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); grpc_completion_queue *cq = grpc_completion_queue_create(NULL); grpc_transport *transport = diff --git a/test/core/end2end/fuzzers/server_fuzzer.c b/test/core/end2end/fuzzers/server_fuzzer.c index 79eaad70c5d..ae4c8e658d0 100644 --- a/test/core/end2end/fuzzers/server_fuzzer.c +++ b/test/core/end2end/fuzzers/server_fuzzer.c @@ -56,7 +56,11 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { grpc_init(); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_endpoint *mock_endpoint = grpc_mock_endpoint_create(discard_write); + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("server_fuzzer"); + grpc_endpoint *mock_endpoint = + grpc_mock_endpoint_create(discard_write, resource_quota); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); grpc_mock_endpoint_put_read( &exec_ctx, mock_endpoint, gpr_slice_from_copied_buffer((const char *)data, size)); diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py index 0bbf10d475c..bdb3874c210 100755 --- a/test/core/end2end/gen_build_yaml.py +++ b/test/core/end2end/gen_build_yaml.py @@ -39,9 +39,9 @@ import hashlib FixtureOptions = collections.namedtuple( 'FixtureOptions', - 'fullstack includes_proxy dns_resolver secure platforms ci_mac tracing exclude_configs exclude_iomgrs') + 'fullstack includes_proxy dns_resolver secure platforms ci_mac tracing exclude_configs exclude_iomgrs large_writes') default_unsecure_fixture_options = FixtureOptions( - True, False, True, False, ['windows', 'linux', 'mac', 'posix'], True, False, [], []) + True, False, True, False, ['windows', 'linux', 'mac', 'posix'], True, False, [], [], True) socketpair_unsecure_fixture_options = default_unsecure_fixture_options._replace(fullstack=False, dns_resolver=False) default_secure_fixture_options = default_unsecure_fixture_options._replace(secure=True) uds_fixture_options = default_unsecure_fixture_options._replace(dns_resolver=False, platforms=['linux', 'mac', 'posix'], exclude_iomgrs=['uv']) @@ -69,11 +69,12 @@ END2END_FIXTURES = { 'h2_proxy': default_unsecure_fixture_options._replace( includes_proxy=True, ci_mac=False, exclude_iomgrs=['uv']), 'h2_sockpair_1byte': socketpair_unsecure_fixture_options._replace( - ci_mac=False, exclude_configs=['msan'], exclude_iomgrs=['uv']), + ci_mac=False, exclude_configs=['msan'], large_writes=False, + exclude_iomgrs=['uv']), 'h2_sockpair': socketpair_unsecure_fixture_options._replace( ci_mac=False, exclude_iomgrs=['uv']), 'h2_sockpair+trace': socketpair_unsecure_fixture_options._replace( - ci_mac=False, tracing=True, exclude_iomgrs=['uv']), + ci_mac=False, tracing=True, large_writes=False, exclude_iomgrs=['uv']), 'h2_ssl': default_secure_fixture_options, 'h2_ssl_cert': default_secure_fixture_options, 'h2_ssl_proxy': default_secure_fixture_options._replace( @@ -83,8 +84,8 @@ END2END_FIXTURES = { TestOptions = collections.namedtuple( 'TestOptions', - 'needs_fullstack needs_dns proxyable secure traceable cpu_cost exclude_iomgrs') -default_test_options = TestOptions(False, False, True, False, True, 1.0, []) + 'needs_fullstack needs_dns proxyable secure traceable cpu_cost exclude_iomgrs large_writes flaky') +default_test_options = TestOptions(False, False, True, False, True, 1.0, [], False, False) connectivity_test_options = default_test_options._replace(needs_fullstack=True) LOWCPU = 0.1 @@ -93,6 +94,8 @@ LOWCPU = 0.1 END2END_TESTS = { 'bad_hostname': default_test_options, 'binary_metadata': default_test_options, + 'resource_quota_server': default_test_options._replace(large_writes=True, + proxyable=False), 'call_creds': default_test_options._replace(secure=True), 'cancel_after_accept': default_test_options._replace(cpu_cost=LOWCPU), 'cancel_after_client_done': default_test_options, @@ -105,7 +108,7 @@ END2END_TESTS = { proxyable=False, cpu_cost=LOWCPU, exclude_iomgrs=['uv']), 'default_host': default_test_options._replace(needs_fullstack=True, needs_dns=True), - 'disappearing_server': connectivity_test_options, + 'disappearing_server': connectivity_test_options._replace(flaky=True), 'empty_batch': default_test_options, 'filter_causes_close': default_test_options, 'filter_call_init_fails': default_test_options, @@ -155,6 +158,9 @@ def compatible(f, t): if not END2END_TESTS[t].traceable: if END2END_FIXTURES[f].tracing: return False + if END2END_TESTS[t].large_writes: + if not END2END_FIXTURES[f].large_writes: + return False return True @@ -257,7 +263,7 @@ def main(): 'ci_platforms': (END2END_FIXTURES[f].platforms if END2END_FIXTURES[f].ci_mac else without( END2END_FIXTURES[f].platforms, 'mac')), - 'flaky': False, + 'flaky': END2END_TESTS[t].flaky, 'language': 'c', 'cpu_cost': END2END_TESTS[t].cpu_cost, } @@ -274,7 +280,7 @@ def main(): 'ci_platforms': (END2END_FIXTURES[f].platforms if END2END_FIXTURES[f].ci_mac else without( END2END_FIXTURES[f].platforms, 'mac')), - 'flaky': False, + 'flaky': END2END_TESTS[t].flaky, 'language': 'c', 'cpu_cost': END2END_TESTS[t].cpu_cost, } diff --git a/test/core/end2end/tests/max_message_length.c b/test/core/end2end/tests/max_message_length.c index e698987a0a4..449a581d232 100644 --- a/test/core/end2end/tests/max_message_length.c +++ b/test/core/end2end/tests/max_message_length.c @@ -402,7 +402,6 @@ static void test_max_message_length_on_response(grpc_end2end_test_config config, GPR_ASSERT(0 == strcmp(call_details.method, "/service/method")); GPR_ASSERT(0 == strcmp(call_details.host, "foo.test.google.fr:1234")); - GPR_ASSERT(was_cancelled == 0); GPR_ASSERT(status == GRPC_STATUS_INVALID_ARGUMENT); GPR_ASSERT(strcmp(details, diff --git a/test/core/end2end/tests/network_status_change.c b/test/core/end2end/tests/network_status_change.c index 8d6b8f87840..fa711bb0b60 100644 --- a/test/core/end2end/tests/network_status_change.c +++ b/test/core/end2end/tests/network_status_change.c @@ -213,7 +213,6 @@ static void test_invoke_network_status_change(grpc_end2end_test_config config) { GPR_ASSERT(status == GRPC_STATUS_UNAVAILABLE); GPR_ASSERT(0 == strcmp(call_details.method, "/foo")); GPR_ASSERT(0 == strcmp(call_details.host, "foo.test.google.fr")); - GPR_ASSERT(was_cancelled == 0); gpr_free(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/resource_quota_server.c b/test/core/end2end/tests/resource_quota_server.c new file mode 100644 index 00000000000..fbd4986dfba --- /dev/null +++ b/test/core/end2end/tests/resource_quota_server.c @@ -0,0 +1,363 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "test/core/end2end/end2end_tests.h" + +#include +#include + +#include +#include +#include +#include +#include +#include "test/core/end2end/cq_verifier.h" + +static void *tag(intptr_t t) { return (void *)t; } + +static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, + const char *test_name, + grpc_channel_args *client_args, + grpc_channel_args *server_args) { + grpc_end2end_test_fixture f; + gpr_log(GPR_INFO, "%s/%s", test_name, config.name); + f = config.create_fixture(client_args, server_args); + config.init_server(&f, server_args); + config.init_client(&f, client_args, NULL); + return f; +} + +static gpr_timespec n_seconds_time(int n) { + return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(n); +} + +static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } + +static void drain_cq(grpc_completion_queue *cq) { + grpc_event ev; + do { + ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + } while (ev.type != GRPC_QUEUE_SHUTDOWN); +} + +static void shutdown_server(grpc_end2end_test_fixture *f) { + if (!f->server) return; + grpc_server_shutdown_and_notify(f->server, f->cq, tag(1000)); + GPR_ASSERT(grpc_completion_queue_pluck( + f->cq, tag(1000), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5), NULL) + .type == GRPC_OP_COMPLETE); + grpc_server_destroy(f->server); + f->server = NULL; +} + +static void shutdown_client(grpc_end2end_test_fixture *f) { + if (!f->client) return; + grpc_channel_destroy(f->client); + f->client = NULL; +} + +static void end_test(grpc_end2end_test_fixture *f) { + shutdown_server(f); + shutdown_client(f); + + grpc_completion_queue_shutdown(f->cq); + drain_cq(f->cq); + grpc_completion_queue_destroy(f->cq); +} + +/* Creates and returns a gpr_slice containing random alphanumeric characters. */ +static gpr_slice generate_random_slice() { + size_t i; + static const char chars[] = "abcdefghijklmnopqrstuvwxyz1234567890"; + char *output; + const size_t output_size = 1024 * 1024; + output = gpr_malloc(output_size); + for (i = 0; i < output_size - 1; ++i) { + output[i] = chars[rand() % (int)(sizeof(chars) - 1)]; + } + output[output_size - 1] = '\0'; + gpr_slice out = gpr_slice_from_copied_string(output); + gpr_free(output); + return out; +} + +void resource_quota_server(grpc_end2end_test_config config) { + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("test_server"); + grpc_resource_quota_resize(resource_quota, 5 * 1024 * 1024); + +#define NUM_CALLS 100 +#define CLIENT_BASE_TAG 1000 +#define SERVER_START_BASE_TAG 2000 +#define SERVER_RECV_BASE_TAG 3000 +#define SERVER_END_BASE_TAG 4000 + + grpc_arg arg; + arg.key = GRPC_ARG_RESOURCE_QUOTA; + arg.type = GRPC_ARG_POINTER; + arg.value.pointer.p = resource_quota; + arg.value.pointer.vtable = grpc_resource_quota_arg_vtable(); + grpc_channel_args args = {1, &arg}; + + grpc_end2end_test_fixture f = + begin_test(config, "resource_quota_server", NULL, &args); + + /* Create large request and response bodies. These are big enough to require + * multiple round trips to deliver to the peer, and their exact contents of + * will be verified on completion. */ + gpr_slice request_payload_slice = generate_random_slice(); + + grpc_call *client_calls[NUM_CALLS]; + grpc_call *server_calls[NUM_CALLS]; + grpc_metadata_array initial_metadata_recv[NUM_CALLS]; + grpc_metadata_array trailing_metadata_recv[NUM_CALLS]; + grpc_metadata_array request_metadata_recv[NUM_CALLS]; + grpc_call_details call_details[NUM_CALLS]; + grpc_status_code status[NUM_CALLS]; + char *details[NUM_CALLS]; + size_t details_capacity[NUM_CALLS]; + grpc_byte_buffer *request_payload_recv[NUM_CALLS]; + int was_cancelled[NUM_CALLS]; + grpc_call_error error; + int pending_client_calls = 0; + int pending_server_start_calls = 0; + int pending_server_recv_calls = 0; + int pending_server_end_calls = 0; + int cancelled_calls_on_client = 0; + int cancelled_calls_on_server = 0; + + grpc_byte_buffer *request_payload = + grpc_raw_byte_buffer_create(&request_payload_slice, 1); + + grpc_op ops[6]; + grpc_op *op; + + for (int i = 0; i < NUM_CALLS; i++) { + grpc_metadata_array_init(&initial_metadata_recv[i]); + grpc_metadata_array_init(&trailing_metadata_recv[i]); + grpc_metadata_array_init(&request_metadata_recv[i]); + grpc_call_details_init(&call_details[i]); + details[i] = NULL; + details_capacity[i] = 0; + request_payload_recv[i] = NULL; + was_cancelled[i] = 0; + } + + for (int i = 0; i < NUM_CALLS; i++) { + error = grpc_server_request_call( + f.server, &server_calls[i], &call_details[i], &request_metadata_recv[i], + f.cq, f.cq, tag(SERVER_START_BASE_TAG + i)); + GPR_ASSERT(GRPC_CALL_OK == error); + + pending_server_start_calls++; + } + + for (int i = 0; i < NUM_CALLS; i++) { + client_calls[i] = grpc_channel_create_call( + f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, "/foo", + "foo.test.google.fr", n_seconds_time(60), NULL); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_MESSAGE; + op->data.send_message = request_payload; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata = &initial_metadata_recv[i]; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = + &trailing_metadata_recv[i]; + op->data.recv_status_on_client.status = &status[i]; + op->data.recv_status_on_client.status_details = &details[i]; + op->data.recv_status_on_client.status_details_capacity = + &details_capacity[i]; + op->flags = 0; + op->reserved = NULL; + op++; + error = grpc_call_start_batch(client_calls[i], ops, (size_t)(op - ops), + tag(CLIENT_BASE_TAG + i), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + pending_client_calls++; + } + + while (pending_client_calls + pending_server_recv_calls + + pending_server_end_calls > + 0) { + grpc_event ev = grpc_completion_queue_next(f.cq, n_seconds_time(10), NULL); + GPR_ASSERT(ev.type == GRPC_OP_COMPLETE); + + int ev_tag = (int)(intptr_t)ev.tag; + if (ev_tag < CLIENT_BASE_TAG) { + abort(); /* illegal tag */ + } else if (ev_tag < SERVER_START_BASE_TAG) { + /* client call finished */ + int call_id = ev_tag - CLIENT_BASE_TAG; + GPR_ASSERT(call_id >= 0); + GPR_ASSERT(call_id < NUM_CALLS); + switch (status[call_id]) { + case GRPC_STATUS_RESOURCE_EXHAUSTED: + cancelled_calls_on_client++; + break; + case GRPC_STATUS_OK: + break; + default: + gpr_log(GPR_ERROR, "Unexpected status code: %d", status[call_id]); + abort(); + } + GPR_ASSERT(pending_client_calls > 0); + + grpc_metadata_array_destroy(&initial_metadata_recv[call_id]); + grpc_metadata_array_destroy(&trailing_metadata_recv[call_id]); + grpc_call_destroy(client_calls[call_id]); + gpr_free(details[call_id]); + + pending_client_calls--; + } else if (ev_tag < SERVER_RECV_BASE_TAG) { + /* new incoming call to the server */ + int call_id = ev_tag - SERVER_START_BASE_TAG; + GPR_ASSERT(call_id >= 0); + GPR_ASSERT(call_id < NUM_CALLS); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_MESSAGE; + op->data.recv_message = &request_payload_recv[call_id]; + op->flags = 0; + op->reserved = NULL; + op++; + error = + grpc_call_start_batch(server_calls[call_id], ops, (size_t)(op - ops), + tag(SERVER_RECV_BASE_TAG + call_id), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + GPR_ASSERT(pending_server_start_calls > 0); + pending_server_start_calls--; + pending_server_recv_calls++; + + grpc_call_details_destroy(&call_details[call_id]); + grpc_metadata_array_destroy(&request_metadata_recv[call_id]); + } else if (ev_tag < SERVER_END_BASE_TAG) { + /* finished read on the server */ + int call_id = ev_tag - SERVER_RECV_BASE_TAG; + GPR_ASSERT(call_id >= 0); + GPR_ASSERT(call_id < NUM_CALLS); + + if (ev.success) { + if (request_payload_recv[call_id] != NULL) { + grpc_byte_buffer_destroy(request_payload_recv[call_id]); + request_payload_recv[call_id] = NULL; + } + } else { + GPR_ASSERT(request_payload_recv[call_id] == NULL); + } + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; + op->data.recv_close_on_server.cancelled = &was_cancelled[call_id]; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; + op->data.send_status_from_server.trailing_metadata_count = 0; + op->data.send_status_from_server.status = GRPC_STATUS_OK; + op->data.send_status_from_server.status_details = "xyz"; + op->flags = 0; + op->reserved = NULL; + op++; + error = + grpc_call_start_batch(server_calls[call_id], ops, (size_t)(op - ops), + tag(SERVER_END_BASE_TAG + call_id), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + GPR_ASSERT(pending_server_recv_calls > 0); + pending_server_recv_calls--; + pending_server_end_calls++; + } else { + int call_id = ev_tag - SERVER_END_BASE_TAG; + GPR_ASSERT(call_id >= 0); + GPR_ASSERT(call_id < NUM_CALLS); + + if (was_cancelled[call_id]) { + cancelled_calls_on_server++; + } + GPR_ASSERT(pending_server_end_calls > 0); + pending_server_end_calls--; + + grpc_call_destroy(server_calls[call_id]); + } + } + + gpr_log( + GPR_INFO, + "Done. %d total calls: %d cancelled at server, %d cancelled at client.", + NUM_CALLS, cancelled_calls_on_server, cancelled_calls_on_client); + + /* The call may be cancelled after the server has sent its status but before + * the client has received it. This means that we should see strictly more + * failures on the client than on the server. */ + GPR_ASSERT(cancelled_calls_on_client >= cancelled_calls_on_server); + /* However, we shouldn't see radically more... 0.9 is a guessed bound on what + * we'd want that ratio to be... to at least trigger some investigation should + * that ratio become much higher. */ + GPR_ASSERT(cancelled_calls_on_server >= 0.9 * cancelled_calls_on_client); + + grpc_byte_buffer_destroy(request_payload); + gpr_slice_unref(request_payload_slice); + grpc_resource_quota_unref(resource_quota); + + end_test(&f); + config.tear_down_data(&f); +} + +void resource_quota_server_pre_init(void) {} diff --git a/test/core/http/httpcli_test.c b/test/core/http/httpcli_test.c index 38b32a3867f..3e312c1ddee 100644 --- a/test/core/http/httpcli_test.c +++ b/test/core/http/httpcli_test.c @@ -89,8 +89,11 @@ static void test_get(int port) { grpc_http_response response; memset(&response, 0, sizeof(response)); - grpc_httpcli_get(&exec_ctx, &g_context, &g_pops, &req, n_seconds_time(15), + grpc_resource_quota *resource_quota = grpc_resource_quota_create("test_get"); + grpc_httpcli_get(&exec_ctx, &g_context, &g_pops, resource_quota, &req, + n_seconds_time(15), grpc_closure_create(on_finish, &response), &response); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; @@ -126,9 +129,11 @@ static void test_post(int port) { grpc_http_response response; memset(&response, 0, sizeof(response)); - grpc_httpcli_post(&exec_ctx, &g_context, &g_pops, &req, "hello", 5, - n_seconds_time(15), + grpc_resource_quota *resource_quota = grpc_resource_quota_create("test_post"); + grpc_httpcli_post(&exec_ctx, &g_context, &g_pops, resource_quota, &req, + "hello", 5, n_seconds_time(15), grpc_closure_create(on_finish, &response), &response); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; diff --git a/test/core/http/httpscli_test.c b/test/core/http/httpscli_test.c index 359e5576894..d06035149ef 100644 --- a/test/core/http/httpscli_test.c +++ b/test/core/http/httpscli_test.c @@ -90,8 +90,11 @@ static void test_get(int port) { grpc_http_response response; memset(&response, 0, sizeof(response)); - grpc_httpcli_get(&exec_ctx, &g_context, &g_pops, &req, n_seconds_time(15), + grpc_resource_quota *resource_quota = grpc_resource_quota_create("test_get"); + grpc_httpcli_get(&exec_ctx, &g_context, &g_pops, resource_quota, &req, + n_seconds_time(15), grpc_closure_create(on_finish, &response), &response); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; @@ -128,9 +131,11 @@ static void test_post(int port) { grpc_http_response response; memset(&response, 0, sizeof(response)); - grpc_httpcli_post(&exec_ctx, &g_context, &g_pops, &req, "hello", 5, - n_seconds_time(15), + grpc_resource_quota *resource_quota = grpc_resource_quota_create("test_post"); + grpc_httpcli_post(&exec_ctx, &g_context, &g_pops, resource_quota, &req, + "hello", 5, n_seconds_time(15), grpc_closure_create(on_finish, &response), &response); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; diff --git a/test/core/internal_api_canaries/iomgr.c b/test/core/internal_api_canaries/iomgr.c index 27d630623ed..f1efa87a696 100644 --- a/test/core/internal_api_canaries/iomgr.c +++ b/test/core/internal_api_canaries/iomgr.c @@ -84,6 +84,7 @@ static void test_code(void) { grpc_endpoint_add_to_pollset_set, grpc_endpoint_shutdown, grpc_endpoint_destroy, + grpc_endpoint_get_resource_user, grpc_endpoint_get_peer}; endpoint.vtable = &vtable; diff --git a/test/core/iomgr/endpoint_pair_test.c b/test/core/iomgr/endpoint_pair_test.c index 99b86b62134..2a257a7ceac 100644 --- a/test/core/iomgr/endpoint_pair_test.c +++ b/test/core/iomgr/endpoint_pair_test.c @@ -49,7 +49,11 @@ static grpc_endpoint_test_fixture create_fixture_endpoint_pair( size_t slice_size) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_endpoint_test_fixture f; - grpc_endpoint_pair p = grpc_iomgr_create_endpoint_pair("test", slice_size); + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("endpoint_pair_test"); + grpc_endpoint_pair p = + grpc_iomgr_create_endpoint_pair("test", resource_quota, slice_size); + grpc_resource_quota_unref(resource_quota); f.client_ep = p.client; f.server_ep = p.server; diff --git a/test/core/iomgr/fd_conservation_posix_test.c b/test/core/iomgr/fd_conservation_posix_test.c index bbb3f46497f..652b37eb6f4 100644 --- a/test/core/iomgr/fd_conservation_posix_test.c +++ b/test/core/iomgr/fd_conservation_posix_test.c @@ -52,15 +52,19 @@ int main(int argc, char **argv) { of descriptors */ rlim.rlim_cur = rlim.rlim_max = 10; GPR_ASSERT(0 == setrlimit(RLIMIT_NOFILE, &rlim)); + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("fd_conservation_posix_test"); for (i = 0; i < 100; i++) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - p = grpc_iomgr_create_endpoint_pair("test", 1); + p = grpc_iomgr_create_endpoint_pair("test", resource_quota, 1); grpc_endpoint_destroy(&exec_ctx, p.client); grpc_endpoint_destroy(&exec_ctx, p.server); grpc_exec_ctx_finish(&exec_ctx); } + grpc_resource_quota_unref(resource_quota); + grpc_iomgr_shutdown(); return 0; } diff --git a/test/core/iomgr/resource_quota_test.c b/test/core/iomgr/resource_quota_test.c new file mode 100644 index 00000000000..34dee1aee19 --- /dev/null +++ b/test/core/iomgr/resource_quota_test.c @@ -0,0 +1,749 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/resource_quota.h" + +#include +#include + +#include "test/core/util/test_config.h" + +static void inc_int_cb(grpc_exec_ctx *exec_ctx, void *a, grpc_error *error) { + ++*(int *)a; +} + +static void set_bool_cb(grpc_exec_ctx *exec_ctx, void *a, grpc_error *error) { + *(bool *)a = true; +} +grpc_closure *set_bool(bool *p) { return grpc_closure_create(set_bool_cb, p); } + +typedef struct { + size_t size; + grpc_resource_user *resource_user; + grpc_closure *then; +} reclaimer_args; +static void reclaimer_cb(grpc_exec_ctx *exec_ctx, void *args, + grpc_error *error) { + GPR_ASSERT(error == GRPC_ERROR_NONE); + reclaimer_args *a = args; + grpc_resource_user_free(exec_ctx, a->resource_user, a->size); + grpc_resource_user_finish_reclamation(exec_ctx, a->resource_user); + grpc_closure_run(exec_ctx, a->then, GRPC_ERROR_NONE); + gpr_free(a); +} +grpc_closure *make_reclaimer(grpc_resource_user *resource_user, size_t size, + grpc_closure *then) { + reclaimer_args *a = gpr_malloc(sizeof(*a)); + a->size = size; + a->resource_user = resource_user; + a->then = then; + return grpc_closure_create(reclaimer_cb, a); +} + +static void unused_reclaimer_cb(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + GPR_ASSERT(error == GRPC_ERROR_CANCELLED); + grpc_closure_run(exec_ctx, arg, GRPC_ERROR_NONE); +} +grpc_closure *make_unused_reclaimer(grpc_closure *then) { + return grpc_closure_create(unused_reclaimer_cb, then); +} + +static void destroy_user(grpc_resource_user *usr) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + bool done = false; + grpc_resource_user_shutdown(&exec_ctx, usr, set_bool(&done)); + grpc_exec_ctx_flush(&exec_ctx); + GPR_ASSERT(done); + grpc_resource_user_destroy(&exec_ctx, usr); + grpc_exec_ctx_finish(&exec_ctx); +} + +static void test_no_op(void) { + gpr_log(GPR_INFO, "** test_no_op **"); + grpc_resource_quota_unref(grpc_resource_quota_create("test_no_op")); +} + +static void test_resize_then_destroy(void) { + gpr_log(GPR_INFO, "** test_resize_then_destroy **"); + grpc_resource_quota *q = + grpc_resource_quota_create("test_resize_then_destroy"); + grpc_resource_quota_resize(q, 1024 * 1024); + grpc_resource_quota_unref(q); +} + +static void test_resource_user_no_op(void) { + gpr_log(GPR_INFO, "** test_resource_user_no_op **"); + grpc_resource_quota *q = + grpc_resource_quota_create("test_resource_user_no_op"); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + grpc_resource_quota_unref(q); + destroy_user(&usr); +} + +static void test_instant_alloc_then_free(void) { + gpr_log(GPR_INFO, "** test_instant_alloc_then_free **"); + grpc_resource_quota *q = + grpc_resource_quota_create("test_instant_alloc_then_free"); + grpc_resource_quota_resize(q, 1024 * 1024); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, NULL); + grpc_exec_ctx_finish(&exec_ctx); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + grpc_resource_quota_unref(q); + destroy_user(&usr); +} + +static void test_instant_alloc_free_pair(void) { + gpr_log(GPR_INFO, "** test_instant_alloc_free_pair **"); + grpc_resource_quota *q = + grpc_resource_quota_create("test_instant_alloc_free_pair"); + grpc_resource_quota_resize(q, 1024 * 1024); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, NULL); + grpc_resource_user_free(&exec_ctx, &usr, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + grpc_resource_quota_unref(q); + destroy_user(&usr); +} + +static void test_simple_async_alloc(void) { + gpr_log(GPR_INFO, "** test_simple_async_alloc **"); + grpc_resource_quota *q = + grpc_resource_quota_create("test_simple_async_alloc"); + grpc_resource_quota_resize(q, 1024 * 1024); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + grpc_resource_quota_unref(q); + destroy_user(&usr); +} + +static void test_async_alloc_blocked_by_size(void) { + gpr_log(GPR_INFO, "** test_async_alloc_blocked_by_size **"); + grpc_resource_quota *q = + grpc_resource_quota_create("test_async_alloc_blocked_by_size"); + grpc_resource_quota_resize(q, 1); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + bool done = false; + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(!done); + } + grpc_resource_quota_resize(q, 1024); + GPR_ASSERT(done); + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + grpc_resource_quota_unref(q); + destroy_user(&usr); +} + +static void test_scavenge(void) { + gpr_log(GPR_INFO, "** test_scavenge **"); + grpc_resource_quota *q = grpc_resource_quota_create("test_scavenge"); + grpc_resource_quota_resize(q, 1024); + grpc_resource_user usr1; + grpc_resource_user usr2; + grpc_resource_user_init(&usr1, q, "usr1"); + grpc_resource_user_init(&usr2, q, "usr2"); + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr1, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr1, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr2, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr2, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + grpc_resource_quota_unref(q); + destroy_user(&usr1); + destroy_user(&usr2); +} + +static void test_scavenge_blocked(void) { + gpr_log(GPR_INFO, "** test_scavenge_blocked **"); + grpc_resource_quota *q = grpc_resource_quota_create("test_scavenge_blocked"); + grpc_resource_quota_resize(q, 1024); + grpc_resource_user usr1; + grpc_resource_user usr2; + grpc_resource_user_init(&usr1, q, "usr1"); + grpc_resource_user_init(&usr2, q, "usr2"); + bool done; + { + done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr1, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + } + { + done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr2, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(!done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr1, 1024); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr2, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + grpc_resource_quota_unref(q); + destroy_user(&usr1); + destroy_user(&usr2); +} + +static void test_blocked_until_scheduled_reclaim(void) { + gpr_log(GPR_INFO, "** test_blocked_until_scheduled_reclaim **"); + grpc_resource_quota *q = + grpc_resource_quota_create("test_blocked_until_scheduled_reclaim"); + grpc_resource_quota_resize(q, 1024); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + } + bool reclaim_done = false; + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_post_reclaimer( + &exec_ctx, &usr, false, + make_reclaimer(&usr, 1024, set_bool(&reclaim_done))); + grpc_exec_ctx_finish(&exec_ctx); + } + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(reclaim_done); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + grpc_resource_quota_unref(q); + destroy_user(&usr); +} + +static void test_blocked_until_scheduled_reclaim_and_scavenge(void) { + gpr_log(GPR_INFO, "** test_blocked_until_scheduled_reclaim_and_scavenge **"); + grpc_resource_quota *q = grpc_resource_quota_create( + "test_blocked_until_scheduled_reclaim_and_scavenge"); + grpc_resource_quota_resize(q, 1024); + grpc_resource_user usr1; + grpc_resource_user usr2; + grpc_resource_user_init(&usr1, q, "usr1"); + grpc_resource_user_init(&usr2, q, "usr2"); + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr1, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + } + bool reclaim_done = false; + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_post_reclaimer( + &exec_ctx, &usr1, false, + make_reclaimer(&usr1, 1024, set_bool(&reclaim_done))); + grpc_exec_ctx_finish(&exec_ctx); + } + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr2, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(reclaim_done); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr2, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + grpc_resource_quota_unref(q); + destroy_user(&usr1); + destroy_user(&usr2); +} + +static void test_blocked_until_scheduled_destructive_reclaim(void) { + gpr_log(GPR_INFO, "** test_blocked_until_scheduled_destructive_reclaim **"); + grpc_resource_quota *q = grpc_resource_quota_create( + "test_blocked_until_scheduled_destructive_reclaim"); + grpc_resource_quota_resize(q, 1024); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + } + bool reclaim_done = false; + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_post_reclaimer( + &exec_ctx, &usr, true, + make_reclaimer(&usr, 1024, set_bool(&reclaim_done))); + grpc_exec_ctx_finish(&exec_ctx); + } + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(reclaim_done); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + grpc_resource_quota_unref(q); + destroy_user(&usr); +} + +static void test_unused_reclaim_is_cancelled(void) { + gpr_log(GPR_INFO, "** test_unused_reclaim_is_cancelled **"); + grpc_resource_quota *q = + grpc_resource_quota_create("test_unused_reclaim_is_cancelled"); + grpc_resource_quota_resize(q, 1024); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + bool benign_done = false; + bool destructive_done = false; + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_post_reclaimer( + &exec_ctx, &usr, false, make_unused_reclaimer(set_bool(&benign_done))); + grpc_resource_user_post_reclaimer( + &exec_ctx, &usr, true, + make_unused_reclaimer(set_bool(&destructive_done))); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(!benign_done); + GPR_ASSERT(!destructive_done); + } + grpc_resource_quota_unref(q); + destroy_user(&usr); + GPR_ASSERT(benign_done); + GPR_ASSERT(destructive_done); +} + +static void test_benign_reclaim_is_preferred(void) { + gpr_log(GPR_INFO, "** test_benign_reclaim_is_preferred **"); + grpc_resource_quota *q = + grpc_resource_quota_create("test_benign_reclaim_is_preferred"); + grpc_resource_quota_resize(q, 1024); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + bool benign_done = false; + bool destructive_done = false; + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_post_reclaimer( + &exec_ctx, &usr, false, + make_reclaimer(&usr, 1024, set_bool(&benign_done))); + grpc_resource_user_post_reclaimer( + &exec_ctx, &usr, true, + make_unused_reclaimer(set_bool(&destructive_done))); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(!benign_done); + GPR_ASSERT(!destructive_done); + } + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(benign_done); + GPR_ASSERT(!destructive_done); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + grpc_resource_quota_unref(q); + destroy_user(&usr); + GPR_ASSERT(benign_done); + GPR_ASSERT(destructive_done); +} + +static void test_multiple_reclaims_can_be_triggered(void) { + gpr_log(GPR_INFO, "** test_multiple_reclaims_can_be_triggered **"); + grpc_resource_quota *q = + grpc_resource_quota_create("test_multiple_reclaims_can_be_triggered"); + grpc_resource_quota_resize(q, 1024); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + bool benign_done = false; + bool destructive_done = false; + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_post_reclaimer( + &exec_ctx, &usr, false, + make_reclaimer(&usr, 512, set_bool(&benign_done))); + grpc_resource_user_post_reclaimer( + &exec_ctx, &usr, true, + make_reclaimer(&usr, 512, set_bool(&destructive_done))); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(!benign_done); + GPR_ASSERT(!destructive_done); + } + { + bool done = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(benign_done); + GPR_ASSERT(destructive_done); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + grpc_resource_quota_unref(q); + destroy_user(&usr); + GPR_ASSERT(benign_done); + GPR_ASSERT(destructive_done); +} + +static void test_resource_user_stays_allocated_until_memory_released(void) { + gpr_log(GPR_INFO, + "** test_resource_user_stays_allocated_until_memory_released **"); + grpc_resource_quota *q = grpc_resource_quota_create( + "test_resource_user_stays_allocated_until_memory_released"); + grpc_resource_quota_resize(q, 1024 * 1024); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + bool done = false; + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, NULL); + grpc_exec_ctx_finish(&exec_ctx); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_quota_unref(q); + grpc_resource_user_shutdown(&exec_ctx, &usr, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(!done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr, 1024); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_destroy(&exec_ctx, &usr); + grpc_exec_ctx_finish(&exec_ctx); + } +} + +static void +test_resource_user_stays_allocated_and_reclaimers_unrun_until_memory_released( + void) { + gpr_log(GPR_INFO, + "** " + "test_resource_user_stays_allocated_and_reclaimers_unrun_until_" + "memory_released **"); + grpc_resource_quota *q = grpc_resource_quota_create( + "test_resource_user_stays_allocated_and_reclaimers_unrun_until_memory_" + "released"); + grpc_resource_quota_resize(q, 1024); + for (int i = 0; i < 10; i++) { + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + bool done = false; + bool reclaimer_cancelled = false; + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_post_reclaimer( + &exec_ctx, &usr, false, + make_unused_reclaimer(set_bool(&reclaimer_cancelled))); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(!reclaimer_cancelled); + } + { + bool allocated = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&allocated)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(allocated); + GPR_ASSERT(!reclaimer_cancelled); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_shutdown(&exec_ctx, &usr, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(!done); + GPR_ASSERT(!reclaimer_cancelled); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr, 1024); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(done); + GPR_ASSERT(reclaimer_cancelled); + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_destroy(&exec_ctx, &usr); + grpc_exec_ctx_finish(&exec_ctx); + } + } + grpc_resource_quota_unref(q); +} + +static void test_reclaimers_can_be_posted_repeatedly(void) { + gpr_log(GPR_INFO, "** test_reclaimers_can_be_posted_repeatedly **"); + grpc_resource_quota *q = + grpc_resource_quota_create("test_reclaimers_can_be_posted_repeatedly"); + grpc_resource_quota_resize(q, 1024); + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + { + bool allocated = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&allocated)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(allocated); + } + for (int i = 0; i < 10; i++) { + bool reclaimer_done = false; + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_post_reclaimer( + &exec_ctx, &usr, false, + make_reclaimer(&usr, 1024, set_bool(&reclaimer_done))); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(!reclaimer_done); + } + { + bool allocated = false; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc(&exec_ctx, &usr, 1024, set_bool(&allocated)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(allocated); + GPR_ASSERT(reclaimer_done); + } + } + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_free(&exec_ctx, &usr, 1024); + grpc_exec_ctx_finish(&exec_ctx); + } + destroy_user(&usr); + grpc_resource_quota_unref(q); +} + +static void test_one_slice(void) { + gpr_log(GPR_INFO, "** test_one_slice **"); + + grpc_resource_quota *q = grpc_resource_quota_create("test_one_slice"); + grpc_resource_quota_resize(q, 1024); + + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + + grpc_resource_user_slice_allocator alloc; + int num_allocs = 0; + grpc_resource_user_slice_allocator_init(&alloc, &usr, inc_int_cb, + &num_allocs); + + gpr_slice_buffer buffer; + gpr_slice_buffer_init(&buffer); + + { + const int start_allocs = num_allocs; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc_slices(&exec_ctx, &alloc, 1024, 1, &buffer); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(num_allocs == start_allocs + 1); + } + + gpr_slice_buffer_destroy(&buffer); + destroy_user(&usr); + grpc_resource_quota_unref(q); +} + +static void test_one_slice_deleted_late(void) { + gpr_log(GPR_INFO, "** test_one_slice_deleted_late **"); + + grpc_resource_quota *q = + grpc_resource_quota_create("test_one_slice_deleted_late"); + grpc_resource_quota_resize(q, 1024); + + grpc_resource_user usr; + grpc_resource_user_init(&usr, q, "usr"); + + grpc_resource_user_slice_allocator alloc; + int num_allocs = 0; + grpc_resource_user_slice_allocator_init(&alloc, &usr, inc_int_cb, + &num_allocs); + + gpr_slice_buffer buffer; + gpr_slice_buffer_init(&buffer); + + { + const int start_allocs = num_allocs; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_alloc_slices(&exec_ctx, &alloc, 1024, 1, &buffer); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(num_allocs == start_allocs + 1); + } + + bool done = false; + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_shutdown(&exec_ctx, &usr, set_bool(&done)); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(!done); + } + + grpc_resource_quota_unref(q); + gpr_slice_buffer_destroy(&buffer); + GPR_ASSERT(done); + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_resource_user_destroy(&exec_ctx, &usr); + grpc_exec_ctx_finish(&exec_ctx); + } +} + +int main(int argc, char **argv) { + grpc_test_init(argc, argv); + grpc_init(); + test_no_op(); + test_resize_then_destroy(); + test_resource_user_no_op(); + test_instant_alloc_then_free(); + test_instant_alloc_free_pair(); + test_simple_async_alloc(); + test_async_alloc_blocked_by_size(); + test_scavenge(); + test_scavenge_blocked(); + test_blocked_until_scheduled_reclaim(); + test_blocked_until_scheduled_reclaim_and_scavenge(); + test_blocked_until_scheduled_destructive_reclaim(); + test_unused_reclaim_is_cancelled(); + test_benign_reclaim_is_preferred(); + test_multiple_reclaims_can_be_triggered(); + test_resource_user_stays_allocated_until_memory_released(); + test_resource_user_stays_allocated_and_reclaimers_unrun_until_memory_released(); + test_reclaimers_can_be_posted_repeatedly(); + test_one_slice(); + test_one_slice_deleted_late(); + grpc_shutdown(); + return 0; +} diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index 6c6962ed7a2..5fab826fb77 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -114,7 +114,7 @@ void test_succeeds(void) { GPR_ASSERT(getsockname(svr_fd, (struct sockaddr *)addr, (socklen_t *)&resolved_addr.len) == 0); grpc_closure_init(&done, must_succeed, NULL); - grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, + grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, NULL, &resolved_addr, gpr_inf_future(GPR_CLOCK_REALTIME)); /* await the connection */ @@ -164,7 +164,7 @@ void test_fails(void) { /* connect to a broken address */ grpc_closure_init(&done, must_fail, NULL); - grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, + grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, NULL, &resolved_addr, gpr_inf_future(GPR_CLOCK_REALTIME)); gpr_mu_lock(g_mu); diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index d998958744c..29f12a5afb2 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -181,7 +181,10 @@ static void read_test(size_t num_bytes, size_t slice_size) { create_sockets(sv); - ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), slice_size, "test"); + grpc_resource_quota *resource_quota = grpc_resource_quota_create("read_test"); + ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), resource_quota, + slice_size, "test"); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket_partial(sv[0], num_bytes); @@ -228,8 +231,11 @@ static void large_read_test(size_t slice_size) { create_sockets(sv); - ep = grpc_tcp_create(grpc_fd_create(sv[1], "large_read_test"), slice_size, - "test"); + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("large_read_test"); + ep = grpc_tcp_create(grpc_fd_create(sv[1], "large_read_test"), resource_quota, + slice_size, "test"); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket(sv[0]); @@ -364,8 +370,11 @@ static void write_test(size_t num_bytes, size_t slice_size) { create_sockets(sv); - ep = grpc_tcp_create(grpc_fd_create(sv[1], "write_test"), + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("write_test"); + ep = grpc_tcp_create(grpc_fd_create(sv[1], "write_test"), resource_quota, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, "test"); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); state.ep = ep; @@ -428,8 +437,12 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { create_sockets(sv); - ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), slice_size, "test"); + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("release_fd_test"); + ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), resource_quota, + slice_size, "test"); GPR_ASSERT(grpc_tcp_fd(ep) == sv[1] && sv[1] >= 0); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket_partial(sv[0], num_bytes); @@ -450,8 +463,10 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { "pollset_work", grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline))); + gpr_log(GPR_DEBUG, "wakeup: read=%" PRIdPTR " target=%" PRIdPTR, + state.read_bytes, state.target_read_bytes); gpr_mu_unlock(g_mu); - grpc_exec_ctx_finish(&exec_ctx); + grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(g_mu); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); @@ -459,6 +474,7 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { gpr_slice_buffer_destroy(&state.incoming); grpc_tcp_destroy_and_release_fd(&exec_ctx, ep, &fd, &fd_released_cb); + grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(g_mu); while (!fd_released_done) { grpc_pollset_worker *worker = NULL; @@ -466,6 +482,7 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { "pollset_work", grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline))); + gpr_log(GPR_DEBUG, "wakeup: fd_released_done=%d", fd_released_done); } gpr_mu_unlock(g_mu); GPR_ASSERT(fd_released_done == 1); @@ -511,10 +528,13 @@ static grpc_endpoint_test_fixture create_fixture_tcp_socketpair( grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; create_sockets(sv); + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("tcp_posix_test_socketpair"); f.client_ep = grpc_tcp_create(grpc_fd_create(sv[0], "fixture:client"), - slice_size, "test"); + resource_quota, slice_size, "test"); f.server_ep = grpc_tcp_create(grpc_fd_create(sv[1], "fixture:server"), - slice_size, "test"); + resource_quota, slice_size, "test"); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, g_pollset); grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, g_pollset); diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index f839911a843..1b8a39c1beb 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -138,7 +138,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, static void test_no_op(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_tcp_server *s; - GPR_ASSERT(GRPC_ERROR_NONE == grpc_tcp_server_create(NULL, NULL, &s)); + GPR_ASSERT(GRPC_ERROR_NONE == + grpc_tcp_server_create(&exec_ctx, NULL, NULL, &s)); grpc_tcp_server_unref(&exec_ctx, s); grpc_exec_ctx_finish(&exec_ctx); } @@ -146,7 +147,8 @@ static void test_no_op(void) { static void test_no_op_with_start(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_tcp_server *s; - GPR_ASSERT(GRPC_ERROR_NONE == grpc_tcp_server_create(NULL, NULL, &s)); + GPR_ASSERT(GRPC_ERROR_NONE == + grpc_tcp_server_create(&exec_ctx, NULL, NULL, &s)); LOG_TEST("test_no_op_with_start"); grpc_tcp_server_start(&exec_ctx, s, NULL, 0, on_connect, NULL); grpc_tcp_server_unref(&exec_ctx, s); @@ -158,7 +160,8 @@ static void test_no_op_with_port(void) { grpc_resolved_address resolved_addr; struct sockaddr_in *addr = (struct sockaddr_in *)resolved_addr.addr; grpc_tcp_server *s; - GPR_ASSERT(GRPC_ERROR_NONE == grpc_tcp_server_create(NULL, NULL, &s)); + GPR_ASSERT(GRPC_ERROR_NONE == + grpc_tcp_server_create(&exec_ctx, NULL, NULL, &s)); LOG_TEST("test_no_op_with_port"); memset(&resolved_addr, 0, sizeof(resolved_addr)); @@ -178,7 +181,8 @@ static void test_no_op_with_port_and_start(void) { grpc_resolved_address resolved_addr; struct sockaddr_in *addr = (struct sockaddr_in *)resolved_addr.addr; grpc_tcp_server *s; - GPR_ASSERT(GRPC_ERROR_NONE == grpc_tcp_server_create(NULL, NULL, &s)); + GPR_ASSERT(GRPC_ERROR_NONE == + grpc_tcp_server_create(&exec_ctx, NULL, NULL, &s)); LOG_TEST("test_no_op_with_port_and_start"); int port; @@ -241,7 +245,8 @@ static void test_connect(unsigned n) { unsigned svr1_fd_count; int svr1_port; grpc_tcp_server *s; - GPR_ASSERT(GRPC_ERROR_NONE == grpc_tcp_server_create(NULL, NULL, &s)); + GPR_ASSERT(GRPC_ERROR_NONE == + grpc_tcp_server_create(&exec_ctx, NULL, NULL, &s)); unsigned i; server_weak_ref weak_ref; server_weak_ref_init(&weak_ref); diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c index 1d2bf73bb1d..2262fde98d2 100644 --- a/test/core/security/secure_endpoint_test.c +++ b/test/core/security/secure_endpoint_test.c @@ -56,7 +56,10 @@ static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( grpc_endpoint_test_fixture f; grpc_endpoint_pair tcp; - tcp = grpc_iomgr_create_endpoint_pair("fixture", slice_size); + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("secure_endpoint_test"); + tcp = grpc_iomgr_create_endpoint_pair("fixture", resource_quota, slice_size); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); grpc_endpoint_add_to_pollset(&exec_ctx, tcp.client, g_pollset); grpc_endpoint_add_to_pollset(&exec_ctx, tcp.server, g_pollset); diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index f9f723baaa1..f9f46754546 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -120,7 +120,7 @@ void bad_server_thread(void *vargs) { struct sockaddr_storage *addr = (struct sockaddr_storage *)resolved_addr.addr; int port; grpc_tcp_server *s; - grpc_error *error = grpc_tcp_server_create(NULL, NULL, &s); + grpc_error *error = grpc_tcp_server_create(&exec_ctx, NULL, NULL, &s); GPR_ASSERT(error == GRPC_ERROR_NONE); memset(&resolved_addr, 0, sizeof(resolved_addr)); addr->ss_family = AF_INET; diff --git a/test/core/util/mock_endpoint.c b/test/core/util/mock_endpoint.c index 13e0e918fbe..2b041a44841 100644 --- a/test/core/util/mock_endpoint.c +++ b/test/core/util/mock_endpoint.c @@ -33,16 +33,20 @@ #include "test/core/util/mock_endpoint.h" +#include + #include #include typedef struct grpc_mock_endpoint { grpc_endpoint base; gpr_mu mu; + int refs; void (*on_write)(gpr_slice slice); gpr_slice_buffer read_buffer; gpr_slice_buffer *on_read_out; grpc_closure *on_read; + grpc_resource_user resource_user; } grpc_mock_endpoint; static void me_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, @@ -74,6 +78,24 @@ static void me_add_to_pollset(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, static void me_add_to_pollset_set(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_pollset_set *pollset) {} +static void unref(grpc_exec_ctx *exec_ctx, grpc_mock_endpoint *m) { + gpr_mu_lock(&m->mu); + if (0 == --m->refs) { + gpr_mu_unlock(&m->mu); + gpr_slice_buffer_destroy(&m->read_buffer); + grpc_resource_user_destroy(exec_ctx, &m->resource_user); + gpr_free(m); + } else { + gpr_mu_unlock(&m->mu); + } +} + +static void me_finish_shutdown(grpc_exec_ctx *exec_ctx, void *me, + grpc_error *error) { + grpc_mock_endpoint *m = me; + unref(exec_ctx, m); +} + static void me_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_mock_endpoint *m = (grpc_mock_endpoint *)ep; gpr_mu_lock(&m->mu); @@ -82,19 +104,25 @@ static void me_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { GRPC_ERROR_CREATE("Endpoint Shutdown"), NULL); m->on_read = NULL; } + grpc_resource_user_shutdown(exec_ctx, &m->resource_user, + grpc_closure_create(me_finish_shutdown, m)); gpr_mu_unlock(&m->mu); } static void me_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_mock_endpoint *m = (grpc_mock_endpoint *)ep; - gpr_slice_buffer_destroy(&m->read_buffer); - gpr_free(m); + unref(exec_ctx, m); } static char *me_get_peer(grpc_endpoint *ep) { return gpr_strdup("fake:mock_endpoint"); } +static grpc_resource_user *me_get_resource_user(grpc_endpoint *ep) { + grpc_mock_endpoint *m = (grpc_mock_endpoint *)ep; + return &m->resource_user; +} + static grpc_workqueue *me_get_workqueue(grpc_endpoint *ep) { return NULL; } static const grpc_endpoint_vtable vtable = { @@ -105,12 +133,19 @@ static const grpc_endpoint_vtable vtable = { me_add_to_pollset_set, me_shutdown, me_destroy, + me_get_resource_user, me_get_peer, }; -grpc_endpoint *grpc_mock_endpoint_create(void (*on_write)(gpr_slice slice)) { +grpc_endpoint *grpc_mock_endpoint_create(void (*on_write)(gpr_slice slice), + grpc_resource_quota *resource_quota) { grpc_mock_endpoint *m = gpr_malloc(sizeof(*m)); m->base.vtable = &vtable; + m->refs = 2; + char *name; + gpr_asprintf(&name, "mock_endpoint_%" PRIxPTR, (intptr_t)m); + grpc_resource_user_init(&m->resource_user, resource_quota, name); + gpr_free(name); gpr_slice_buffer_init(&m->read_buffer); gpr_mu_init(&m->mu); m->on_write = on_write; diff --git a/test/core/util/mock_endpoint.h b/test/core/util/mock_endpoint.h index 051af9866b6..b3a464ca01c 100644 --- a/test/core/util/mock_endpoint.h +++ b/test/core/util/mock_endpoint.h @@ -36,7 +36,8 @@ #include "src/core/lib/iomgr/endpoint.h" -grpc_endpoint *grpc_mock_endpoint_create(void (*on_write)(gpr_slice slice)); +grpc_endpoint *grpc_mock_endpoint_create(void (*on_write)(gpr_slice slice), + grpc_resource_quota *resource_quota); void grpc_mock_endpoint_put_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *mock_endpoint, gpr_slice slice); diff --git a/test/core/util/passthru_endpoint.c b/test/core/util/passthru_endpoint.c index 7ed9e97bd6a..ee6ef7da601 100644 --- a/test/core/util/passthru_endpoint.c +++ b/test/core/util/passthru_endpoint.c @@ -33,6 +33,8 @@ #include "test/core/util/passthru_endpoint.h" +#include + #include #include @@ -44,6 +46,7 @@ typedef struct { gpr_slice_buffer read_buffer; gpr_slice_buffer *on_read_out; grpc_closure *on_read; + grpc_resource_user resource_user; } half; struct passthru_endpoint { @@ -122,7 +125,8 @@ static void me_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { gpr_mu_unlock(&m->parent->mu); } -static void me_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { +static void me_really_destroy(grpc_exec_ctx *exec_ctx, void *ep, + grpc_error *error) { passthru_endpoint *p = ((half *)ep)->parent; gpr_mu_lock(&p->mu); if (0 == --p->halves) { @@ -136,12 +140,23 @@ static void me_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { } } +static void me_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { + half *m = (half *)ep; + grpc_resource_user_shutdown(exec_ctx, &m->resource_user, + grpc_closure_create(me_really_destroy, m)); +} + static char *me_get_peer(grpc_endpoint *ep) { return gpr_strdup("fake:mock_endpoint"); } static grpc_workqueue *me_get_workqueue(grpc_endpoint *ep) { return NULL; } +static grpc_resource_user *me_get_resource_user(grpc_endpoint *ep) { + half *m = (half *)ep; + return &m->resource_user; +} + static const grpc_endpoint_vtable vtable = { me_read, me_write, @@ -150,23 +165,32 @@ static const grpc_endpoint_vtable vtable = { me_add_to_pollset_set, me_shutdown, me_destroy, + me_get_resource_user, me_get_peer, }; -static void half_init(half *m, passthru_endpoint *parent) { +static void half_init(half *m, passthru_endpoint *parent, + grpc_resource_quota *resource_quota, + const char *half_name) { m->base.vtable = &vtable; m->parent = parent; gpr_slice_buffer_init(&m->read_buffer); m->on_read = NULL; + char *name; + gpr_asprintf(&name, "passthru_endpoint_%s_%" PRIxPTR, half_name, + (intptr_t)parent); + grpc_resource_user_init(&m->resource_user, resource_quota, name); + gpr_free(name); } void grpc_passthru_endpoint_create(grpc_endpoint **client, - grpc_endpoint **server) { + grpc_endpoint **server, + grpc_resource_quota *resource_quota) { passthru_endpoint *m = gpr_malloc(sizeof(*m)); m->halves = 2; m->shutdown = 0; - half_init(&m->client, m); - half_init(&m->server, m); + half_init(&m->client, m, resource_quota, "client"); + half_init(&m->server, m, resource_quota, "server"); gpr_mu_init(&m->mu); *client = &m->client.base; *server = &m->server.base; diff --git a/test/core/util/passthru_endpoint.h b/test/core/util/passthru_endpoint.h index aa1d3a17637..b81ac5571c2 100644 --- a/test/core/util/passthru_endpoint.h +++ b/test/core/util/passthru_endpoint.h @@ -37,6 +37,7 @@ #include "src/core/lib/iomgr/endpoint.h" void grpc_passthru_endpoint_create(grpc_endpoint **client, - grpc_endpoint **server); + grpc_endpoint **server, + grpc_resource_quota *resource_quota); #endif diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index a5c8c496502..b2342feeb40 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -99,9 +99,12 @@ void grpc_free_port_using_server(char *server, int port) { req.http.path = path; grpc_httpcli_context_init(&context); - grpc_httpcli_get(&exec_ctx, &context, &pr.pops, &req, + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("port_server_client/free"); + grpc_httpcli_get(&exec_ctx, &context, &pr.pops, resource_quota, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), grpc_closure_create(freed_port_from_server, &pr), &rsp); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); gpr_mu_lock(pr.mu); while (!pr.done) { grpc_pollset_worker *worker = NULL; @@ -167,10 +170,13 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, req.http.path = "/get"; grpc_http_response_destroy(&pr->response); memset(&pr->response, 0, sizeof(pr->response)); - grpc_httpcli_get(exec_ctx, pr->ctx, &pr->pops, &req, + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("port_server_client/pick_retry"); + grpc_httpcli_get(exec_ctx, pr->ctx, &pr->pops, resource_quota, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), grpc_closure_create(got_port_from_server, pr), &pr->response); + grpc_resource_quota_internal_unref(exec_ctx, resource_quota); return; } GPR_ASSERT(response); @@ -211,9 +217,13 @@ int grpc_pick_port_using_server(char *server) { req.http.path = "/get"; grpc_httpcli_context_init(&context); - grpc_httpcli_get( - &exec_ctx, &context, &pr.pops, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), - grpc_closure_create(got_port_from_server, &pr), &pr.response); + grpc_resource_quota *resource_quota = + grpc_resource_quota_create("port_server_client/pick"); + grpc_httpcli_get(&exec_ctx, &context, &pr.pops, resource_quota, &req, + GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), + grpc_closure_create(got_port_from_server, &pr), + &pr.response); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); grpc_exec_ctx_finish(&exec_ctx); gpr_mu_lock(pr.mu); while (pr.port == -1) { diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c index 9ac97413d45..16df91d9688 100644 --- a/test/core/util/test_tcp_server.c +++ b/test/core/util/test_tcp_server.c @@ -75,8 +75,8 @@ void test_tcp_server_start(test_tcp_server *server, int port) { addr->sin_port = htons((uint16_t)port); memset(&addr->sin_addr, 0, sizeof(addr->sin_addr)); - grpc_error *error = grpc_tcp_server_create(&server->shutdown_complete, NULL, - &server->tcp_server); + grpc_error *error = grpc_tcp_server_create( + &exec_ctx, &server->shutdown_complete, NULL, &server->tcp_server); GPR_ASSERT(error == GRPC_ERROR_NONE); error = grpc_tcp_server_add_port(server->tcp_server, &resolved_addr, &port_added); diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index 954cf2d80bc..43b7d44255e 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -240,6 +241,7 @@ class End2endTest : public ::testing::TestWithParam { server_address_ << "127.0.0.1:" << port; // Setup server ServerBuilder builder; + ConfigureServerBuilder(&builder); auto server_creds = GetServerCredentials(GetParam().credentials_type); if (GetParam().credentials_type != kInsecureCredentialsType) { server_creds->SetAuthMetadataProcessor(processor); @@ -247,8 +249,6 @@ class End2endTest : public ::testing::TestWithParam { builder.AddListeningPort(server_address_.str(), server_creds); builder.RegisterService(&service_); builder.RegisterService("foo.test.youtube.com", &special_service_); - builder.SetMaxMessageSize( - kMaxMessageSize_); // For testing max message size. builder.RegisterService(&dup_pkg_service_); builder.SetSyncServerOption(ServerBuilder::SyncServerOption::NUM_CQS, 4); @@ -259,6 +259,11 @@ class End2endTest : public ::testing::TestWithParam { is_server_started_ = true; } + virtual void ConfigureServerBuilder(ServerBuilder* builder) { + builder->SetMaxMessageSize( + kMaxMessageSize_); // For testing max message size. + } + void ResetChannel() { if (!is_server_started_) { StartServer(std::shared_ptr()); @@ -1486,6 +1491,32 @@ TEST_P(SecureEnd2endTest, ClientAuthContext) { } } +class ResourceQuotaEnd2endTest : public End2endTest { + public: + ResourceQuotaEnd2endTest() + : server_resource_quota_("server_resource_quota") {} + + virtual void ConfigureServerBuilder(ServerBuilder* builder) GRPC_OVERRIDE { + builder->SetResourceQuota(server_resource_quota_); + } + + private: + ResourceQuota server_resource_quota_; +}; + +TEST_P(ResourceQuotaEnd2endTest, SimpleRequest) { + ResetStub(); + + EchoRequest request; + EchoResponse response; + request.set_message("Hello"); + + ClientContext context; + Status s = stub_->Echo(&context, request, &response); + EXPECT_EQ(response.message(), request.message()); + EXPECT_TRUE(s.ok()); +} + std::vector CreateTestScenarios(bool use_proxy, bool test_insecure, bool test_secure) { @@ -1523,6 +1554,10 @@ INSTANTIATE_TEST_CASE_P(SecureEnd2end, SecureEnd2endTest, ::testing::ValuesIn(CreateTestScenarios(false, false, true))); +INSTANTIATE_TEST_CASE_P(ResourceQuotaEnd2end, ResourceQuotaEnd2endTest, + ::testing::ValuesIn(CreateTestScenarios(false, true, + true))); + } // namespace } // namespace testing } // namespace grpc diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc index 4197ba8bab2..c58910abc3f 100644 --- a/test/cpp/interop/client.cc +++ b/test/cpp/interop/client.cc @@ -54,33 +54,35 @@ DEFINE_int32(server_port, 0, "Server port."); DEFINE_string(server_host, "127.0.0.1", "Server host to connect to"); DEFINE_string(server_host_override, "foo.test.google.fr", "Override the server host which is sent in HTTP header"); -DEFINE_string(test_case, "large_unary", - "Configure different test cases. Valid options are:\n\n" - "all : all test cases;\n" - "cancel_after_begin : cancel stream after starting it;\n" - "cancel_after_first_response: cancel on first response;\n" - "client_compressed_streaming : compressed request streaming with " - "client_compressed_unary : single compressed request;\n" - "client_streaming : request streaming with single response;\n" - "compute_engine_creds: large_unary with compute engine auth;\n" - "custom_metadata: server will echo custom metadata;\n" - "empty_stream : bi-di stream with no request/response;\n" - "empty_unary : empty (zero bytes) request and response;\n" - "half_duplex : half-duplex streaming;\n" - "jwt_token_creds: large_unary with JWT token auth;\n" - "large_unary : single request and (large) response;\n" - "oauth2_auth_token: raw oauth2 access token auth;\n" - "per_rpc_creds: raw oauth2 access token on a single rpc;\n" - "ping_pong : full-duplex streaming;\n" - "response streaming;\n" - "server_compressed_streaming : single request with compressed " - "server_compressed_unary : single compressed response;\n" - "server_streaming : single request with response streaming;\n" - "slow_consumer : single request with response streaming with " - "slow client consumer;\n" - "status_code_and_message: verify status code & message;\n" - "timeout_on_sleeping_server: deadline exceeds on stream;\n" - "unimplemented_method: client calls an unimplemented method;\n"); +DEFINE_string( + test_case, "large_unary", + "Configure different test cases. Valid options are:\n\n" + "all : all test cases;\n" + "cancel_after_begin : cancel stream after starting it;\n" + "cancel_after_first_response: cancel on first response;\n" + "client_compressed_streaming : compressed request streaming with " + "client_compressed_unary : single compressed request;\n" + "client_streaming : request streaming with single response;\n" + "compute_engine_creds: large_unary with compute engine auth;\n" + "custom_metadata: server will echo custom metadata;\n" + "empty_stream : bi-di stream with no request/response;\n" + "empty_unary : empty (zero bytes) request and response;\n" + "half_duplex : half-duplex streaming;\n" + "jwt_token_creds: large_unary with JWT token auth;\n" + "large_unary : single request and (large) response;\n" + "oauth2_auth_token: raw oauth2 access token auth;\n" + "per_rpc_creds: raw oauth2 access token on a single rpc;\n" + "ping_pong : full-duplex streaming;\n" + "response streaming;\n" + "server_compressed_streaming : single request with compressed " + "server_compressed_unary : single compressed response;\n" + "server_streaming : single request with response streaming;\n" + "slow_consumer : single request with response streaming with " + "slow client consumer;\n" + "status_code_and_message: verify status code & message;\n" + "timeout_on_sleeping_server: deadline exceeds on stream;\n" + "unimplemented_method: client calls an unimplemented method;\n" + "unimplemented_service: client calls an unimplemented service;\n"); DEFINE_string(default_service_account, "", "Email of GCE default service account"); DEFINE_string(service_account_key_file, "", @@ -152,6 +154,8 @@ int main(int argc, char** argv) { client.DoCustomMetadata(); } else if (FLAGS_test_case == "unimplemented_method") { client.DoUnimplementedMethod(); + } else if (FLAGS_test_case == "unimplemented_service") { + client.DoUnimplementedService(); } else if (FLAGS_test_case == "cacheable_unary") { client.DoCacheableUnary(); } else if (FLAGS_test_case == "all") { @@ -172,6 +176,7 @@ int main(int argc, char** argv) { client.DoStatusWithMessage(); client.DoCustomMetadata(); client.DoUnimplementedMethod(); + client.DoUnimplementedService(); client.DoCacheableUnary(); // service_account_creds and jwt_token_creds can only run with ssl. if (FLAGS_use_tls) { @@ -207,7 +212,8 @@ int main(int argc, char** argv) { "server_streaming", "status_code_and_message", "timeout_on_sleeping_server", - "unimplemented_method"}; + "unimplemented_method", + "unimplemented_service"}; char* joined_testcases = gpr_strjoin_sep(testcases, GPR_ARRAY_SIZE(testcases), "\n", NULL); diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc index 1668589cc4a..d1242627ef3 100644 --- a/test/cpp/interop/interop_client.cc +++ b/test/cpp/interop/interop_client.cc @@ -107,6 +107,11 @@ TestService::Stub* InteropClient::ServiceStub::Get() { return stub_.get(); } +UnimplementedService::Stub* +InteropClient::ServiceStub::GetUnimplementedServiceStub() { + return UnimplementedService::NewStub(channel_).get(); +} + void InteropClient::ServiceStub::Reset(std::shared_ptr channel) { channel_ = channel; @@ -162,8 +167,8 @@ bool InteropClient::AssertStatusCode(const Status& s, bool InteropClient::DoEmpty() { gpr_log(GPR_DEBUG, "Sending an empty rpc..."); - Empty request = Empty::default_instance(); - Empty response = Empty::default_instance(); + Empty request; + Empty response; ClientContext context; Status s = serviceStub_.Get()->EmptyCall(&context, request, &response); @@ -1002,11 +1007,30 @@ bool InteropClient::DoCustomMetadata() { return true; } +bool InteropClient::DoUnimplementedService() { + gpr_log(GPR_DEBUG, "Sending a request for an unimplemented service..."); + + Empty request; + Empty response; + ClientContext context; + + UnimplementedService::Stub* stub = serviceStub_.GetUnimplementedServiceStub(); + + Status s = stub->UnimplementedCall(&context, request, &response); + + if (!AssertStatusCode(s, StatusCode::UNIMPLEMENTED)) { + return false; + } + + gpr_log(GPR_DEBUG, "unimplemented service done."); + return true; +} + bool InteropClient::DoUnimplementedMethod() { gpr_log(GPR_DEBUG, "Sending a request for an unimplemented rpc..."); - Empty request = Empty::default_instance(); - Empty response = Empty::default_instance(); + Empty request; + Empty response; ClientContext context; Status s = diff --git a/test/cpp/interop/interop_client.h b/test/cpp/interop/interop_client.h index 0a96e7734d8..7ec7ebee209 100644 --- a/test/cpp/interop/interop_client.h +++ b/test/cpp/interop/interop_client.h @@ -80,6 +80,7 @@ class InteropClient { bool DoStatusWithMessage(); bool DoCustomMetadata(); bool DoUnimplementedMethod(); + bool DoUnimplementedService(); bool DoCacheableUnary(); // Auth tests. // username is a string containing the user email @@ -100,6 +101,7 @@ class InteropClient { ServiceStub(std::shared_ptr channel, bool new_stub_every_call); TestService::Stub* Get(); + UnimplementedService::Stub* GetUnimplementedServiceStub(); void Reset(std::shared_ptr channel); diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 5fb87b27825..9983c8a7b08 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -36,6 +36,7 @@ #include #include +#include #include #include @@ -114,19 +115,37 @@ class ClientRequestCreator { class HistogramEntry GRPC_FINAL { public: - HistogramEntry() : used_(false) {} - bool used() const { return used_; } + HistogramEntry() : value_used_(false), status_used_(false) {} + bool value_used() const { return value_used_; } double value() const { return value_; } void set_value(double v) { - used_ = true; + value_used_ = true; value_ = v; } + bool status_used() const { return status_used_; } + int status() const { return status_; } + void set_status(int status) { + status_used_ = true; + status_ = status; + } private: - bool used_; + bool value_used_; double value_; + bool status_used_; + int status_; }; +typedef std::unordered_map StatusHistogram; + +inline void MergeStatusHistogram(const StatusHistogram& from, + StatusHistogram* to) { + for (StatusHistogram::const_iterator it = from.begin(); it != from.end(); + ++it) { + (*to)[it->first] += it->second; + } +} + class Client { public: Client() @@ -139,6 +158,7 @@ class Client { ClientStats Mark(bool reset) { Histogram latencies; + StatusHistogram statuses; UsageTimer::Result timer_result; MaybeStartRequests(); @@ -146,27 +166,36 @@ class Client { // avoid std::vector for old compilers that expect a copy constructor if (reset) { Histogram* to_merge = new Histogram[threads_.size()]; + StatusHistogram* to_merge_status = new StatusHistogram[threads_.size()]; + for (size_t i = 0; i < threads_.size(); i++) { - threads_[i]->BeginSwap(&to_merge[i]); + threads_[i]->BeginSwap(&to_merge[i], &to_merge_status[i]); } std::unique_ptr timer(new UsageTimer); timer_.swap(timer); for (size_t i = 0; i < threads_.size(); i++) { - threads_[i]->EndSwap(); latencies.Merge(to_merge[i]); + MergeStatusHistogram(to_merge_status[i], &statuses); } delete[] to_merge; + delete[] to_merge_status; timer_result = timer->Mark(); } else { // merge snapshots of each thread histogram for (size_t i = 0; i < threads_.size(); i++) { - threads_[i]->MergeStatsInto(&latencies); + threads_[i]->MergeStatsInto(&latencies, &statuses); } timer_result = timer_->Mark(); } ClientStats stats; latencies.FillProto(stats.mutable_latencies()); + for (StatusHistogram::const_iterator it = statuses.begin(); + it != statuses.end(); ++it) { + RequestResultCount* rrc = stats.add_request_results(); + rrc->set_status_code(it->first); + rrc->set_count(it->second); + } stats.set_time_elapsed(timer_result.wall); stats.set_time_system(timer_result.system); stats.set_time_user(timer_result.user); @@ -258,16 +287,16 @@ class Client { ~Thread() { impl_.join(); } - void BeginSwap(Histogram* n) { + void BeginSwap(Histogram* n, StatusHistogram* s) { std::lock_guard g(mu_); n->Swap(&histogram_); + s->swap(statuses_); } - void EndSwap() {} - - void MergeStatsInto(Histogram* hist) { + void MergeStatsInto(Histogram* hist, StatusHistogram* s) { std::unique_lock g(mu_); hist->Merge(histogram_); + MergeStatusHistogram(statuses_, s); } private: @@ -288,9 +317,12 @@ class Client { const bool thread_still_ok = client_->ThreadFunc(&entry, idx_); // lock, update histogram if needed and see if we're done std::lock_guard g(mu_); - if (entry.used()) { + if (entry.value_used()) { histogram_.Add(entry.value()); } + if (entry.status_used()) { + statuses_[entry.status()]++; + } if (!thread_still_ok) { gpr_log(GPR_ERROR, "Finishing client thread due to RPC error"); } @@ -304,6 +336,7 @@ class Client { std::mutex mu_; Histogram histogram_; + StatusHistogram statuses_; Client* client_; const size_t idx_; std::thread impl_; diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 081114859ce..4d36a6ba423 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -83,7 +83,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, CompletionQueue*)> start_req, - std::function on_done) + std::function on_done) : context_(), stub_(stub), cq_(nullptr), @@ -113,7 +113,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { return true; case State::RESP_DONE: entry->set_value((UsageTimer::Now() - start_) * 1e9); - callback_(status_, &response_); + callback_(status_, &response_, entry); next_state_ = State::INVALID; return false; default: @@ -135,7 +135,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { ResponseType response_; enum State { INVALID, READY, RESP_DONE }; State next_state_; - std::function callback_; + std::function callback_; std::function next_issue_; std::function>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, @@ -290,7 +290,10 @@ class AsyncUnaryClient GRPC_FINAL ~AsyncUnaryClient() GRPC_OVERRIDE {} private: - static void CheckDone(grpc::Status s, SimpleResponse* response) {} + static void CheckDone(grpc::Status s, SimpleResponse* response, + HistogramEntry* entry) { + entry->set_status(s.error_code()); + } static std::unique_ptr> StartReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, const SimpleRequest& request, CompletionQueue* cq) { diff --git a/test/cpp/qps/client_sync.cc b/test/cpp/qps/client_sync.cc index 0ccf4e270b6..f61e80d76be 100644 --- a/test/cpp/qps/client_sync.cc +++ b/test/cpp/qps/client_sync.cc @@ -130,11 +130,8 @@ class SynchronousUnaryClient GRPC_FINAL : public SynchronousClient { grpc::Status s = stub->UnaryCall(&context, request_, &responses_[thread_idx]); entry->set_value((UsageTimer::Now() - start) * 1e9); - if (!s.ok()) { - gpr_log(GPR_ERROR, "RPC error: %d: %s", s.error_code(), - s.error_message().c_str()); - } - return s.ok(); + entry->set_status(s.error_code()); + return true; } }; diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 6c675e34830..a440341ccf4 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -132,7 +132,8 @@ static void postprocess_scenario_result(ScenarioResult* result) { Histogram histogram; histogram.MergeProto(result->latencies()); - auto qps = histogram.Count() / average(result->client_stats(), WallTime); + auto time_estimate = average(result->client_stats(), WallTime); + auto qps = histogram.Count() / time_estimate; auto qps_per_server_core = qps / sum(result->server_cores(), Cores); result->mutable_summary()->set_qps(qps); @@ -157,6 +158,23 @@ static void postprocess_scenario_result(ScenarioResult* result) { result->mutable_summary()->set_server_user_time(server_user_time); result->mutable_summary()->set_client_system_time(client_system_time); result->mutable_summary()->set_client_user_time(client_user_time); + + if (result->request_results_size() > 0) { + int64_t successes = 0; + int64_t failures = 0; + for (int i = 0; i < result->request_results_size(); i++) { + RequestResultCount rrc = result->request_results(i); + if (rrc.status_code() == 0) { + successes += rrc.count(); + } else { + failures += rrc.count(); + } + } + result->mutable_summary()->set_successful_requests_per_second( + successes / time_estimate); + result->mutable_summary()->set_failed_requests_per_second(failures / + time_estimate); + } } // Namespace for classes and functions used only in RunScenario @@ -444,6 +462,7 @@ std::unique_ptr RunScenario( // Finish a run std::unique_ptr result(new ScenarioResult); Histogram merged_latencies; + std::unordered_map merged_statuses; gpr_log(GPR_INFO, "Finishing clients"); for (size_t i = 0; i < num_clients; i++) { @@ -462,6 +481,10 @@ std::unique_ptr RunScenario( gpr_log(GPR_INFO, "Received final status from client %zu", i); const auto& stats = client_status.stats(); merged_latencies.MergeProto(stats.latencies()); + for (int i = 0; i < stats.request_results_size(); i++) { + merged_statuses[stats.request_results(i).status_code()] += + stats.request_results(i).count(); + } result->add_client_stats()->CopyFrom(stats); // That final status should be the last message on the client stream GPR_ASSERT(!client->stream->Read(&client_status)); @@ -481,6 +504,12 @@ std::unique_ptr RunScenario( delete[] clients; merged_latencies.FillProto(result->mutable_latencies()); + for (std::unordered_map::iterator it = merged_statuses.begin(); + it != merged_statuses.end(); ++it) { + RequestResultCount* rrc = result->add_request_results(); + rrc->set_status_code(it->first); + rrc->set_count(it->second); + } gpr_log(GPR_INFO, "Finishing servers"); for (size_t i = 0; i < num_servers; i++) { diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 2ec7d8676c2..41617e968a5 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -73,6 +73,12 @@ void CompositeReporter::ReportTimes(const ScenarioResult& result) { void GprLogReporter::ReportQPS(const ScenarioResult& result) { gpr_log(GPR_INFO, "QPS: %.1f", result.summary().qps()); + if (result.summary().failed_requests_per_second() > 0) { + gpr_log(GPR_INFO, "failed requests/second: %.1f", + result.summary().failed_requests_per_second()); + gpr_log(GPR_INFO, "successful requests/second: %.1f", + result.summary().successful_requests_per_second()); + } } void GprLogReporter::ReportQPSPerCore(const ScenarioResult& result) { diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index 2fcc64819b3..bc4c896d83a 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -95,6 +96,11 @@ class AsyncQpsServerTest GRPC_FINAL : public grpc::testing::Server { srv_cqs_.emplace_back(builder.AddCompletionQueue()); } + if (config.resource_quota_size() > 0) { + builder.SetResourceQuota(ResourceQuota("AsyncQpsServerTest") + .Resize(config.resource_quota_size())); + } + server_ = builder.BuildAndStart(); using namespace std::placeholders; diff --git a/test/cpp/qps/server_sync.cc b/test/cpp/qps/server_sync.cc index 0caed0ab49e..07f48e26447 100644 --- a/test/cpp/qps/server_sync.cc +++ b/test/cpp/qps/server_sync.cc @@ -31,6 +31,7 @@ * */ +#include #include #include #include @@ -91,6 +92,11 @@ class SynchronousServer GRPC_FINAL : public grpc::testing::Server { Server::CreateServerCredentials(config)); gpr_free(server_address); + if (config.resource_quota_size() > 0) { + builder.SetResourceQuota(ResourceQuota("AsyncQpsServerTest") + .Resize(config.resource_quota_size())); + } + builder.RegisterService(&service_); impl_ = builder.BuildAndStart(); diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 4b0be37e096..00f970a4cbd 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -787,6 +787,7 @@ include/grpc++/impl/sync_no_cxx11.h \ include/grpc++/impl/thd.h \ include/grpc++/impl/thd_cxx11.h \ include/grpc++/impl/thd_no_cxx11.h \ +include/grpc++/resource_quota.h \ include/grpc++/security/auth_context.h \ include/grpc++/security/auth_metadata_processor.h \ include/grpc++/security/credentials.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 7a95526adbe..6c2b475ed0a 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -787,6 +787,7 @@ include/grpc++/impl/sync_no_cxx11.h \ include/grpc++/impl/thd.h \ include/grpc++/impl/thd_cxx11.h \ include/grpc++/impl/thd_no_cxx11.h \ +include/grpc++/resource_quota.h \ include/grpc++/security/auth_context.h \ include/grpc++/security/auth_metadata_processor.h \ include/grpc++/security/credentials.h \ @@ -883,6 +884,7 @@ src/cpp/common/channel_arguments.cc \ src/cpp/common/channel_filter.cc \ src/cpp/common/completion_queue_cc.cc \ src/cpp/common/core_codegen.cc \ +src/cpp/common/resource_quota_cc.cc \ src/cpp/common/rpc_method.cc \ src/cpp/server/async_generic_service.cc \ src/cpp/server/create_default_thread_pool.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 8db1a79be07..e83298766fd 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -828,6 +828,7 @@ src/core/lib/iomgr/pollset_uv.h \ src/core/lib/iomgr/pollset_windows.h \ src/core/lib/iomgr/port.h \ src/core/lib/iomgr/resolve_address.h \ +src/core/lib/iomgr/resource_quota.h \ src/core/lib/iomgr/sockaddr.h \ src/core/lib/iomgr/sockaddr_posix.h \ src/core/lib/iomgr/sockaddr_utils.h \ @@ -836,6 +837,7 @@ src/core/lib/iomgr/socket_utils.h \ src/core/lib/iomgr/socket_utils_posix.h \ src/core/lib/iomgr/socket_windows.h \ src/core/lib/iomgr/tcp_client.h \ +src/core/lib/iomgr/tcp_client_posix.h \ src/core/lib/iomgr/tcp_posix.h \ src/core/lib/iomgr/tcp_server.h \ src/core/lib/iomgr/tcp_uv.h \ @@ -1003,6 +1005,7 @@ src/core/lib/iomgr/pollset_windows.c \ src/core/lib/iomgr/resolve_address_posix.c \ src/core/lib/iomgr/resolve_address_uv.c \ src/core/lib/iomgr/resolve_address_windows.c \ +src/core/lib/iomgr/resource_quota.c \ src/core/lib/iomgr/sockaddr_utils.c \ src/core/lib/iomgr/socket_utils_common_posix.c \ src/core/lib/iomgr/socket_utils_linux.c \ diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index d36f963a7c6..65d34e17e1c 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -109,8 +109,8 @@ class PythonArtifact: # TODO(atash) get better platform-detection support in core so we don't # need to do this manually... environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' - environ['BUILD_HEALTH_CHECKING'] = 'TRUE' - environ['BUILD_MANYLINUX_WHEEL'] = 'TRUE' + environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' + environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, 'tools/run_tests/build_artifact_python.sh', diff --git a/tools/run_tests/build_artifact_node.bat b/tools/run_tests/build_artifact_node.bat index c5bd726db7e..57d55ef19ec 100644 --- a/tools/run_tests/build_artifact_node.bat +++ b/tools/run_tests/build_artifact_node.bat @@ -52,4 +52,4 @@ if %errorlevel% neq 0 exit /b %errorlevel% goto :EOF :error -exit /b 1 \ No newline at end of file +exit /b 1 diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 9fed7c5028a..2a1d41fd686 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -66,7 +66,7 @@ ${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py sdist # Build gRPC tools package binary distribution ${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel -if [ "$BUILD_MANYLINUX_WHEEL" != "" ] +if [ "$GRPC_BUILD_MANYLINUX_WHEEL" != "" ] then for wheel in dist/*.whl; do ${AUDITWHEEL} repair $wheel -w "$ARTIFACT_DIR" @@ -82,16 +82,21 @@ fi # Wheels are not supported by setup_requires/dependency_links, so we # manually install the dependency. Note we should only do this if we # are in a docker image or in a virtualenv. -if [ "$BUILD_HEALTH_CHECKING" != "" ] +if [ "$GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS" != "" ] then ${PIP} install -rrequirements.txt ${PIP} install grpcio --no-index --find-links "file://$ARTIFACT_DIR/" ${PIP} install grpcio-tools --no-index --find-links "file://$ARTIFACT_DIR/" - # Build gRPC health check source distribution + # Build gRPC health-checking source distribution ${SETARCH_CMD} ${PYTHON} src/python/grpcio_health_checking/setup.py \ preprocess build_package_protos sdist cp -r src/python/grpcio_health_checking/dist/* "$ARTIFACT_DIR" + + # Build gRPC reflection source distribution + ${SETARCH_CMD} ${PYTHON} src/python/grpcio_reflection/setup.py \ + preprocess build_package_protos sdist + cp -r src/python/grpcio_reflection/dist/* "$ARTIFACT_DIR" fi cp -r dist/* "$ARTIFACT_DIR" diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index 54e2fe53478..fb884ad1668 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -180,9 +180,18 @@ pip_install_dir $ROOT/tools/distrib/python/grpcio_tools # TODO(atash) figure out namespace packages and grpcio-tools and auditwheel # etc... pip_install_dir $ROOT + +# Build/install health checking $VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py preprocess $VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py build_package_protos pip_install_dir $ROOT/src/python/grpcio_health_checking + +# Build/install reflection +$VENV_PYTHON $ROOT/src/python/grpcio_reflection/setup.py preprocess +$VENV_PYTHON $ROOT/src/python/grpcio_reflection/setup.py build_package_protos +pip_install_dir $ROOT/src/python/grpcio_reflection + +# Build/install tests $VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py preprocess $VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_package_protos pip_install_dir $ROOT/src/python/grpcio_tests diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 4a4ce4537de..4e4c16642ba 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -114,7 +114,8 @@ def _ping_pong_scenario(name, rpc_type, warmup_seconds=WARMUP_SECONDS, categories=DEFAULT_CATEGORIES, channels=None, - outstanding=None): + outstanding=None, + resource_quota_size=None): """Creates a basic ping pong scenario.""" scenario = { 'name': name, @@ -141,6 +142,8 @@ def _ping_pong_scenario(name, rpc_type, 'warmup_seconds': warmup_seconds, 'benchmark_seconds': BENCHMARK_SECONDS } + if resource_quota_size: + scenario['server_config']['resource_quota_size'] = resource_quota_size if use_generic_payload: if server_type != 'ASYNC_GENERIC_SERVER': raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.') @@ -257,6 +260,16 @@ class CXXLanguage: secure=secure, categories=smoketest_categories+[SCALABLE]) + yield _ping_pong_scenario( + 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), + rpc_type=rpc_type.upper(), + client_type='%s_CLIENT' % synchronicity.upper(), + server_type='%s_SERVER' % synchronicity.upper(), + unconstrained_client=synchronicity, + secure=secure, + categories=smoketest_categories+[SCALABLE], + resource_quota_size=500*1024) + for channels in geometric_progression(1, 20000, math.sqrt(10)): for outstanding in geometric_progression(1, 200000, math.sqrt(10)): if synchronicity == 'sync' and outstanding > 1200: continue diff --git a/tools/run_tests/pre_build_node.bat b/tools/run_tests/pre_build_node.bat index a29456f9ed9..addb01a2a4d 100644 --- a/tools/run_tests/pre_build_node.bat +++ b/tools/run_tests/pre_build_node.bat @@ -29,6 +29,5 @@ set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm -@rem Expire cache after 1 week -call npm update --cache-min 604800 - +@rem Expire cache after 1 day +call npm update --cache-min 86400 diff --git a/tools/run_tests/pre_build_node.sh b/tools/run_tests/pre_build_node.sh index 4879e7ad9bd..e63be9da523 100755 --- a/tools/run_tests/pre_build_node.sh +++ b/tools/run_tests/pre_build_node.sh @@ -37,8 +37,8 @@ set -ex export GRPC_CONFIG=${CONFIG:-opt} -# Expire cache after 1 week -npm update --cache-min 604800 +# Expire cache after 1 day +npm update --cache-min 86400 npm install node-gyp-install ./node_modules/.bin/node-gyp-install diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py index efe5dc999d1..3e18f365102 100644 --- a/tools/run_tests/report_utils.py +++ b/tools/run_tests/report_utils.py @@ -36,7 +36,7 @@ try: from mako.template import Template from mako import exceptions except (ImportError): - pass # Mako not installed but it is ok. + pass # Mako not installed but it is ok. import os import string import xml.etree.cElementTree as ET @@ -63,9 +63,9 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc', root = ET.Element('testsuites') testsuite = ET.SubElement(root, 'testsuite', id='1', package=suite_package, name=suite_name) - for shortname, results in resultset.items(): + for shortname, results in resultset.iteritems(): for result in results: - xml_test = ET.SubElement(testsuite, 'testcase', name=shortname) + xml_test = ET.SubElement(testsuite, 'testcase', name=shortname) if result.elapsed_time: xml_test.set('time', str(result.elapsed_time)) ET.SubElement(xml_test, 'system-out').text = _filter_msg(result.message, @@ -79,7 +79,7 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc', def render_interop_html_report( - client_langs, server_langs, test_cases, auth_test_cases, http2_cases, + client_langs, server_langs, test_cases, auth_test_cases, http2_cases, resultset, num_failures, cloud_to_prod, prod_servers, http2_interop): """Generate HTML report for interop tests.""" template_file = 'tools/run_tests/interop_html_report.template' @@ -99,7 +99,7 @@ def render_interop_html_report( sorted_server_langs = sorted(server_langs) sorted_prod_servers = sorted(prod_servers) - args = {'client_langs': sorted_client_langs, + args = {'client_langs': sorted_client_langs, 'server_langs': sorted_server_langs, 'test_cases': sorted_test_cases, 'auth_test_cases': sorted_auth_test_cases, @@ -110,9 +110,9 @@ def render_interop_html_report( 'prod_servers': sorted_prod_servers, 'http2_interop': http2_interop} - html_report_out_dir = 'reports' + html_report_out_dir = 'reports' if not os.path.exists(html_report_out_dir): - os.mkdir(html_report_out_dir) + os.mkdir(html_report_out_dir) html_file_path = os.path.join(html_report_out_dir, 'index.html') try: with open(html_file_path, 'w') as output_file: @@ -120,4 +120,3 @@ def render_interop_html_report( except: print(exceptions.text_error_template().render()) raise - diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 0c6efda1f47..c73de6b7174 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -64,7 +64,9 @@ _SKIP_SERVER_COMPRESSION = ['server_compressed_unary', _SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION -_SKIP_ADVANCED_GO = ['custom_metadata', 'unimplemented_method'] +_SKIP_ADVANCED_GO = ['custom_metadata', + 'unimplemented_method', + 'unimplemented_service'] _SKIP_ADVANCED = _SKIP_ADVANCED_GO + ['status_code_and_message'] @@ -416,7 +418,8 @@ _TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong', 'timeout_on_sleeping_server', 'custom_metadata', 'status_code_and_message', 'unimplemented_method', 'client_compressed_unary', 'server_compressed_unary', - 'client_compressed_streaming', 'server_compressed_streaming'] + 'client_compressed_streaming', 'server_compressed_streaming', + 'unimplemented_service'] _AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds', 'oauth2_auth_token', 'per_rpc_creds'] diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 099ab89ddff..7cb827ecea9 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -42,6 +42,7 @@ import os import performance.scenario_config as scenario_config import pipes import re +import report_utils import subprocess import sys import tempfile @@ -453,6 +454,7 @@ if not scenarios: total_scenario_failures = 0 qps_workers_killed = 0 +merged_resultset = {} for scenario in scenarios: if args.dry_run: print(scenario.name) @@ -460,14 +462,20 @@ for scenario in scenarios: try: for worker in scenario.workers: worker.start() - scenario_failures, _ = jobset.run([scenario.jobspec, - create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], - newline_on_success=True, maxjobs=1) + scenario_failures, resultset = jobset.run([scenario.jobspec, + create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], + newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures + merged_resultset = dict(itertools.chain(merged_resultset.iteritems(), + resultset.iteritems())) finally: # Consider qps workers that need to be killed as failures qps_workers_killed += finish_qps_workers(scenario.workers) + +report_utils.render_junit_xml_report(merged_resultset, 'report.xml', + suite_name='benchmarks') + if total_scenario_failures > 0 or qps_workers_killed > 0: print ("%s scenarios failed and %s qps worker jobs killed" % (total_scenario_failures, qps_workers_killed)) sys.exit(1) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 2886870d384..911843e9f36 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -363,7 +363,8 @@ class NodeLanguage(object): self.config = config self.args = args _check_compiler(self.args.compiler, ['default', 'node0.12', - 'node4', 'node5', 'node6']) + 'node4', 'node5', 'node6', + 'node7']) if self.args.compiler == 'default': self.node_version = '4' else: @@ -1064,6 +1065,7 @@ argp.add_argument('--compiler', 'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7', 'vs2010', 'vs2013', 'vs2015', 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', + 'node0.12', 'node4', 'node5', 'node6', 'node7', 'coreclr'], default='default', help='Selects compiler to use. Allowed values depend on the platform and language.') diff --git a/tools/run_tests/run_tests_in_workspace.sh b/tools/run_tests/run_tests_in_workspace.sh index 98ef3566db1..9c6c5b76e06 100755 --- a/tools/run_tests/run_tests_in_workspace.sh +++ b/tools/run_tests/run_tests_in_workspace.sh @@ -35,11 +35,13 @@ set -ex cd $(dirname $0)/../.. +export repo_root=$(pwd) rm -rf "${WORKSPACE_NAME}" -# TODO(jtattermusch): clone --recursive fetches the submodules from github. -# Try avoiding that to save time and network capacity. -git clone --recursive . "${WORKSPACE_NAME}" +git clone . "${WORKSPACE_NAME}" +# clone gRPC submodules, use data from locally cloned submodules where possible +git submodule foreach 'cd "${repo_root}/${WORKSPACE_NAME}" \ + && git submodule update --init --reference ${repo_root}/${name} ${name}' echo "Running run_tests.py in workspace ${WORKSPACE_NAME}" python "${WORKSPACE_NAME}/tools/run_tests/run_tests.py" $@ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index eafba13a38e..87ac8add3bd 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -1721,6 +1721,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "resource_quota_test", + "src": [ + "test/core/iomgr/resource_quota_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", @@ -6191,6 +6208,7 @@ "test/core/end2end/tests/registered_call.c", "test/core/end2end/tests/request_with_flags.c", "test/core/end2end/tests/request_with_payload.c", + "test/core/end2end/tests/resource_quota_server.c", "test/core/end2end/tests/server_finishes_request.c", "test/core/end2end/tests/shutdown_finishes_calls.c", "test/core/end2end/tests/shutdown_finishes_tags.c", @@ -6256,6 +6274,7 @@ "test/core/end2end/tests/registered_call.c", "test/core/end2end/tests/request_with_flags.c", "test/core/end2end/tests/request_with_payload.c", + "test/core/end2end/tests/resource_quota_server.c", "test/core/end2end/tests/server_finishes_request.c", "test/core/end2end/tests/shutdown_finishes_calls.c", "test/core/end2end/tests/shutdown_finishes_tags.c", @@ -6555,6 +6574,7 @@ "src/core/lib/iomgr/pollset_windows.h", "src/core/lib/iomgr/port.h", "src/core/lib/iomgr/resolve_address.h", + "src/core/lib/iomgr/resource_quota.h", "src/core/lib/iomgr/sockaddr.h", "src/core/lib/iomgr/sockaddr_posix.h", "src/core/lib/iomgr/sockaddr_utils.h", @@ -6563,6 +6583,7 @@ "src/core/lib/iomgr/socket_utils_posix.h", "src/core/lib/iomgr/socket_windows.h", "src/core/lib/iomgr/tcp_client.h", + "src/core/lib/iomgr/tcp_client_posix.h", "src/core/lib/iomgr/tcp_posix.h", "src/core/lib/iomgr/tcp_server.h", "src/core/lib/iomgr/tcp_uv.h", @@ -6703,6 +6724,8 @@ "src/core/lib/iomgr/resolve_address_posix.c", "src/core/lib/iomgr/resolve_address_uv.c", "src/core/lib/iomgr/resolve_address_windows.c", + "src/core/lib/iomgr/resource_quota.c", + "src/core/lib/iomgr/resource_quota.h", "src/core/lib/iomgr/sockaddr.h", "src/core/lib/iomgr/sockaddr_posix.h", "src/core/lib/iomgr/sockaddr_utils.c", @@ -6719,6 +6742,7 @@ "src/core/lib/iomgr/socket_windows.h", "src/core/lib/iomgr/tcp_client.h", "src/core/lib/iomgr/tcp_client_posix.c", + "src/core/lib/iomgr/tcp_client_posix.h", "src/core/lib/iomgr/tcp_client_uv.c", "src/core/lib/iomgr/tcp_client_windows.c", "src/core/lib/iomgr/tcp_posix.c", @@ -7420,6 +7444,7 @@ "include/grpc++/impl/thd.h", "include/grpc++/impl/thd_cxx11.h", "include/grpc++/impl/thd_no_cxx11.h", + "include/grpc++/resource_quota.h", "include/grpc++/security/auth_context.h", "include/grpc++/security/auth_metadata_processor.h", "include/grpc++/security/credentials.h", @@ -7477,6 +7502,7 @@ "include/grpc++/impl/thd.h", "include/grpc++/impl/thd_cxx11.h", "include/grpc++/impl/thd_no_cxx11.h", + "include/grpc++/resource_quota.h", "include/grpc++/security/auth_context.h", "include/grpc++/security/auth_metadata_processor.h", "include/grpc++/security/credentials.h", @@ -7510,6 +7536,7 @@ "src/cpp/common/channel_filter.h", "src/cpp/common/completion_queue_cc.cc", "src/cpp/common/core_codegen.cc", + "src/cpp/common/resource_quota_cc.cc", "src/cpp/common/rpc_method.cc", "src/cpp/server/async_generic_service.cc", "src/cpp/server/create_default_thread_pool.cc", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 8aeefe3411d..733bfe3b8fd 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -1765,6 +1765,28 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 30, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "resource_quota_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ @@ -5133,7 +5155,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_census_test", "platforms": [ @@ -5649,6 +5671,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_census_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -6147,7 +6192,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_compress_test", "platforms": [ @@ -6663,6 +6708,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -7161,7 +7229,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_fake_resolver_test", "platforms": [ @@ -7677,6 +7745,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_fake_resolver_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -8162,7 +8253,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_fakesec_test", "platforms": [ @@ -8656,6 +8747,28 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_fakesec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -9567,6 +9680,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_fd_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -10042,7 +10178,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_full_test", "platforms": [ @@ -10558,6 +10694,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -11005,7 +11164,7 @@ "exclude_iomgrs": [ "uv" ], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_full+pipe_test", "platforms": [ @@ -11430,6 +11589,25 @@ "linux" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_test", + "platforms": [ + "linux" + ] + }, { "args": [ "server_finishes_request" @@ -11892,7 +12070,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_full+trace_test", "platforms": [ @@ -12362,6 +12540,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -12871,7 +13072,7 @@ "exclude_iomgrs": [ "uv" ], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_http_proxy_test", "platforms": [ @@ -13409,6 +13610,30 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_http_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -13916,7 +14141,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_load_reporting_test", "platforms": [ @@ -14432,6 +14657,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_load_reporting_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -14941,7 +15189,7 @@ "exclude_iomgrs": [ "uv" ], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_oauth2_test", "platforms": [ @@ -15479,6 +15727,30 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_oauth2_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -15949,7 +16221,7 @@ "exclude_iomgrs": [ "uv" ], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_proxy_test", "platforms": [ @@ -17351,6 +17623,30 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -19736,7 +20032,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_ssl_test", "platforms": [ @@ -20252,6 +20548,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -20750,7 +21069,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_ssl_cert_test", "platforms": [ @@ -21266,6 +21585,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_cert_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -21727,7 +22069,7 @@ "exclude_iomgrs": [ "uv" ], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_ssl_proxy_test", "platforms": [ @@ -22652,7 +22994,7 @@ "exclude_iomgrs": [ "uv" ], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_uds_test", "platforms": [ @@ -23167,6 +23509,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_uds_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -23642,7 +24007,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_census_nosec_test", "platforms": [ @@ -24158,6 +24523,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_census_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -24633,7 +25021,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_compress_nosec_test", "platforms": [ @@ -25149,6 +25537,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -25624,7 +26035,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_fake_resolver_nosec_test", "platforms": [ @@ -26142,7 +26553,7 @@ }, { "args": [ - "server_finishes_request" + "resource_quota_server" ], "ci_platforms": [ "windows", @@ -26165,7 +26576,7 @@ }, { "args": [ - "shutdown_finishes_calls" + "server_finishes_request" ], "ci_platforms": [ "windows", @@ -26188,7 +26599,7 @@ }, { "args": [ - "shutdown_finishes_tags" + "shutdown_finishes_calls" ], "ci_platforms": [ "windows", @@ -26211,7 +26622,7 @@ }, { "args": [ - "simple_cacheable_request" + "shutdown_finishes_tags" ], "ci_platforms": [ "windows", @@ -26234,7 +26645,7 @@ }, { "args": [ - "simple_delayed_request" + "simple_cacheable_request" ], "ci_platforms": [ "windows", @@ -26257,7 +26668,7 @@ }, { "args": [ - "simple_metadata" + "simple_delayed_request" ], "ci_platforms": [ "windows", @@ -26280,7 +26691,7 @@ }, { "args": [ - "simple_request" + "simple_metadata" ], "ci_platforms": [ "windows", @@ -26303,7 +26714,7 @@ }, { "args": [ - "streaming_error_response" + "simple_request" ], "ci_platforms": [ "windows", @@ -26326,7 +26737,7 @@ }, { "args": [ - "trailing_metadata" + "streaming_error_response" ], "ci_platforms": [ "windows", @@ -26349,22 +26760,22 @@ }, { "args": [ - "bad_hostname" + "trailing_metadata" ], "ci_platforms": [ + "windows", "linux", "mac", "posix" ], "cpu_cost": 1.0, "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], + "exclude_iomgrs": [], "flaky": false, "language": "c", - "name": "h2_fd_nosec_test", + "name": "h2_fake_resolver_nosec_test", "platforms": [ + "windows", "linux", "mac", "posix" @@ -26372,7 +26783,7 @@ }, { "args": [ - "binary_metadata" + "bad_hostname" ], "ci_platforms": [ "linux", @@ -26395,30 +26806,7 @@ }, { "args": [ - "cancel_after_accept" - ], - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_fd_nosec_test", - "platforms": [ - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "cancel_after_client_done" + "binary_metadata" ], "ci_platforms": [ "linux", @@ -26441,30 +26829,7 @@ }, { "args": [ - "cancel_after_invoke" - ], - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_fd_nosec_test", - "platforms": [ - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "cancel_before_invoke" + "cancel_after_accept" ], "ci_platforms": [ "linux", @@ -26487,14 +26852,14 @@ }, { "args": [ - "cancel_in_a_vacuum" + "cancel_after_client_done" ], "ci_platforms": [ "linux", "mac", "posix" ], - "cpu_cost": 0.1, + "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [ "uv" @@ -26510,7 +26875,76 @@ }, { "args": [ - "cancel_with_status" + "cancel_after_invoke" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_fd_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ + "cancel_before_invoke" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_fd_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ + "cancel_in_a_vacuum" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_fd_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ + "cancel_with_status" ], "ci_platforms": [ "linux", @@ -27037,6 +27471,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_fd_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -27489,7 +27946,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_full_nosec_test", "platforms": [ @@ -28005,6 +28462,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -28433,7 +28913,7 @@ "exclude_iomgrs": [ "uv" ], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_full+pipe_nosec_test", "platforms": [ @@ -28858,6 +29338,25 @@ "linux" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_nosec_test", + "platforms": [ + "linux" + ] + }, { "args": [ "server_finishes_request" @@ -29297,7 +29796,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_full+trace_nosec_test", "platforms": [ @@ -29767,6 +30266,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -30252,7 +30774,7 @@ "exclude_iomgrs": [ "uv" ], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_http_proxy_nosec_test", "platforms": [ @@ -30790,6 +31312,30 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_http_proxy_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -31274,7 +31820,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_load_reporting_nosec_test", "platforms": [ @@ -31790,6 +32336,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_load_reporting_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -32227,7 +32796,7 @@ "exclude_iomgrs": [ "uv" ], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_proxy_nosec_test", "platforms": [ @@ -33605,6 +34174,30 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -35893,7 +36486,7 @@ "exclude_iomgrs": [ "uv" ], - "flaky": false, + "flaky": true, "language": "c", "name": "h2_uds_nosec_test", "platforms": [ @@ -36408,6 +37001,29 @@ "posix" ] }, + { + "args": [ + "resource_quota_server" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_uds_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "server_finishes_request" @@ -36762,6 +37378,27 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure", "timeout_seconds": 180 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": 8, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_resource_quota", + "timeout_seconds": 180 + }, { "args": [ "--scenarios_json", @@ -36804,6 +37441,27 @@ "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure", "timeout_seconds": 180 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": 8, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_resource_quota", + "timeout_seconds": 180 + }, { "args": [ "--scenarios_json", @@ -36846,6 +37504,27 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_secure", "timeout_seconds": 180 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": 8, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_resource_quota", + "timeout_seconds": 180 + }, { "args": [ "--scenarios_json", @@ -36888,6 +37567,27 @@ "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure", "timeout_seconds": 180 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": 8, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_resource_quota", + "timeout_seconds": 180 + }, { "args": [ "--scenarios_json", @@ -37035,6 +37735,27 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure", "timeout_seconds": 180 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": 8, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_resource_quota", + "timeout_seconds": 180 + }, { "args": [ "--scenarios_json", @@ -37077,6 +37798,27 @@ "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure", "timeout_seconds": 180 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": 8, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_resource_quota", + "timeout_seconds": 180 + }, { "args": [ "--scenarios_json", @@ -37119,6 +37861,27 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_insecure", "timeout_seconds": 180 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": 8, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_resource_quota", + "timeout_seconds": 180 + }, { "args": [ "--scenarios_json", @@ -37161,6 +37924,27 @@ "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure", "timeout_seconds": 180 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"resource_quota_size\": 512000, \"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": 8, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_resource_quota", + "timeout_seconds": 180 + }, { "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/00.bin" diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index 339b42f9d7c..b1918809bf0 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -1317,6 +1317,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "resolve_address_test", "vcx {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "resource_quota_test", "vcxproj\test\resource_quota_test\resource_quota_test.vcxproj", "{6084F546-5D66-5CB5-63CF-DC960F14B545}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "secure_channel_create_test", "vcxproj\test\secure_channel_create_test\secure_channel_create_test.vcxproj", "{62B25398-7173-928E-689E-53860B0ACFC4}" ProjectSection(myProperties) = preProject lib = "False" @@ -3538,6 +3549,22 @@ Global {8279AF6C-9584-67F3-1547-B204864FCCA7}.Release-DLL|Win32.Build.0 = Release|Win32 {8279AF6C-9584-67F3-1547-B204864FCCA7}.Release-DLL|x64.ActiveCfg = Release|x64 {8279AF6C-9584-67F3-1547-B204864FCCA7}.Release-DLL|x64.Build.0 = Release|x64 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Debug|Win32.ActiveCfg = Debug|Win32 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Debug|x64.ActiveCfg = Debug|x64 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Release|Win32.ActiveCfg = Release|Win32 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Release|x64.ActiveCfg = Release|x64 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Debug|Win32.Build.0 = Debug|Win32 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Debug|x64.Build.0 = Debug|x64 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Release|Win32.Build.0 = Release|Win32 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Release|x64.Build.0 = Release|x64 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Debug-DLL|x64.Build.0 = Debug|x64 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Release-DLL|Win32.Build.0 = Release|Win32 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Release-DLL|x64.ActiveCfg = Release|x64 + {6084F546-5D66-5CB5-63CF-DC960F14B545}.Release-DLL|x64.Build.0 = Release|x64 {62B25398-7173-928E-689E-53860B0ACFC4}.Debug|Win32.ActiveCfg = Debug|Win32 {62B25398-7173-928E-689E-53860B0ACFC4}.Debug|x64.ActiveCfg = Debug|x64 {62B25398-7173-928E-689E-53860B0ACFC4}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index ad217dae7e1..bf9c3a5c9d4 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -285,6 +285,7 @@ + @@ -404,6 +405,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index d4ad8c4c978..b88a78ad92d 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -58,6 +58,9 @@ src\cpp\common + + src\cpp\common + src\cpp\common @@ -189,6 +192,9 @@ include\grpc++\impl + + include\grpc++ + include\grpc++\security diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 01940c31af7..5d0759790c8 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -285,6 +285,7 @@ + @@ -390,6 +391,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index f261c04c7dc..bdb71340817 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -43,6 +43,9 @@ src\cpp\common + + src\cpp\common + src\cpp\common @@ -174,6 +177,9 @@ include\grpc++\impl + + include\grpc++ + include\grpc++\security diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index b9d97fec6c1..d9d0d42d677 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -337,6 +337,7 @@ + @@ -345,6 +346,7 @@ + @@ -559,6 +561,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 86f7edae29f..ab9b76073cf 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -136,6 +136,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -830,6 +833,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -854,6 +860,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 35ace6eabbd..39c144d992f 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -230,6 +230,7 @@ + @@ -238,6 +239,7 @@ + @@ -410,6 +412,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 4b104aeb9bf..0fbfc3acd37 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -193,6 +193,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -626,6 +629,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -650,6 +656,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 377bbf9bbd5..87e3921ff96 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -327,6 +327,7 @@ + @@ -335,6 +336,7 @@ + @@ -527,6 +529,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 9171f823775..1e719226faf 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -139,6 +139,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -743,6 +746,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -767,6 +773,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj index d6ec36962fc..a47c40f0f73 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj @@ -221,6 +221,8 @@ + + diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters index a6c24a895a4..71cf6838fe2 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters @@ -106,6 +106,9 @@ test\core\end2end\tests + + test\core\end2end\tests + test\core\end2end\tests diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj index 085b14700c0..68ff5f1ebd8 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj @@ -223,6 +223,8 @@ + + diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters index d224a9ced96..7a620f61a54 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters @@ -109,6 +109,9 @@ test\core\end2end\tests + + test\core\end2end\tests + test\core\end2end\tests diff --git a/vsprojects/vcxproj/test/resource_quota_test/resource_quota_test.vcxproj b/vsprojects/vcxproj/test/resource_quota_test/resource_quota_test.vcxproj new file mode 100644 index 00000000000..389340e9a03 --- /dev/null +++ b/vsprojects/vcxproj/test/resource_quota_test/resource_quota_test.vcxproj @@ -0,0 +1,199 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {6084F546-5D66-5CB5-63CF-DC960F14B545} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + resource_quota_test + static + Debug + static + Debug + + + resource_quota_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/resource_quota_test/resource_quota_test.vcxproj.filters b/vsprojects/vcxproj/test/resource_quota_test/resource_quota_test.vcxproj.filters new file mode 100644 index 00000000000..d76c8aed5d5 --- /dev/null +++ b/vsprojects/vcxproj/test/resource_quota_test/resource_quota_test.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\core\iomgr + + + + + + {06783e32-dbf0-7e7c-7b50-12b278f9cc12} + + + {c4f432b6-577b-e3ed-fec9-a915af5ebbd5} + + + {fcc82d68-ffb2-0843-83aa-175006c43aeb} + + + +