diff --git a/.pylintrc b/.pylintrc index 05b4e685fb0..453b45aab52 100644 --- a/.pylintrc +++ b/.pylintrc @@ -38,6 +38,9 @@ disable= # TODO(https://github.com/grpc/grpc/issues/261): This doesn't seem to # work for now? Try with a later pylint? locally-disabled, + # NOTE(nathaniel): What even is this? *Enabling* an inspection results + # in a warning? How does that encourage more analysis and coverage? + locally-enabled, # NOTE(nathaniel): We don't write doc strings for most private code # elements. missing-docstring, diff --git a/BUILD b/BUILD index 6b499737bd5..49c340d070c 100644 --- a/BUILD +++ b/BUILD @@ -593,6 +593,9 @@ grpc_cc_library( "src/core/lib/iomgr/ev_windows.c", "src/core/lib/iomgr/exec_ctx.c", "src/core/lib/iomgr/executor.c", + "src/core/lib/iomgr/gethostname_host_name_max.c", + "src/core/lib/iomgr/gethostname_sysconf.c", + "src/core/lib/iomgr/gethostname_fallback.c", "src/core/lib/iomgr/iocp_windows.c", "src/core/lib/iomgr/iomgr.c", "src/core/lib/iomgr/iomgr_posix.c", @@ -718,6 +721,7 @@ grpc_cc_library( "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", "src/core/lib/iomgr/executor.h", + "src/core/lib/iomgr/gethostname.h", "src/core/lib/iomgr/iocp_windows.h", "src/core/lib/iomgr/iomgr.h", "src/core/lib/iomgr/iomgr_internal.h", @@ -774,6 +778,7 @@ grpc_cc_library( "src/core/lib/slice/slice_hash_table.h", "src/core/lib/slice/slice_internal.h", "src/core/lib/slice/slice_string_helpers.h", + "src/core/lib/surface/alarm_internal.h", "src/core/lib/surface/api_trace.h", "src/core/lib/surface/call.h", "src/core/lib/surface/call_test_only.h", @@ -1406,32 +1411,46 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "tsi_interface", + srcs = [ + "src/core/tsi/transport_security.c", + "src/core/tsi/transport_security_adapter.c", + ], + hdrs = [ + "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_adapter.h", + "src/core/tsi/transport_security_interface.h", + ], + language = "c", + deps = [ + "gpr", + "grpc_trace", + ], +) + grpc_cc_library( name = "tsi", srcs = [ "src/core/tsi/fake_transport_security.c", "src/core/tsi/gts_transport_security.c", "src/core/tsi/ssl_transport_security.c", - "src/core/tsi/transport_security.c", - "src/core/tsi/transport_security_adapter.c", + "src/core/tsi/transport_security_grpc.c", ], hdrs = [ "src/core/tsi/fake_transport_security.h", "src/core/tsi/gts_transport_security.h", "src/core/tsi/ssl_transport_security.h", "src/core/tsi/ssl_types.h", - "src/core/tsi/transport_security.h", - "src/core/tsi/transport_security_adapter.h", - "src/core/tsi/transport_security_interface.h", + "src/core/tsi/transport_security_grpc.h", ], external_deps = [ "libssl", ], language = "c", deps = [ - "gpr", "grpc_base", - "grpc_trace", + "tsi_interface", ], ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e1b471449d..8dc4758d238 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -978,6 +978,9 @@ add_library(grpc src/core/lib/iomgr/ev_windows.c src/core/lib/iomgr/exec_ctx.c src/core/lib/iomgr/executor.c + src/core/lib/iomgr/gethostname_fallback.c + src/core/lib/iomgr/gethostname_host_name_max.c + src/core/lib/iomgr/gethostname_sysconf.c src/core/lib/iomgr/iocp_windows.c src/core/lib/iomgr/iomgr.c src/core/lib/iomgr/iomgr_posix.c @@ -1129,6 +1132,7 @@ add_library(grpc src/core/tsi/fake_transport_security.c src/core/tsi/gts_transport_security.c src/core/tsi/ssl_transport_security.c + src/core/tsi/transport_security_grpc.c src/core/tsi/transport_security.c src/core/tsi/transport_security_adapter.c src/core/ext/transport/chttp2/server/chttp2_server.c @@ -1322,6 +1326,9 @@ add_library(grpc_cronet src/core/lib/iomgr/ev_windows.c src/core/lib/iomgr/exec_ctx.c src/core/lib/iomgr/executor.c + src/core/lib/iomgr/gethostname_fallback.c + src/core/lib/iomgr/gethostname_host_name_max.c + src/core/lib/iomgr/gethostname_sysconf.c src/core/lib/iomgr/iocp_windows.c src/core/lib/iomgr/iomgr.c src/core/lib/iomgr/iomgr_posix.c @@ -1497,6 +1504,7 @@ add_library(grpc_cronet src/core/tsi/fake_transport_security.c src/core/tsi/gts_transport_security.c src/core/tsi/ssl_transport_security.c + src/core/tsi/transport_security_grpc.c src/core/tsi/transport_security.c src/core/tsi/transport_security_adapter.c src/core/ext/transport/chttp2/client/chttp2_connector.c @@ -1634,6 +1642,9 @@ add_library(grpc_test_util src/core/lib/iomgr/ev_windows.c src/core/lib/iomgr/exec_ctx.c src/core/lib/iomgr/executor.c + src/core/lib/iomgr/gethostname_fallback.c + src/core/lib/iomgr/gethostname_host_name_max.c + src/core/lib/iomgr/gethostname_sysconf.c src/core/lib/iomgr/iocp_windows.c src/core/lib/iomgr/iomgr.c src/core/lib/iomgr/iomgr_posix.c @@ -1891,6 +1902,9 @@ add_library(grpc_test_util_unsecure src/core/lib/iomgr/ev_windows.c src/core/lib/iomgr/exec_ctx.c src/core/lib/iomgr/executor.c + src/core/lib/iomgr/gethostname_fallback.c + src/core/lib/iomgr/gethostname_host_name_max.c + src/core/lib/iomgr/gethostname_sysconf.c src/core/lib/iomgr/iocp_windows.c src/core/lib/iomgr/iomgr.c src/core/lib/iomgr/iomgr_posix.c @@ -2134,6 +2148,9 @@ add_library(grpc_unsecure src/core/lib/iomgr/ev_windows.c src/core/lib/iomgr/exec_ctx.c src/core/lib/iomgr/executor.c + src/core/lib/iomgr/gethostname_fallback.c + src/core/lib/iomgr/gethostname_host_name_max.c + src/core/lib/iomgr/gethostname_sysconf.c src/core/lib/iomgr/iocp_windows.c src/core/lib/iomgr/iomgr.c src/core/lib/iomgr/iomgr_posix.c @@ -2827,6 +2844,9 @@ add_library(grpc++_cronet src/core/lib/iomgr/ev_windows.c src/core/lib/iomgr/exec_ctx.c src/core/lib/iomgr/executor.c + src/core/lib/iomgr/gethostname_fallback.c + src/core/lib/iomgr/gethostname_host_name_max.c + src/core/lib/iomgr/gethostname_sysconf.c src/core/lib/iomgr/iocp_windows.c src/core/lib/iomgr/iomgr.c src/core/lib/iomgr/iomgr_posix.c diff --git a/Makefile b/Makefile index 5168b30e7b9..74f05f592dc 100644 --- a/Makefile +++ b/Makefile @@ -2925,6 +2925,9 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/ev_windows.c \ src/core/lib/iomgr/exec_ctx.c \ src/core/lib/iomgr/executor.c \ + src/core/lib/iomgr/gethostname_fallback.c \ + src/core/lib/iomgr/gethostname_host_name_max.c \ + src/core/lib/iomgr/gethostname_sysconf.c \ src/core/lib/iomgr/iocp_windows.c \ src/core/lib/iomgr/iomgr.c \ src/core/lib/iomgr/iomgr_posix.c \ @@ -3076,6 +3079,7 @@ LIBGRPC_SRC = \ src/core/tsi/fake_transport_security.c \ src/core/tsi/gts_transport_security.c \ src/core/tsi/ssl_transport_security.c \ + src/core/tsi/transport_security_grpc.c \ src/core/tsi/transport_security.c \ src/core/tsi/transport_security_adapter.c \ src/core/ext/transport/chttp2/server/chttp2_server.c \ @@ -3267,6 +3271,9 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/ev_windows.c \ src/core/lib/iomgr/exec_ctx.c \ src/core/lib/iomgr/executor.c \ + src/core/lib/iomgr/gethostname_fallback.c \ + src/core/lib/iomgr/gethostname_host_name_max.c \ + src/core/lib/iomgr/gethostname_sysconf.c \ src/core/lib/iomgr/iocp_windows.c \ src/core/lib/iomgr/iomgr.c \ src/core/lib/iomgr/iomgr_posix.c \ @@ -3442,6 +3449,7 @@ LIBGRPC_CRONET_SRC = \ src/core/tsi/fake_transport_security.c \ src/core/tsi/gts_transport_security.c \ src/core/tsi/ssl_transport_security.c \ + src/core/tsi/transport_security_grpc.c \ src/core/tsi/transport_security.c \ src/core/tsi/transport_security_adapter.c \ src/core/ext/transport/chttp2/client/chttp2_connector.c \ @@ -3576,6 +3584,9 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/ev_windows.c \ src/core/lib/iomgr/exec_ctx.c \ src/core/lib/iomgr/executor.c \ + src/core/lib/iomgr/gethostname_fallback.c \ + src/core/lib/iomgr/gethostname_host_name_max.c \ + src/core/lib/iomgr/gethostname_sysconf.c \ src/core/lib/iomgr/iocp_windows.c \ src/core/lib/iomgr/iomgr.c \ src/core/lib/iomgr/iomgr_posix.c \ @@ -3822,6 +3833,9 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/lib/iomgr/ev_windows.c \ src/core/lib/iomgr/exec_ctx.c \ src/core/lib/iomgr/executor.c \ + src/core/lib/iomgr/gethostname_fallback.c \ + src/core/lib/iomgr/gethostname_host_name_max.c \ + src/core/lib/iomgr/gethostname_sysconf.c \ src/core/lib/iomgr/iocp_windows.c \ src/core/lib/iomgr/iomgr.c \ src/core/lib/iomgr/iomgr_posix.c \ @@ -4041,6 +4055,9 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/ev_windows.c \ src/core/lib/iomgr/exec_ctx.c \ src/core/lib/iomgr/executor.c \ + src/core/lib/iomgr/gethostname_fallback.c \ + src/core/lib/iomgr/gethostname_host_name_max.c \ + src/core/lib/iomgr/gethostname_sysconf.c \ src/core/lib/iomgr/iocp_windows.c \ src/core/lib/iomgr/iomgr.c \ src/core/lib/iomgr/iomgr_posix.c \ @@ -4717,6 +4734,9 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/ev_windows.c \ src/core/lib/iomgr/exec_ctx.c \ src/core/lib/iomgr/executor.c \ + src/core/lib/iomgr/gethostname_fallback.c \ + src/core/lib/iomgr/gethostname_host_name_max.c \ + src/core/lib/iomgr/gethostname_sysconf.c \ src/core/lib/iomgr/iocp_windows.c \ src/core/lib/iomgr/iomgr.c \ src/core/lib/iomgr/iomgr_posix.c \ @@ -19703,6 +19723,7 @@ src/core/tsi/gts_transport_security.c: $(OPENSSL_DEP) src/core/tsi/ssl_transport_security.c: $(OPENSSL_DEP) src/core/tsi/transport_security.c: $(OPENSSL_DEP) src/core/tsi/transport_security_adapter.c: $(OPENSSL_DEP) +src/core/tsi/transport_security_grpc.c: $(OPENSSL_DEP) src/cpp/client/cronet_credentials.cc: $(OPENSSL_DEP) src/cpp/client/secure_credentials.cc: $(OPENSSL_DEP) src/cpp/common/auth_property_iterator.cc: $(OPENSSL_DEP) diff --git a/binding.gyp b/binding.gyp index 8a2900b0100..bbefd05c20b 100644 --- a/binding.gyp +++ b/binding.gyp @@ -687,6 +687,9 @@ 'src/core/lib/iomgr/ev_windows.c', 'src/core/lib/iomgr/exec_ctx.c', 'src/core/lib/iomgr/executor.c', + 'src/core/lib/iomgr/gethostname_fallback.c', + 'src/core/lib/iomgr/gethostname_host_name_max.c', + 'src/core/lib/iomgr/gethostname_sysconf.c', 'src/core/lib/iomgr/iocp_windows.c', 'src/core/lib/iomgr/iomgr.c', 'src/core/lib/iomgr/iomgr_posix.c', @@ -838,6 +841,7 @@ 'src/core/tsi/fake_transport_security.c', 'src/core/tsi/gts_transport_security.c', 'src/core/tsi/ssl_transport_security.c', + 'src/core/tsi/transport_security_grpc.c', 'src/core/tsi/transport_security.c', 'src/core/tsi/transport_security_adapter.c', 'src/core/ext/transport/chttp2/server/chttp2_server.c', diff --git a/build.yaml b/build.yaml index 81bcf4481fb..a459b9d6883 100644 --- a/build.yaml +++ b/build.yaml @@ -214,6 +214,9 @@ filegroups: - src/core/lib/iomgr/ev_windows.c - src/core/lib/iomgr/exec_ctx.c - src/core/lib/iomgr/executor.c + - src/core/lib/iomgr/gethostname_fallback.c + - src/core/lib/iomgr/gethostname_host_name_max.c + - src/core/lib/iomgr/gethostname_sysconf.c - src/core/lib/iomgr/iocp_windows.c - src/core/lib/iomgr/iomgr.c - src/core/lib/iomgr/iomgr_posix.c @@ -359,6 +362,7 @@ filegroups: - src/core/lib/iomgr/ev_posix.h - src/core/lib/iomgr/exec_ctx.h - src/core/lib/iomgr/executor.h + - src/core/lib/iomgr/gethostname.h - src/core/lib/iomgr/iocp_windows.h - src/core/lib/iomgr/iomgr.h - src/core/lib/iomgr/iomgr_internal.h @@ -415,6 +419,7 @@ filegroups: - src/core/lib/slice/slice_hash_table.h - src/core/lib/slice/slice_internal.h - src/core/lib/slice/slice_string_helpers.h + - src/core/lib/surface/alarm_internal.h - src/core/lib/surface/api_trace.h - src/core/lib/surface/call.h - src/core/lib/surface/call_test_only.h @@ -920,22 +925,33 @@ filegroups: - src/core/tsi/gts_transport_security.h - src/core/tsi/ssl_transport_security.h - src/core/tsi/ssl_types.h - - src/core/tsi/transport_security.h - - src/core/tsi/transport_security_adapter.h - - src/core/tsi/transport_security_interface.h + - src/core/tsi/transport_security_grpc.h src: - src/core/tsi/fake_transport_security.c - src/core/tsi/gts_transport_security.c - src/core/tsi/ssl_transport_security.c + - src/core/tsi/transport_security_grpc.c + deps: + - gpr + plugin: grpc_tsi_gts + secure: true + uses: + - tsi_interface + - grpc_base + - grpc_trace +- name: tsi_interface + headers: + - src/core/tsi/transport_security.h + - src/core/tsi/transport_security_adapter.h + - src/core/tsi/transport_security_interface.h + src: - src/core/tsi/transport_security.c - src/core/tsi/transport_security_adapter.c deps: - gpr - plugin: grpc_tsi_gts secure: true uses: - grpc_trace - - grpc_base - name: grpc++_codegen_base language: c++ public_headers: diff --git a/config.m4 b/config.m4 index c5332f13f04..f6f8531b2fd 100644 --- a/config.m4 +++ b/config.m4 @@ -116,6 +116,9 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/ev_windows.c \ src/core/lib/iomgr/exec_ctx.c \ src/core/lib/iomgr/executor.c \ + src/core/lib/iomgr/gethostname_fallback.c \ + src/core/lib/iomgr/gethostname_host_name_max.c \ + src/core/lib/iomgr/gethostname_sysconf.c \ src/core/lib/iomgr/iocp_windows.c \ src/core/lib/iomgr/iomgr.c \ src/core/lib/iomgr/iomgr_posix.c \ @@ -267,6 +270,7 @@ if test "$PHP_GRPC" != "no"; then src/core/tsi/fake_transport_security.c \ src/core/tsi/gts_transport_security.c \ src/core/tsi/ssl_transport_security.c \ + src/core/tsi/transport_security_grpc.c \ src/core/tsi/transport_security.c \ src/core/tsi/transport_security_adapter.c \ src/core/ext/transport/chttp2/server/chttp2_server.c \ diff --git a/config.w32 b/config.w32 index 8529bd549f5..1d1a0a4b63f 100644 --- a/config.w32 +++ b/config.w32 @@ -93,6 +93,9 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\iomgr\\ev_windows.c " + "src\\core\\lib\\iomgr\\exec_ctx.c " + "src\\core\\lib\\iomgr\\executor.c " + + "src\\core\\lib\\iomgr\\gethostname_fallback.c " + + "src\\core\\lib\\iomgr\\gethostname_host_name_max.c " + + "src\\core\\lib\\iomgr\\gethostname_sysconf.c " + "src\\core\\lib\\iomgr\\iocp_windows.c " + "src\\core\\lib\\iomgr\\iomgr.c " + "src\\core\\lib\\iomgr\\iomgr_posix.c " + @@ -244,6 +247,7 @@ if (PHP_GRPC != "no") { "src\\core\\tsi\\fake_transport_security.c " + "src\\core\\tsi\\gts_transport_security.c " + "src\\core\\tsi\\ssl_transport_security.c " + + "src\\core\\tsi\\transport_security_grpc.c " + "src\\core\\tsi\\transport_security.c " + "src\\core\\tsi\\transport_security_adapter.c " + "src\\core\\ext\\transport\\chttp2\\server\\chttp2_server.c " + diff --git a/doc/environment_variables.md b/doc/environment_variables.md index 036824d573f..a2cde927361 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -69,6 +69,7 @@ some configuration as environment variables that can be set. The following tracers will only run in binaries built in DEBUG mode. This is accomplished by invoking `CONFIG=dbg make ` + - alarm_refcount - refcounting traces for grpc_alarm structure - metadata - tracks creation and mutation of metadata - closure - tracks closure creation, scheduling, and completion - pending_tags - traces still-in-progress tags on completion queues diff --git a/doc/epoll-polling-engine.md b/doc/epoll-polling-engine.md index ab7030a211d..1f5d855743a 100644 --- a/doc/epoll-polling-engine.md +++ b/doc/epoll-polling-engine.md @@ -5,7 +5,7 @@ Sree Kuchibhotla (sreek@) [May - 2016] > Status: As of June 2016, this change is implemented and merged. -> * The bulk of the functionality is in: [ev_poll_linux.c](https://github.com/grpc/grpc/blob/master/src/core/lib/iomgr/ev_epoll_linux.c) +> * The bulk of the functionality is in: [ev_epollsig_linux.c](https://github.com/grpc/grpc/blob/master/src/core/lib/iomgr/ev_epollsig_linux.c) > * Pull request: https://github.com/grpc/grpc/pull/6803 ## 1. Introduction diff --git a/doc/load-balancing.md b/doc/load-balancing.md index f56d2b0c73a..88ff35496f3 100644 --- a/doc/load-balancing.md +++ b/doc/load-balancing.md @@ -113,8 +113,8 @@ works: that indicates which client-side load-balancing policy to use (e.g., `round_robin` or `grpclb`). 2. The client instantiates the load balancing policy. - - Note: If all addresses returned by the resolver are balancer - addresses, then the client will use the `grpclb` policy, regardless + - Note: If any one of the addresses returned by the resolver is a balancer + address, then the client will use the `grpclb` policy, regardless of what load-balancing policy was requested by the service config. Otherwise, the client will use the load-balancing policy requested by the service config. If no load-balancing policy is requested diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 11422f56d47..4b1a8f38a34 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -290,6 +290,7 @@ Pod::Spec.new do |s| 'src/core/tsi/gts_transport_security.h', 'src/core/tsi/ssl_transport_security.h', 'src/core/tsi/ssl_types.h', + 'src/core/tsi/transport_security_grpc.h', 'src/core/tsi/transport_security.h', 'src/core/tsi/transport_security_adapter.h', 'src/core/tsi/transport_security_interface.h', @@ -344,6 +345,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', 'src/core/lib/iomgr/executor.h', + 'src/core/lib/iomgr/gethostname.h', 'src/core/lib/iomgr/iocp_windows.h', 'src/core/lib/iomgr/iomgr.h', 'src/core/lib/iomgr/iomgr_internal.h', @@ -400,6 +402,7 @@ Pod::Spec.new do |s| 'src/core/lib/slice/slice_hash_table.h', 'src/core/lib/slice/slice_internal.h', 'src/core/lib/slice/slice_string_helpers.h', + 'src/core/lib/surface/alarm_internal.h', 'src/core/lib/surface/api_trace.h', 'src/core/lib/surface/call.h', 'src/core/lib/surface/call_test_only.h', @@ -492,6 +495,9 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/ev_windows.c', 'src/core/lib/iomgr/exec_ctx.c', 'src/core/lib/iomgr/executor.c', + 'src/core/lib/iomgr/gethostname_fallback.c', + 'src/core/lib/iomgr/gethostname_host_name_max.c', + 'src/core/lib/iomgr/gethostname_sysconf.c', 'src/core/lib/iomgr/iocp_windows.c', 'src/core/lib/iomgr/iomgr.c', 'src/core/lib/iomgr/iomgr_posix.c', @@ -643,6 +649,7 @@ Pod::Spec.new do |s| 'src/core/tsi/fake_transport_security.c', 'src/core/tsi/gts_transport_security.c', 'src/core/tsi/ssl_transport_security.c', + 'src/core/tsi/transport_security_grpc.c', 'src/core/tsi/transport_security.c', 'src/core/tsi/transport_security_adapter.c', 'src/core/ext/transport/chttp2/server/chttp2_server.c', @@ -777,6 +784,7 @@ Pod::Spec.new do |s| 'src/core/tsi/gts_transport_security.h', 'src/core/tsi/ssl_transport_security.h', 'src/core/tsi/ssl_types.h', + 'src/core/tsi/transport_security_grpc.h', 'src/core/tsi/transport_security.h', 'src/core/tsi/transport_security_adapter.h', 'src/core/tsi/transport_security_interface.h', @@ -831,6 +839,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', 'src/core/lib/iomgr/executor.h', + 'src/core/lib/iomgr/gethostname.h', 'src/core/lib/iomgr/iocp_windows.h', 'src/core/lib/iomgr/iomgr.h', 'src/core/lib/iomgr/iomgr_internal.h', @@ -887,6 +896,7 @@ Pod::Spec.new do |s| 'src/core/lib/slice/slice_hash_table.h', 'src/core/lib/slice/slice_internal.h', 'src/core/lib/slice/slice_string_helpers.h', + 'src/core/lib/surface/alarm_internal.h', 'src/core/lib/surface/api_trace.h', 'src/core/lib/surface/call.h', 'src/core/lib/surface/call_test_only.h', diff --git a/grpc.gemspec b/grpc.gemspec old mode 100755 new mode 100644 index a4566672acc..f04a14167b1 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -222,6 +222,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/tsi/gts_transport_security.h ) s.files += %w( src/core/tsi/ssl_transport_security.h ) s.files += %w( src/core/tsi/ssl_types.h ) + s.files += %w( src/core/tsi/transport_security_grpc.h ) s.files += %w( src/core/tsi/transport_security.h ) s.files += %w( src/core/tsi/transport_security_adapter.h ) s.files += %w( src/core/tsi/transport_security_interface.h ) @@ -276,6 +277,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/ev_posix.h ) s.files += %w( src/core/lib/iomgr/exec_ctx.h ) s.files += %w( src/core/lib/iomgr/executor.h ) + s.files += %w( src/core/lib/iomgr/gethostname.h ) s.files += %w( src/core/lib/iomgr/iocp_windows.h ) s.files += %w( src/core/lib/iomgr/iomgr.h ) s.files += %w( src/core/lib/iomgr/iomgr_internal.h ) @@ -332,6 +334,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/slice/slice_hash_table.h ) s.files += %w( src/core/lib/slice/slice_internal.h ) s.files += %w( src/core/lib/slice/slice_string_helpers.h ) + s.files += %w( src/core/lib/surface/alarm_internal.h ) s.files += %w( src/core/lib/surface/api_trace.h ) s.files += %w( src/core/lib/surface/call.h ) s.files += %w( src/core/lib/surface/call_test_only.h ) @@ -424,6 +427,9 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/ev_windows.c ) s.files += %w( src/core/lib/iomgr/exec_ctx.c ) s.files += %w( src/core/lib/iomgr/executor.c ) + s.files += %w( src/core/lib/iomgr/gethostname_fallback.c ) + s.files += %w( src/core/lib/iomgr/gethostname_host_name_max.c ) + s.files += %w( src/core/lib/iomgr/gethostname_sysconf.c ) s.files += %w( src/core/lib/iomgr/iocp_windows.c ) s.files += %w( src/core/lib/iomgr/iomgr.c ) s.files += %w( src/core/lib/iomgr/iomgr_posix.c ) @@ -575,6 +581,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/tsi/fake_transport_security.c ) s.files += %w( src/core/tsi/gts_transport_security.c ) s.files += %w( src/core/tsi/ssl_transport_security.c ) + s.files += %w( src/core/tsi/transport_security_grpc.c ) s.files += %w( src/core/tsi/transport_security.c ) s.files += %w( src/core/tsi/transport_security_adapter.c ) s.files += %w( src/core/ext/transport/chttp2/server/chttp2_server.c ) diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index f6eefb9f7ff..33d8f4ce65e 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -349,6 +349,28 @@ class CallOpRecvMessage { bool allow_not_getting_message_; }; +namespace CallOpGenericRecvMessageHelper { +class DeserializeFunc { + public: + virtual Status Deserialize(grpc_byte_buffer* buf) = 0; + virtual ~DeserializeFunc() {} +}; + +template +class DeserializeFuncType final : public DeserializeFunc { + public: + DeserializeFuncType(R* message) : message_(message) {} + Status Deserialize(grpc_byte_buffer* buf) override { + return SerializationTraits::Deserialize(buf, message_); + } + + ~DeserializeFuncType() override {} + + private: + R* message_; // Not a managed pointer because management is external to this +}; +} // namespace CallOpGenericRecvMessageHelper + class CallOpGenericRecvMessage { public: CallOpGenericRecvMessage() @@ -356,9 +378,11 @@ class CallOpGenericRecvMessage { template void RecvMessage(R* message) { - deserialize_ = [message](grpc_byte_buffer* buf) -> Status { - return SerializationTraits::Deserialize(buf, message); - }; + // Use an explicit base class pointer to avoid resolution error in the + // following unique_ptr::reset for some old implementations. + CallOpGenericRecvMessageHelper::DeserializeFunc* func = + new CallOpGenericRecvMessageHelper::DeserializeFuncType(message); + deserialize_.reset(func); } // Do not change status if no message is received. @@ -381,7 +405,7 @@ class CallOpGenericRecvMessage { if (recv_buf_) { if (*status) { got_message = true; - *status = deserialize_(recv_buf_).ok(); + *status = deserialize_->Deserialize(recv_buf_).ok(); } else { got_message = false; g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); @@ -392,12 +416,11 @@ class CallOpGenericRecvMessage { *status = false; } } - deserialize_ = DeserializeFunc(); + deserialize_.reset(); } private: - typedef std::function DeserializeFunc; - DeserializeFunc deserialize_; + std::unique_ptr deserialize_; grpc_byte_buffer* recv_buf_; bool allow_not_getting_message_; }; diff --git a/include/grpc++/support/slice.h b/include/grpc++/support/slice.h index 0b4ba7cecb4..bbf97f280ef 100644 --- a/include/grpc++/support/slice.h +++ b/include/grpc++/support/slice.h @@ -67,6 +67,20 @@ class Slice final { return *this; } + /// Create a slice pointing at some data. Calls malloc to allocate a refcount + /// for the object, and arranges that destroy will be called with the + /// user data pointer passed in at destruction. Can be the same as buf or + /// different (e.g., if data is part of a larger structure that must be + /// destroyed when the data is no longer needed) + Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); + + /// Specialization of above for common case where buf == user_data + Slice(void* buf, size_t len, void (*destroy)(void*)) + : Slice(buf, len, destroy, buf) {} + + /// Similar to the above but has a destroy that also takes slice length + Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); + /// Byte size. size_t size() const { return GRPC_SLICE_LENGTH(slice_); } diff --git a/include/grpc/impl/codegen/byte_buffer_reader.h b/include/grpc/impl/codegen/byte_buffer_reader.h index 2ae3f1e20e4..dc0f15496fd 100644 --- a/include/grpc/impl/codegen/byte_buffer_reader.h +++ b/include/grpc/impl/codegen/byte_buffer_reader.h @@ -29,7 +29,7 @@ struct grpc_byte_buffer_reader { struct grpc_byte_buffer *buffer_in; struct grpc_byte_buffer *buffer_out; /** Different current objects correspond to different types of byte buffers */ - union { + union grpc_byte_buffer_reader_current { /** Index into a slice buffer's array of slices */ unsigned index; } current; diff --git a/include/grpc/impl/codegen/compression_types.h b/include/grpc/impl/codegen/compression_types.h index e39c13e88d8..f1b2de3f7df 100644 --- a/include/grpc/impl/codegen/compression_types.h +++ b/include/grpc/impl/codegen/compression_types.h @@ -84,7 +84,7 @@ typedef struct grpc_compression_options { * behind \a GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL. If present, takes * precedence over \a default_algorithm. * TODO(dgq): currently only available for server channels. */ - struct { + struct grpc_compression_options_default_level { int is_set; grpc_compression_level level; } default_level; @@ -92,7 +92,7 @@ typedef struct grpc_compression_options { /** The default channel compression algorithm. It'll be used in the absence of * call specific settings. This option corresponds to the channel argument key * behind \a GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM. */ - struct { + struct grpc_compression_options_default_algorithm { int is_set; grpc_compression_algorithm algorithm; } default_algorithm; diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index bb3c90e303e..8813ec8f35f 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -41,11 +41,11 @@ typedef enum { typedef struct grpc_byte_buffer { void *reserved; grpc_byte_buffer_type type; - union { - struct { + union grpc_byte_buffer_data { + struct /* internal */ { void *reserved[8]; } reserved; - struct { + struct grpc_compressed_buffer { grpc_compression_algorithm compression; grpc_slice_buffer slice_buffer; } raw; @@ -104,10 +104,10 @@ typedef struct grpc_arg_pointer_vtable { typedef struct { grpc_arg_type type; char *key; - union { + union grpc_arg_value { char *string; int integer; - struct { + struct grpc_arg_pointer { void *p; const grpc_arg_pointer_vtable *vtable; } pointer; @@ -258,8 +258,12 @@ typedef struct { #define GRPC_ARG_RESOURCE_QUOTA "grpc.resource_quota" /** If non-zero, expand wildcard addresses to a list of local addresses. */ #define GRPC_ARG_EXPAND_WILDCARD_ADDRS "grpc.expand_wildcard_addrs" -/** Service config data in JSON form. Not intended for use outside of tests. */ +/** Service config data in JSON form. + This value will be ignored if the name resolver returns a service config. */ #define GRPC_ARG_SERVICE_CONFIG "grpc.service_config" +/** Disable looking up the service config via the name resolver. */ +#define GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION \ + "grpc.service_config_disable_resolution" /** LB policy name. */ #define GRPC_ARG_LB_POLICY_NAME "grpc.lb_policy_name" /** The grpc_socket_mutator instance that set the socket options. A pointer. */ @@ -387,7 +391,7 @@ typedef struct grpc_metadata { /** The following fields are reserved for grpc internal use. There is no need to initialize them, and they will be set to garbage during calls to grpc. */ - struct { + struct /* internal */ { void *obfuscated[4]; } internal_data; } grpc_metadata; @@ -487,25 +491,25 @@ typedef struct grpc_op { uint32_t flags; /** Reserved for future usage */ void *reserved; - union { + union grpc_op_data { /** Reserved for future usage */ - struct { + struct /* internal */ { void *reserved[8]; } reserved; - struct { + struct grpc_op_send_initial_metadata { size_t count; grpc_metadata *metadata; /** If \a is_set, \a compression_level will be used for the call. * Otherwise, \a compression_level won't be considered */ - struct { + struct grpc_op_send_initial_metadata_maybe_compression_level { uint8_t is_set; grpc_compression_level level; } maybe_compression_level; } send_initial_metadata; - struct { + struct grpc_op_send_message { struct grpc_byte_buffer *send_message; } send_message; - struct { + struct grpc_op_send_status_from_server { size_t trailing_metadata_count; grpc_metadata *trailing_metadata; grpc_status_code status; @@ -519,16 +523,16 @@ typedef struct grpc_op { object, recv_initial_metadata->array is owned by the caller). After the operation completes, call grpc_metadata_array_destroy on this value, or reuse it in a future op. */ - struct { + struct grpc_op_recv_initial_metadata { grpc_metadata_array *recv_initial_metadata; } recv_initial_metadata; /** ownership of the byte buffer is moved to the caller; the caller must call grpc_byte_buffer_destroy on this value, or reuse it in a future op. */ - struct { + struct grpc_op_recv_message { struct grpc_byte_buffer **recv_message; } recv_message; - struct { + struct grpc_op_recv_status_on_client { /** ownership of the array is with the caller, but ownership of the elements stays with the call object (ie key, value members are owned by the call object, trailing_metadata->array is owned by the caller). @@ -538,7 +542,7 @@ typedef struct grpc_op { grpc_status_code *status; grpc_slice *status_details; } recv_status_on_client; - struct { + struct grpc_op_recv_close_on_server { /** out argument, set to 1 if the call failed in any way (seen as a cancellation on the server), or 0 if the call succeeded */ int *cancelled; diff --git a/include/grpc/impl/codegen/slice.h b/include/grpc/impl/codegen/slice.h index 5ec439eb377..a04c683a558 100644 --- a/include/grpc/impl/codegen/slice.h +++ b/include/grpc/impl/codegen/slice.h @@ -75,12 +75,12 @@ typedef struct grpc_slice_refcount { of data that is copied by value. */ struct grpc_slice { struct grpc_slice_refcount *refcount; - union { - struct { + union grpc_slice_data { + struct grpc_slice_refcounted { uint8_t *bytes; size_t length; } refcounted; - struct { + struct grpc_slice_inlined { uint8_t length; uint8_t bytes[GRPC_SLICE_INLINED_SIZE]; } inlined; diff --git a/package.xml b/package.xml index 9bdebe34700..4e288b3c682 100644 --- a/package.xml +++ b/package.xml @@ -236,6 +236,7 @@ + @@ -290,6 +291,7 @@ + @@ -346,6 +348,7 @@ + @@ -438,6 +441,9 @@ + + + @@ -589,6 +595,7 @@ + diff --git a/src/core/ext/census/tracing.c b/src/core/ext/census/tracing.c index 543a73c5ad4..823c681abf3 100644 --- a/src/core/ext/census/tracing.c +++ b/src/core/ext/census/tracing.c @@ -21,7 +21,6 @@ #include #include #include -#include #include "src/core/ext/census/mlog.h" void trace_start_span(const trace_span_context *span_ctxt, diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c index 04a7852323d..f1480bb1ae4 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c @@ -19,7 +19,10 @@ #include #if GRPC_ARES == 1 && !defined(GRPC_UV) +#include +#include #include +#include #include #include @@ -31,11 +34,14 @@ #include "src/core/ext/filters/client_channel/resolver_registry.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/gethostname.h" #include "src/core/lib/iomgr/resolve_address.h" #include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/json/json.h" #include "src/core/lib/support/backoff.h" #include "src/core/lib/support/env.h" #include "src/core/lib/support/string.h" +#include "src/core/lib/transport/service_config.h" #define GRPC_DNS_MIN_CONNECT_TIMEOUT_SECONDS 1 #define GRPC_DNS_INITIAL_CONNECT_BACKOFF_SECONDS 1 @@ -54,6 +60,8 @@ typedef struct { char *default_port; /** channel args. */ grpc_channel_args *channel_args; + /** whether to request the service config */ + bool request_service_config; /** pollset_set to drive the name resolution process */ grpc_pollset_set *interested_parties; @@ -85,6 +93,8 @@ typedef struct { /** currently resolving addresses */ grpc_lb_addresses *lb_addresses; + /** currently resolving service config */ + char *service_config_json; } ares_dns_resolver; static void dns_ares_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *r); @@ -144,6 +154,77 @@ static void dns_ares_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, GRPC_RESOLVER_UNREF(exec_ctx, &r->base, "retry-timer"); } +static bool value_in_json_array(grpc_json *array, const char *value) { + for (grpc_json *entry = array->child; entry != NULL; entry = entry->next) { + if (entry->type == GRPC_JSON_STRING && strcmp(entry->value, value) == 0) { + return true; + } + } + return false; +} + +static char *choose_service_config(char *service_config_choice_json) { + grpc_json *choices_json = grpc_json_parse_string(service_config_choice_json); + if (choices_json == NULL || choices_json->type != GRPC_JSON_ARRAY) { + gpr_log(GPR_ERROR, "cannot parse service config JSON string"); + return NULL; + } + char *service_config = NULL; + for (grpc_json *choice = choices_json->child; choice != NULL; + choice = choice->next) { + if (choice->type != GRPC_JSON_OBJECT) { + gpr_log(GPR_ERROR, "cannot parse service config JSON string"); + break; + } + grpc_json *service_config_json = NULL; + for (grpc_json *field = choice->child; field != NULL; field = field->next) { + // Check client language, if specified. + if (strcmp(field->key, "clientLanguage") == 0) { + if (field->type != GRPC_JSON_ARRAY || + !value_in_json_array(field, "c++")) { + service_config_json = NULL; + break; + } + } + // Check client hostname, if specified. + if (strcmp(field->key, "clientHostname") == 0) { + char *hostname = grpc_gethostname(); + if (hostname == NULL || field->type != GRPC_JSON_ARRAY || + !value_in_json_array(field, hostname)) { + service_config_json = NULL; + break; + } + } + // Check percentage, if specified. + if (strcmp(field->key, "percentage") == 0) { + if (field->type != GRPC_JSON_NUMBER) { + service_config_json = NULL; + break; + } + int random_pct = rand() % 100; + int percentage; + if (sscanf(field->value, "%d", &percentage) != 1 || + random_pct > percentage) { + service_config_json = NULL; + break; + } + } + // Save service config. + if (strcmp(field->key, "serviceConfig") == 0) { + if (field->type == GRPC_JSON_OBJECT) { + service_config_json = field; + } + } + } + if (service_config_json != NULL) { + service_config = grpc_json_dump_to_string(service_config_json, 0); + break; + } + } + grpc_json_destroy(choices_json); + return service_config; +} + static void dns_ares_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { ares_dns_resolver *r = arg; @@ -152,8 +233,40 @@ static void dns_ares_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg, r->resolving = false; r->pending_request = NULL; if (r->lb_addresses != NULL) { - grpc_arg new_arg = grpc_lb_addresses_create_channel_arg(r->lb_addresses); - result = grpc_channel_args_copy_and_add(r->channel_args, &new_arg, 1); + static const char *args_to_remove[2]; + size_t num_args_to_remove = 0; + grpc_arg new_args[3]; + size_t num_args_to_add = 0; + new_args[num_args_to_add++] = + grpc_lb_addresses_create_channel_arg(r->lb_addresses); + grpc_service_config *service_config = NULL; + char *service_config_string = NULL; + if (r->service_config_json != NULL) { + service_config_string = choose_service_config(r->service_config_json); + gpr_free(r->service_config_json); + if (service_config_string != NULL) { + gpr_log(GPR_INFO, "selected service config choice: %s", + service_config_string); + args_to_remove[num_args_to_remove++] = GRPC_ARG_SERVICE_CONFIG; + new_args[num_args_to_add++] = grpc_channel_arg_string_create( + GRPC_ARG_SERVICE_CONFIG, service_config_string); + service_config = grpc_service_config_create(service_config_string); + if (service_config != NULL) { + const char *lb_policy_name = + grpc_service_config_get_lb_policy_name(service_config); + if (lb_policy_name != NULL) { + args_to_remove[num_args_to_remove++] = GRPC_ARG_LB_POLICY_NAME; + new_args[num_args_to_add++] = grpc_channel_arg_string_create( + GRPC_ARG_LB_POLICY_NAME, (char *)lb_policy_name); + } + } + } + } + result = grpc_channel_args_copy_and_add_and_remove( + r->channel_args, args_to_remove, num_args_to_remove, new_args, + num_args_to_add); + if (service_config != NULL) grpc_service_config_destroy(service_config); + gpr_free(service_config_string); grpc_lb_addresses_destroy(exec_ctx, r->lb_addresses); } else { const char *msg = grpc_error_string(error); @@ -207,10 +320,12 @@ static void dns_ares_start_resolving_locked(grpc_exec_ctx *exec_ctx, GPR_ASSERT(!r->resolving); r->resolving = true; r->lb_addresses = NULL; + r->service_config_json = NULL; r->pending_request = grpc_dns_lookup_ares( exec_ctx, r->dns_server, r->name_to_resolve, r->default_port, r->interested_parties, &r->dns_ares_on_resolved_locked, &r->lb_addresses, - true /* check_grpclb */); + true /* check_grpclb */, + r->request_service_config ? &r->service_config_json : NULL); } static void dns_ares_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx, @@ -256,6 +371,10 @@ static grpc_resolver *dns_ares_create(grpc_exec_ctx *exec_ctx, r->name_to_resolve = gpr_strdup(path); r->default_port = gpr_strdup(default_port); r->channel_args = grpc_channel_args_copy(args->args); + const grpc_arg *arg = grpc_channel_args_find( + r->channel_args, GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION); + r->request_service_config = !grpc_channel_arg_get_integer( + arg, (grpc_integer_options){false, false, true}); r->interested_parties = grpc_pollset_set_create(); if (args->pollset_set != NULL) { grpc_pollset_set_add_pollset_set(exec_ctx, r->interested_parties, diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c index 6ec3790a5fe..e65723a63be 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c @@ -54,6 +54,8 @@ struct grpc_ares_request { grpc_closure *on_done; /** the pointer to receive the resolved addresses */ grpc_lb_addresses **lb_addrs_out; + /** the pointer to receive the service config in JSON */ + char **service_config_json_out; /** the evernt driver used by this request */ grpc_ares_ev_driver *ev_driver; /** number of ongoing queries */ @@ -266,10 +268,68 @@ static void on_srv_query_done_cb(void *arg, int status, int timeouts, grpc_exec_ctx_finish(&exec_ctx); } +static const char g_service_config_attribute_prefix[] = "grpc_config="; + +static void on_txt_done_cb(void *arg, int status, int timeouts, + unsigned char *buf, int len) { + gpr_log(GPR_DEBUG, "on_txt_done_cb"); + char *error_msg; + grpc_ares_request *r = (grpc_ares_request *)arg; + gpr_mu_lock(&r->mu); + if (status != ARES_SUCCESS) goto fail; + struct ares_txt_ext *reply = NULL; + status = ares_parse_txt_reply_ext(buf, len, &reply); + if (status != ARES_SUCCESS) goto fail; + // Find service config in TXT record. + const size_t prefix_len = sizeof(g_service_config_attribute_prefix) - 1; + struct ares_txt_ext *result; + for (result = reply; result != NULL; result = result->next) { + if (result->record_start && + memcmp(result->txt, g_service_config_attribute_prefix, prefix_len) == + 0) { + break; + } + } + // Found a service config record. + if (result != NULL) { + size_t service_config_len = result->length - prefix_len; + *r->service_config_json_out = gpr_malloc(service_config_len + 1); + memcpy(*r->service_config_json_out, result->txt + prefix_len, + service_config_len); + for (result = result->next; result != NULL && !result->record_start; + result = result->next) { + *r->service_config_json_out = gpr_realloc( + *r->service_config_json_out, service_config_len + result->length + 1); + memcpy(*r->service_config_json_out + service_config_len, result->txt, + result->length); + service_config_len += result->length; + } + (*r->service_config_json_out)[service_config_len] = '\0'; + gpr_log(GPR_INFO, "found service config: %s", *r->service_config_json_out); + } + // Clean up. + ares_free_data(reply); + goto done; +fail: + gpr_asprintf(&error_msg, "C-ares TXT lookup status is not ARES_SUCCESS: %s", + ares_strerror(status)); + grpc_error *error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg); + gpr_free(error_msg); + if (r->error == GRPC_ERROR_NONE) { + r->error = error; + } else { + r->error = grpc_error_add_child(error, r->error); + } +done: + gpr_mu_unlock(&r->mu); + grpc_ares_request_unref(NULL, r); +} + static grpc_ares_request *grpc_dns_lookup_ares_impl( grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name, const char *default_port, grpc_pollset_set *interested_parties, - grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb) { + grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb, + char **service_config_json) { grpc_error *error = GRPC_ERROR_NONE; /* TODO(zyc): Enable tracing after #9603 is checked in */ /* if (grpc_dns_trace) { @@ -300,11 +360,12 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( error = grpc_ares_ev_driver_create(&ev_driver, interested_parties); if (error != GRPC_ERROR_NONE) goto error_cleanup; - grpc_ares_request *r = gpr_malloc(sizeof(grpc_ares_request)); + grpc_ares_request *r = gpr_zalloc(sizeof(grpc_ares_request)); gpr_mu_init(&r->mu); r->ev_driver = ev_driver; r->on_done = on_done; r->lb_addrs_out = addrs; + r->service_config_json_out = service_config_json; r->success = false; r->error = GRPC_ERROR_NONE; ares_channel *channel = grpc_ares_ev_driver_get_channel(r->ev_driver); @@ -315,13 +376,17 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( grpc_resolved_address addr; if (grpc_parse_ipv4_hostport(dns_server, &addr, false /* log_errors */)) { r->dns_server_addr.family = AF_INET; - memcpy(&r->dns_server_addr.addr.addr4, addr.addr, addr.len); + struct sockaddr_in *in = (struct sockaddr_in *)addr.addr; + memcpy(&r->dns_server_addr.addr.addr4, &in->sin_addr, + sizeof(struct in_addr)); r->dns_server_addr.tcp_port = grpc_sockaddr_get_port(&addr); r->dns_server_addr.udp_port = grpc_sockaddr_get_port(&addr); } else if (grpc_parse_ipv6_hostport(dns_server, &addr, false /* log_errors */)) { r->dns_server_addr.family = AF_INET6; - memcpy(&r->dns_server_addr.addr.addr6, addr.addr, addr.len); + struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)addr.addr; + memcpy(&r->dns_server_addr.addr.addr6, &in6->sin6_addr, + sizeof(struct in6_addr)); r->dns_server_addr.tcp_port = grpc_sockaddr_get_port(&addr); r->dns_server_addr.udp_port = grpc_sockaddr_get_port(&addr); } else { @@ -342,8 +407,6 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( goto error_cleanup; } } - // An extra reference is put here to avoid destroying the request in - // on_done_cb before calling grpc_ares_ev_driver_start. gpr_ref_init(&r->pending_queries, 1); if (grpc_ipv6_loopback_available()) { grpc_ares_hostbyname_request *hr = create_hostbyname_request( @@ -362,6 +425,10 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( r); gpr_free(service_name); } + if (service_config_json != NULL) { + grpc_ares_request_ref(r); + ares_search(*channel, hr->host, ns_c_in, ns_t_txt, on_txt_done_cb, r); + } /* TODO(zyc): Handle CNAME records here. */ grpc_ares_ev_driver_start(exec_ctx, r->ev_driver); grpc_ares_request_unref(exec_ctx, r); @@ -379,8 +446,8 @@ error_cleanup: grpc_ares_request *(*grpc_dns_lookup_ares)( grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name, const char *default_port, grpc_pollset_set *interested_parties, - grpc_closure *on_done, grpc_lb_addresses **addrs, - bool check_grpclb) = grpc_dns_lookup_ares_impl; + grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb, + char **service_config_json) = grpc_dns_lookup_ares_impl; void grpc_cancel_ares_request(grpc_exec_ctx *exec_ctx, grpc_ares_request *r) { if (grpc_dns_lookup_ares == grpc_dns_lookup_ares_impl) { @@ -465,7 +532,8 @@ static void grpc_resolve_address_ares_impl(grpc_exec_ctx *exec_ctx, grpc_schedule_on_exec_ctx); grpc_dns_lookup_ares(exec_ctx, NULL /* dns_server */, name, default_port, interested_parties, &r->on_dns_lookup_done, &r->lb_addrs, - false /* check_grpclb */); + false /* check_grpclb */, + NULL /* service_config_json */); } void (*grpc_resolve_address_ares)( diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h index 5d2d6c993bd..108333047d3 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h @@ -27,29 +27,30 @@ typedef struct grpc_ares_request grpc_ares_request; -/* Asynchronously resolve addr. Use \a default_port if a port isn't designated - in addr, otherwise use the port in addr. grpc_ares_init() must be called at - least once before this function. \a on_done may be called directly in this - function without being scheduled with \a exec_ctx, it must not try to acquire - locks that are being held by the caller. */ +/* Asynchronously resolve \a name. Use \a default_port if a port isn't + designated in \a name, otherwise use the port in \a name. grpc_ares_init() + must be called at least once before this function. \a on_done may be + called directly in this function without being scheduled with \a exec_ctx, + so it must not try to acquire locks that are being held by the caller. */ extern void (*grpc_resolve_address_ares)(grpc_exec_ctx *exec_ctx, - const char *addr, + const char *name, const char *default_port, grpc_pollset_set *interested_parties, grpc_closure *on_done, grpc_resolved_addresses **addresses); -/* Asynchronously resolve addr. It will try to resolve grpclb SRV records in +/* Asynchronously resolve \a name. It will try to resolve grpclb SRV records in addition to the normal address records. For normal address records, it uses - \a default_port if a port isn't designated in \a addr, otherwise it uses the - port in \a addr. grpc_ares_init() must be called at least once before this + \a default_port if a port isn't designated in \a name, otherwise it uses the + port in \a name. grpc_ares_init() must be called at least once before this function. \a on_done may be called directly in this function without being - scheduled with \a exec_ctx, it must not try to acquire locks that are being - held by the caller. */ + scheduled with \a exec_ctx, so it must not try to acquire locks that are + being held by the caller. */ extern grpc_ares_request *(*grpc_dns_lookup_ares)( - grpc_exec_ctx *exec_ctx, const char *dns_server, const char *addr, + grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name, const char *default_port, grpc_pollset_set *interested_parties, - grpc_closure *on_done, grpc_lb_addresses **addresses, bool check_grpclb); + grpc_closure *on_done, grpc_lb_addresses **addresses, bool check_grpclb, + char **service_config_json); /* Cancel the pending grpc_ares_request \a request */ void grpc_cancel_ares_request(grpc_exec_ctx *exec_ctx, diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c index b67636a3e44..f2587c4520f 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c @@ -28,15 +28,16 @@ struct grpc_ares_request { static grpc_ares_request *grpc_dns_lookup_ares_impl( grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name, const char *default_port, grpc_pollset_set *interested_parties, - grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb) { + grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb, + char **service_config_json) { return NULL; } grpc_ares_request *(*grpc_dns_lookup_ares)( grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name, const char *default_port, grpc_pollset_set *interested_parties, - grpc_closure *on_done, grpc_lb_addresses **addrs, - bool check_grpclb) = grpc_dns_lookup_ares_impl; + grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb, + char **service_config_json) = grpc_dns_lookup_ares_impl; void grpc_cancel_ares_request(grpc_exec_ctx *exec_ctx, grpc_ares_request *r) {} diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index aabe7b4a8ef..8976686082f 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1788,9 +1788,8 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, bool pending_data = s->pending_byte_stream || s->unprocessed_incoming_frames_buffer.length > 0; if (s->stream_compression_recv_enabled && s->read_closed && - s->frame_storage.length > 0 && - s->unprocessed_incoming_frames_buffer.length == 0 && !pending_data && - !s->seen_error && s->recv_trailing_metadata_finished != NULL) { + s->frame_storage.length > 0 && !pending_data && !s->seen_error && + s->recv_trailing_metadata_finished != NULL) { /* Maybe some SYNC_FLUSH data is left in frame_storage. Consume them and * maybe decompress the next 5 bytes in the stream. */ bool end_of_context; @@ -1817,7 +1816,6 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, } } if (s->read_closed && s->frame_storage.length == 0 && - s->unprocessed_incoming_frames_buffer.length == 0 && (!pending_data || s->seen_error) && s->recv_trailing_metadata_finished != NULL) { grpc_chttp2_incoming_metadata_buffer_publish( diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 776e0765fe3..abb558982bc 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -637,7 +637,8 @@ static void on_response_trailers_received( Utility function that takes the data from s->write_slice_buffer and assembles into a contiguous byte stream with 5 byte gRPC header prepended. */ -static void create_grpc_frame(grpc_slice_buffer *write_slice_buffer, +static void create_grpc_frame(grpc_exec_ctx *exec_ctx, + grpc_slice_buffer *write_slice_buffer, char **pp_write_buffer, size_t *p_write_buffer_size, uint32_t flags) { grpc_slice slice = grpc_slice_buffer_take_first(write_slice_buffer); @@ -657,6 +658,7 @@ static void create_grpc_frame(grpc_slice_buffer *write_slice_buffer, *p++ = (uint8_t)(length); /* append actual data */ memcpy(p, GRPC_SLICE_START_PTR(slice), length); + grpc_slice_unref_internal(exec_ctx, slice); } /* @@ -1017,14 +1019,15 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, } if (write_slice_buffer.count > 0) { size_t write_buffer_size; - create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, - &write_buffer_size, + create_grpc_frame(exec_ctx, &write_slice_buffer, + &stream_state->ws.write_buffer, &write_buffer_size, stream_op->payload->send_message.send_message->flags); CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, %p)", s->cbs, stream_state->ws.write_buffer); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, (int)write_buffer_size, false); + grpc_slice_buffer_destroy_internal(exec_ctx, &write_slice_buffer); if (t->use_packet_coalescing) { if (!stream_op->send_trailing_metadata) { CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); @@ -1153,6 +1156,9 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, } else { stream_state->rs.remaining_bytes = 0; CRONET_LOG(GPR_DEBUG, "read operation complete. Empty response."); + /* Clean up read_slice_buffer in case there is unread data. */ + grpc_slice_buffer_destroy_internal( + exec_ctx, &stream_state->rs.read_slice_buffer); grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer); grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); @@ -1206,6 +1212,9 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, memcpy(dst_p, stream_state->rs.read_buffer, (size_t)stream_state->rs.length_field); null_and_maybe_free_read_buffer(s); + /* Clean up read_slice_buffer in case there is unread data. */ + grpc_slice_buffer_destroy_internal(exec_ctx, + &stream_state->rs.read_slice_buffer); grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer); grpc_slice_buffer_add(&stream_state->rs.read_slice_buffer, read_data_slice); @@ -1369,6 +1378,8 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_closure *then_schedule_closure) { stream_obj *s = (stream_obj *)gs; null_and_maybe_free_read_buffer(s); + /* Clean up read_slice_buffer in case there is unread data. */ + grpc_slice_buffer_destroy_internal(exec_ctx, &s->state.rs.read_slice_buffer); GRPC_ERROR_UNREF(s->state.cancel_error); GRPC_CLOSURE_SCHED(exec_ctx, then_schedule_closure, GRPC_ERROR_NONE); } diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index dc48d73df90..90e0ce36cd7 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -237,28 +237,41 @@ static grpc_fd *fd_create(int fd, const char *name) { static int fd_wrapped_fd(grpc_fd *fd) { return fd->fd; } -/* Might be called multiple times */ -static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { +/* if 'releasing_fd' is true, it means that we are going to detach the internal + * fd from grpc_fd structure (i.e which means we should not be calling + * shutdown() syscall on that fd) */ +static void fd_shutdown_internal(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_error *why, bool releasing_fd) { if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, GRPC_ERROR_REF(why))) { - shutdown(fd->fd, SHUT_RDWR); + if (!releasing_fd) { + shutdown(fd->fd, SHUT_RDWR); + } grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); } GRPC_ERROR_UNREF(why); } +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + fd_shutdown_internal(exec_ctx, fd, why, false); +} + static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done, int *release_fd, bool already_closed, const char *reason) { grpc_error *error = GRPC_ERROR_NONE; + bool is_release_fd = (release_fd != NULL); if (!grpc_lfev_is_shutdown(&fd->read_closure)) { - fd_shutdown(exec_ctx, fd, GRPC_ERROR_CREATE_FROM_COPIED_STRING(reason)); + fd_shutdown_internal(exec_ctx, fd, + GRPC_ERROR_CREATE_FROM_COPIED_STRING(reason), + is_release_fd); } /* If release_fd is not NULL, we should be relinquishing control of the file descriptor fd->fd (but we still own the grpc_fd structure). */ - if (release_fd != NULL) { + if (is_release_fd) { *release_fd = fd->fd; } else if (!already_closed) { close(fd->fd); diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 365aa583bb8..9472a8e5208 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -42,6 +42,7 @@ #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" +#include "src/core/lib/support/murmur_hash.h" #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) @@ -239,22 +240,43 @@ struct grpc_pollset_set { * condition variable polling definitions */ +#define POLLCV_THREAD_GRACE_MS 1000 #define CV_POLL_PERIOD_MS 1000 #define CV_DEFAULT_TABLE_SIZE 16 -typedef enum poll_status_t { INPROGRESS, COMPLETED, CANCELLED } poll_status_t; - -typedef struct poll_args { +typedef struct poll_result { gpr_refcount refcount; - gpr_cv *cv; + cv_node *watchers; + int watchcount; struct pollfd *fds; nfds_t nfds; - int timeout; int retval; int err; - gpr_atm status; + int completed; +} poll_result; + +typedef struct poll_args { + gpr_cv trigger; + int trigger_set; + struct pollfd *fds; + nfds_t nfds; + poll_result *result; + struct poll_args *next; + struct poll_args *prev; } poll_args; +// This is a 2-tiered cache, we mantain a hash table +// of active poll calls, so we can wait on the result +// of that call. We also maintain a freelist of inactive +// poll threads. +typedef struct poll_hash_table { + poll_args *free_pollers; + poll_args **active_pollers; + unsigned int size; + unsigned int count; +} poll_hash_table; + +poll_hash_table poll_cache; cv_fd_table g_cvfds; /******************************************************************************* @@ -1277,43 +1299,205 @@ static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, * Condition Variable polling extensions */ -static void decref_poll_args(poll_args *args) { - if (gpr_unref(&args->refcount)) { - gpr_free(args->fds); - gpr_cv_destroy(args->cv); - gpr_free(args->cv); - gpr_free(args); +static void run_poll(void *args); +static void cache_poller_locked(poll_args *args); + +static void cache_insert_locked(poll_args *args) { + uint32_t key = gpr_murmur_hash3(args->fds, args->nfds * sizeof(struct pollfd), + 0xDEADBEEF); + key = key % poll_cache.size; + if (poll_cache.active_pollers[key]) { + poll_cache.active_pollers[key]->prev = args; } + args->next = poll_cache.active_pollers[key]; + args->prev = NULL; + poll_cache.active_pollers[key] = args; + poll_cache.count++; } -// Poll in a background thread -static void run_poll(void *arg) { - int timeout, retval; - poll_args *pargs = (poll_args *)arg; - while (gpr_atm_no_barrier_load(&pargs->status) == INPROGRESS) { - if (pargs->timeout < 0) { - timeout = CV_POLL_PERIOD_MS; - } else { - timeout = GPR_MIN(CV_POLL_PERIOD_MS, pargs->timeout); - pargs->timeout -= timeout; +static void init_result(poll_args *pargs) { + pargs->result = gpr_malloc(sizeof(poll_result)); + gpr_ref_init(&pargs->result->refcount, 1); + pargs->result->watchers = NULL; + pargs->result->watchcount = 0; + pargs->result->fds = gpr_malloc(sizeof(struct pollfd) * pargs->nfds); + memcpy(pargs->result->fds, pargs->fds, sizeof(struct pollfd) * pargs->nfds); + pargs->result->nfds = pargs->nfds; + pargs->result->retval = 0; + pargs->result->err = 0; + pargs->result->completed = 0; +} + +// Creates a poll_args object for a given arguments to poll(). +// This object may return a poll_args in the cache. +static poll_args *get_poller_locked(struct pollfd *fds, nfds_t count) { + uint32_t key = + gpr_murmur_hash3(fds, count * sizeof(struct pollfd), 0xDEADBEEF); + key = key % poll_cache.size; + poll_args *curr = poll_cache.active_pollers[key]; + while (curr) { + if (curr->nfds == count && + memcmp(curr->fds, fds, count * sizeof(struct pollfd)) == 0) { + gpr_free(fds); + return curr; } - retval = g_cvfds.poll(pargs->fds, pargs->nfds, timeout); - if (retval != 0 || pargs->timeout == 0) { - pargs->retval = retval; - pargs->err = errno; - break; + curr = curr->next; + } + + if (poll_cache.free_pollers) { + poll_args *pargs = poll_cache.free_pollers; + poll_cache.free_pollers = pargs->next; + if (poll_cache.free_pollers) { + poll_cache.free_pollers->prev = NULL; } + pargs->fds = fds; + pargs->nfds = count; + pargs->next = NULL; + pargs->prev = NULL; + init_result(pargs); + cache_poller_locked(pargs); + return pargs; + } + + poll_args *pargs = gpr_malloc(sizeof(struct poll_args)); + gpr_cv_init(&pargs->trigger); + pargs->fds = fds; + pargs->nfds = count; + pargs->next = NULL; + pargs->prev = NULL; + pargs->trigger_set = 0; + init_result(pargs); + cache_poller_locked(pargs); + gpr_thd_id t_id; + gpr_thd_options opt = gpr_thd_options_default(); + gpr_ref(&g_cvfds.pollcount); + gpr_thd_options_set_detached(&opt); + GPR_ASSERT(gpr_thd_new(&t_id, &run_poll, pargs, &opt)); + return pargs; +} + +static void cache_delete_locked(poll_args *args) { + if (!args->prev) { + uint32_t key = gpr_murmur_hash3( + args->fds, args->nfds * sizeof(struct pollfd), 0xDEADBEEF); + key = key % poll_cache.size; + GPR_ASSERT(poll_cache.active_pollers[key] == args); + poll_cache.active_pollers[key] = args->next; + } else { + args->prev->next = args->next; } - gpr_mu_lock(&g_cvfds.mu); - if (gpr_atm_no_barrier_load(&pargs->status) == INPROGRESS) { - // Signal main thread that the poll completed - gpr_atm_no_barrier_store(&pargs->status, COMPLETED); - gpr_cv_signal(pargs->cv); + + if (args->next) { + args->next->prev = args->prev; } - decref_poll_args(pargs); - g_cvfds.pollcount--; - if (g_cvfds.shutdown && g_cvfds.pollcount == 0) { - gpr_cv_signal(&g_cvfds.shutdown_complete); + + poll_cache.count--; + if (poll_cache.free_pollers) { + poll_cache.free_pollers->prev = args; + } + args->prev = NULL; + args->next = poll_cache.free_pollers; + gpr_free(args->fds); + poll_cache.free_pollers = args; +} + +static void cache_poller_locked(poll_args *args) { + if (poll_cache.count + 1 > poll_cache.size / 2) { + poll_args **old_active_pollers = poll_cache.active_pollers; + poll_cache.size = poll_cache.size * 2; + poll_cache.count = 0; + poll_cache.active_pollers = gpr_malloc(sizeof(void *) * poll_cache.size); + for (unsigned int i = 0; i < poll_cache.size; i++) { + poll_cache.active_pollers[i] = NULL; + } + for (unsigned int i = 0; i < poll_cache.size / 2; i++) { + poll_args *curr = old_active_pollers[i]; + poll_args *next = NULL; + while (curr) { + next = curr->next; + cache_insert_locked(curr); + curr = next; + } + } + gpr_free(old_active_pollers); + } + + cache_insert_locked(args); +} + +static void cache_destroy_locked(poll_args *args) { + if (args->next) { + args->next->prev = args->prev; + } + + if (args->prev) { + args->prev->next = args->next; + } else { + poll_cache.free_pollers = args->next; + } + + gpr_free(args); +} + +static void decref_poll_result(poll_result *res) { + if (gpr_unref(&res->refcount)) { + GPR_ASSERT(!res->watchers); + gpr_free(res->fds); + gpr_free(res); + } +} + +void remove_cvn(cv_node **head, cv_node *target) { + if (target->next) { + target->next->prev = target->prev; + } + + if (target->prev) { + target->prev->next = target->next; + } else { + *head = target->next; + } +} + +gpr_timespec thread_grace; + +// Poll in a background thread +static void run_poll(void *args) { + poll_args *pargs = (poll_args *)args; + while (1) { + poll_result *result = pargs->result; + int retval = g_cvfds.poll(result->fds, result->nfds, CV_POLL_PERIOD_MS); + gpr_mu_lock(&g_cvfds.mu); + if (retval != 0) { + result->completed = 1; + result->retval = retval; + result->err = errno; + cv_node *watcher = result->watchers; + while (watcher) { + gpr_cv_signal(watcher->cv); + watcher = watcher->next; + } + } + if (result->watchcount == 0 || result->completed) { + cache_delete_locked(pargs); + decref_poll_result(result); + // Leave this polling thread alive for a grace period to do another poll() + // op + gpr_timespec deadline = gpr_now(GPR_CLOCK_REALTIME); + deadline = gpr_time_add(deadline, thread_grace); + pargs->trigger_set = 0; + gpr_cv_wait(&pargs->trigger, &g_cvfds.mu, deadline); + if (!pargs->trigger_set) { + cache_destroy_locked(pargs); + break; + } + } + gpr_mu_unlock(&g_cvfds.mu); + } + + // We still have the lock here + if (gpr_unref(&g_cvfds.pollcount)) { + gpr_cv_signal(&g_cvfds.shutdown_cv); } gpr_mu_unlock(&g_cvfds.mu); } @@ -1322,24 +1506,29 @@ static void run_poll(void *arg) { static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) { unsigned int i; int res, idx; - gpr_cv *pollcv; - cv_node *cvn, *prev; + cv_node *pollcv; int skip_poll = 0; nfds_t nsockfds = 0; - gpr_thd_id t_id; - gpr_thd_options opt; - poll_args *pargs = NULL; + poll_result *result = NULL; gpr_mu_lock(&g_cvfds.mu); - pollcv = gpr_malloc(sizeof(gpr_cv)); - gpr_cv_init(pollcv); + pollcv = gpr_malloc(sizeof(cv_node)); + pollcv->next = NULL; + gpr_cv pollcv_cv; + gpr_cv_init(&pollcv_cv); + pollcv->cv = &pollcv_cv; + cv_node *fd_cvs = gpr_malloc(nfds * sizeof(cv_node)); + for (i = 0; i < nfds; i++) { fds[i].revents = 0; if (fds[i].fd < 0 && (fds[i].events & POLLIN)) { idx = FD_TO_IDX(fds[i].fd); - cvn = gpr_malloc(sizeof(cv_node)); - cvn->cv = pollcv; - cvn->next = g_cvfds.cvfds[idx].cvs; - g_cvfds.cvfds[idx].cvs = cvn; + fd_cvs[i].cv = &pollcv_cv; + fd_cvs[i].prev = NULL; + fd_cvs[i].next = g_cvfds.cvfds[idx].cvs; + if (g_cvfds.cvfds[idx].cvs) { + g_cvfds.cvfds[idx].cvs->prev = &(fd_cvs[i]); + } + g_cvfds.cvfds[idx].cvs = &(fd_cvs[i]); // Don't bother polling if a wakeup fd is ready if (g_cvfds.cvfds[idx].is_set) { skip_poll = 1; @@ -1349,81 +1538,68 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) { } } + gpr_timespec deadline = gpr_now(GPR_CLOCK_REALTIME); + if (timeout < 0) { + deadline = gpr_inf_future(GPR_CLOCK_REALTIME); + } else { + deadline = + gpr_time_add(deadline, gpr_time_from_millis(timeout, GPR_TIMESPAN)); + } + res = 0; if (!skip_poll && nsockfds > 0) { - pargs = gpr_malloc(sizeof(struct poll_args)); - // Both the main thread and calling thread get a reference - gpr_ref_init(&pargs->refcount, 2); - pargs->cv = pollcv; - pargs->fds = gpr_malloc(sizeof(struct pollfd) * nsockfds); - pargs->nfds = nsockfds; - pargs->timeout = timeout; - pargs->retval = 0; - pargs->err = 0; - gpr_atm_no_barrier_store(&pargs->status, INPROGRESS); + struct pollfd *pollfds = gpr_malloc(sizeof(struct pollfd) * nsockfds); idx = 0; for (i = 0; i < nfds; i++) { if (fds[i].fd >= 0) { - pargs->fds[idx].fd = fds[i].fd; - pargs->fds[idx].events = fds[i].events; - pargs->fds[idx].revents = 0; + pollfds[idx].fd = fds[i].fd; + pollfds[idx].events = fds[i].events; + pollfds[idx].revents = 0; idx++; } } - g_cvfds.pollcount++; - opt = gpr_thd_options_default(); - gpr_thd_options_set_detached(&opt); - GPR_ASSERT(gpr_thd_new(&t_id, &run_poll, pargs, &opt)); - // We want the poll() thread to trigger the deadline, so wait forever here - gpr_cv_wait(pollcv, &g_cvfds.mu, gpr_inf_future(GPR_CLOCK_MONOTONIC)); - if (gpr_atm_no_barrier_load(&pargs->status) == COMPLETED) { - res = pargs->retval; - errno = pargs->err; - } else { - errno = 0; - gpr_atm_no_barrier_store(&pargs->status, CANCELLED); + poll_args *pargs = get_poller_locked(pollfds, nsockfds); + result = pargs->result; + pollcv->next = result->watchers; + pollcv->prev = NULL; + if (result->watchers) { + result->watchers->prev = pollcv; } + result->watchers = pollcv; + result->watchcount++; + gpr_ref(&result->refcount); + + pargs->trigger_set = 1; + gpr_cv_signal(&pargs->trigger); + gpr_cv_wait(&pollcv_cv, &g_cvfds.mu, deadline); + res = result->retval; + errno = result->err; + result->watchcount--; + remove_cvn(&result->watchers, pollcv); } else if (!skip_poll) { - gpr_timespec deadline = gpr_now(GPR_CLOCK_REALTIME); - deadline = - gpr_time_add(deadline, gpr_time_from_millis(timeout, GPR_TIMESPAN)); - gpr_cv_wait(pollcv, &g_cvfds.mu, deadline); + gpr_cv_wait(&pollcv_cv, &g_cvfds.mu, deadline); } idx = 0; for (i = 0; i < nfds; i++) { if (fds[i].fd < 0 && (fds[i].events & POLLIN)) { - cvn = g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs; - prev = NULL; - while (cvn->cv != pollcv) { - prev = cvn; - cvn = cvn->next; - GPR_ASSERT(cvn); - } - if (!prev) { - g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs = cvn->next; - } else { - prev->next = cvn->next; - } - gpr_free(cvn); - + remove_cvn(&g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs, &(fd_cvs[i])); if (g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].is_set) { fds[i].revents = POLLIN; if (res >= 0) res++; } - } else if (!skip_poll && fds[i].fd >= 0 && - gpr_atm_no_barrier_load(&pargs->status) == COMPLETED) { - fds[i].revents = pargs->fds[idx].revents; + } else if (!skip_poll && fds[i].fd >= 0 && result->completed) { + fds[i].revents = result->fds[idx].revents; idx++; } } - if (pargs) { - decref_poll_args(pargs); - } else { - gpr_cv_destroy(pollcv); - gpr_free(pollcv); + gpr_free(fd_cvs); + gpr_free(pollcv); + if (result) { + decref_poll_result(result); } + gpr_mu_unlock(&g_cvfds.mu); return res; @@ -1432,12 +1608,12 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) { static void global_cv_fd_table_init() { gpr_mu_init(&g_cvfds.mu); gpr_mu_lock(&g_cvfds.mu); - gpr_cv_init(&g_cvfds.shutdown_complete); - g_cvfds.shutdown = 0; - g_cvfds.pollcount = 0; + gpr_cv_init(&g_cvfds.shutdown_cv); + gpr_ref_init(&g_cvfds.pollcount, 1); g_cvfds.size = CV_DEFAULT_TABLE_SIZE; g_cvfds.cvfds = gpr_malloc(sizeof(fd_node) * CV_DEFAULT_TABLE_SIZE); g_cvfds.free_fds = NULL; + thread_grace = gpr_time_from_millis(POLLCV_THREAD_GRACE_MS, GPR_TIMESPAN); for (int i = 0; i < CV_DEFAULT_TABLE_SIZE; i++) { g_cvfds.cvfds[i].is_set = 0; g_cvfds.cvfds[i].cvs = NULL; @@ -1447,23 +1623,35 @@ static void global_cv_fd_table_init() { // Override the poll function with one that supports cvfds g_cvfds.poll = grpc_poll_function; grpc_poll_function = &cvfd_poll; + + // Initialize the cache + poll_cache.size = 32; + poll_cache.count = 0; + poll_cache.free_pollers = NULL; + poll_cache.active_pollers = gpr_malloc(sizeof(void *) * 32); + for (unsigned int i = 0; i < poll_cache.size; i++) { + poll_cache.active_pollers[i] = NULL; + } + gpr_mu_unlock(&g_cvfds.mu); } static void global_cv_fd_table_shutdown() { gpr_mu_lock(&g_cvfds.mu); - g_cvfds.shutdown = 1; // Attempt to wait for all abandoned poll() threads to terminate // Not doing so will result in reported memory leaks - if (g_cvfds.pollcount > 0) { - int res = gpr_cv_wait(&g_cvfds.shutdown_complete, &g_cvfds.mu, + if (!gpr_unref(&g_cvfds.pollcount)) { + int res = gpr_cv_wait(&g_cvfds.shutdown_cv, &g_cvfds.mu, gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_seconds(3, GPR_TIMESPAN))); GPR_ASSERT(res == 0); } - gpr_cv_destroy(&g_cvfds.shutdown_complete); + gpr_cv_destroy(&g_cvfds.shutdown_cv); grpc_poll_function = g_cvfds.poll; gpr_free(g_cvfds.cvfds); + + gpr_free(poll_cache.active_pollers); + gpr_mu_unlock(&g_cvfds.mu); gpr_mu_destroy(&g_cvfds.mu); } diff --git a/src/core/lib/iomgr/exec_ctx.c b/src/core/lib/iomgr/exec_ctx.c index 833170ceed5..41c69add17c 100644 --- a/src/core/lib/iomgr/exec_ctx.c +++ b/src/core/lib/iomgr/exec_ctx.c @@ -51,33 +51,6 @@ bool grpc_exec_ctx_has_work(grpc_exec_ctx *exec_ctx) { !grpc_closure_list_empty(exec_ctx->closure_list); } -bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { - bool did_something = 0; - GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0); - for (;;) { - if (!grpc_closure_list_empty(exec_ctx->closure_list)) { - grpc_closure *c = exec_ctx->closure_list.head; - exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL; - while (c != NULL) { - grpc_closure *next = c->next_data.next; - grpc_error *error = c->error_data.error; - did_something = true; -#ifndef NDEBUG - c->scheduled = false; -#endif - c->cb(exec_ctx, c->cb_arg, error); - GRPC_ERROR_UNREF(error); - c = next; - } - } else if (!grpc_combiner_continue_exec_ctx(exec_ctx)) { - break; - } - } - GPR_ASSERT(exec_ctx->active_combiner == NULL); - GPR_TIMER_END("grpc_exec_ctx_flush", 0); - return did_something; -} - void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) { exec_ctx->flags |= GRPC_EXEC_CTX_FLAG_IS_FINISHED; grpc_exec_ctx_flush(exec_ctx); @@ -103,6 +76,29 @@ static void exec_ctx_run(grpc_exec_ctx *exec_ctx, grpc_closure *closure, GRPC_ERROR_UNREF(error); } +bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { + bool did_something = 0; + GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0); + for (;;) { + if (!grpc_closure_list_empty(exec_ctx->closure_list)) { + grpc_closure *c = exec_ctx->closure_list.head; + exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL; + while (c != NULL) { + grpc_closure *next = c->next_data.next; + grpc_error *error = c->error_data.error; + did_something = true; + exec_ctx_run(exec_ctx, c, error); + c = next; + } + } else if (!grpc_combiner_continue_exec_ctx(exec_ctx)) { + break; + } + } + GPR_ASSERT(exec_ctx->active_combiner == NULL); + GPR_TIMER_END("grpc_exec_ctx_flush", 0); + return did_something; +} + static void exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_error *error) { grpc_closure_list_append(&exec_ctx->closure_list, closure, error); diff --git a/src/core/lib/iomgr/gethostname.h b/src/core/lib/iomgr/gethostname.h new file mode 100644 index 00000000000..9c6b9d8d420 --- /dev/null +++ b/src/core/lib/iomgr/gethostname.h @@ -0,0 +1,26 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_GETHOSTNAME_H +#define GRPC_CORE_LIB_IOMGR_GETHOSTNAME_H + +// Returns the hostname of the local machine. +// Caller takes ownership of result. +char *grpc_gethostname(); + +#endif /* GRPC_CORE_LIB_IOMGR_GETHOSTNAME_H */ diff --git a/src/core/lib/iomgr/gethostname_fallback.c b/src/core/lib/iomgr/gethostname_fallback.c new file mode 100644 index 00000000000..6229461568b --- /dev/null +++ b/src/core/lib/iomgr/gethostname_fallback.c @@ -0,0 +1,27 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "src/core/lib/iomgr/port.h" + +#ifdef GRPC_GETHOSTNAME_FALLBACK + +#include + +char *grpc_gethostname() { return NULL; } + +#endif // GRPC_GETHOSTNAME_FALLBACK diff --git a/src/core/lib/iomgr/gethostname_host_name_max.c b/src/core/lib/iomgr/gethostname_host_name_max.c new file mode 100644 index 00000000000..4d0511412ec --- /dev/null +++ b/src/core/lib/iomgr/gethostname_host_name_max.c @@ -0,0 +1,37 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "src/core/lib/iomgr/port.h" + +#ifdef GRPC_POSIX_HOST_NAME_MAX + +#include +#include + +#include + +char *grpc_gethostname() { + char *hostname = (char *)gpr_malloc(HOST_NAME_MAX); + if (gethostname(hostname, HOST_NAME_MAX) != 0) { + gpr_free(hostname); + return NULL; + } + return hostname; +} + +#endif // GRPC_POSIX_HOST_NAME_MAX diff --git a/src/core/lib/iomgr/gethostname_sysconf.c b/src/core/lib/iomgr/gethostname_sysconf.c new file mode 100644 index 00000000000..51bac5d69dd --- /dev/null +++ b/src/core/lib/iomgr/gethostname_sysconf.c @@ -0,0 +1,37 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "src/core/lib/iomgr/port.h" + +#ifdef GRPC_POSIX_SYSCONF + +#include + +#include + +char *grpc_gethostname() { + size_t host_name_max = (size_t)sysconf(_SC_HOST_NAME_MAX); + char *hostname = (char *)gpr_malloc(host_name_max); + if (gethostname(hostname, host_name_max) != 0) { + gpr_free(hostname); + return NULL; + } + return hostname; +} + +#endif // GRPC_POSIX_SYSCONF diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h index c12058f890b..42033d0ba4b 100644 --- a/src/core/lib/iomgr/port.h +++ b/src/core/lib/iomgr/port.h @@ -59,6 +59,7 @@ #define GRPC_HAVE_MSG_NOSIGNAL 1 #define GRPC_HAVE_UNIX_SOCKET 1 #define GRPC_LINUX_MULTIPOLL_WITH_EPOLL 1 +#define GRPC_POSIX_HOST_NAME_MAX 1 #define GRPC_POSIX_SOCKET 1 #define GRPC_POSIX_SOCKETADDR 1 #define GRPC_POSIX_WAKEUP_FD 1 @@ -93,6 +94,7 @@ #define GRPC_POSIX_SOCKET 1 #define GRPC_POSIX_SOCKETADDR 1 #define GRPC_POSIX_SOCKETUTILS 1 +#define GRPC_POSIX_SYSCONF 1 #define GRPC_POSIX_WAKEUP_FD 1 #define GRPC_TIMER_USE_GENERIC 1 #elif defined(GPR_FREEBSD) @@ -125,4 +127,11 @@ #error Must define exactly one of GRPC_POSIX_SOCKET, GRPC_WINSOCK_SOCKET, GPR_CUSTOM_SOCKET #endif +#if defined(GRPC_POSIX_HOST_NAME_MAX) && defined(GRPC_POSIX_SYSCONF) +#error "Cannot define both GRPC_POSIX_HOST_NAME_MAX and GRPC_POSIX_SYSCONF" +#endif +#if !defined(GRPC_POSIX_HOST_NAME_MAX) && !defined(GRPC_POSIX_SYSCONF) +#define GRPC_GETHOSTNAME_FALLBACK 1 +#endif + #endif /* GRPC_CORE_LIB_IOMGR_PORT_H */ diff --git a/src/core/lib/iomgr/tcp_server_utils_posix_common.c b/src/core/lib/iomgr/tcp_server_utils_posix_common.c index dbb43186bde..ad535bc43ea 100644 --- a/src/core/lib/iomgr/tcp_server_utils_posix_common.c +++ b/src/core/lib/iomgr/tcp_server_utils_posix_common.c @@ -39,7 +39,7 @@ #define MIN_SAFE_ACCEPT_QUEUE_SIZE 100 -static gpr_once s_init_max_accept_queue_size; +static gpr_once s_init_max_accept_queue_size = GPR_ONCE_INIT; static int s_max_accept_queue_size; /* get max listen queue size on linux */ diff --git a/src/core/lib/iomgr/wakeup_fd_cv.h b/src/core/lib/iomgr/wakeup_fd_cv.h index c5dcdc9746a..46e84f58437 100644 --- a/src/core/lib/iomgr/wakeup_fd_cv.h +++ b/src/core/lib/iomgr/wakeup_fd_cv.h @@ -43,6 +43,7 @@ typedef struct cv_node { gpr_cv* cv; struct cv_node* next; + struct cv_node* prev; } cv_node; typedef struct fd_node { @@ -53,9 +54,8 @@ typedef struct fd_node { typedef struct cv_fd_table { gpr_mu mu; - int pollcount; - int shutdown; - gpr_cv shutdown_complete; + gpr_refcount pollcount; + gpr_cv shutdown_cv; fd_node* cvfds; fd_node* free_fds; unsigned int size; diff --git a/src/core/lib/security/transport/security_handshaker.c b/src/core/lib/security/transport/security_handshaker.c index b9da6e16b25..fc9c9f980f0 100644 --- a/src/core/lib/security/transport/security_handshaker.c +++ b/src/core/lib/security/transport/security_handshaker.c @@ -261,7 +261,7 @@ static grpc_error *do_handshaker_next_locked( grpc_exec_ctx *exec_ctx, security_handshaker *h, const unsigned char *bytes_received, size_t bytes_received_size) { // Invoke TSI handshaker. - unsigned char *bytes_to_send = NULL; + const unsigned char *bytes_to_send = NULL; size_t bytes_to_send_size = 0; tsi_handshaker_result *handshaker_result = NULL; tsi_result result = tsi_handshaker_next( diff --git a/src/core/lib/surface/alarm.c b/src/core/lib/surface/alarm.c index 55934964f3a..7d60b1de17e 100644 --- a/src/core/lib/surface/alarm.c +++ b/src/core/lib/surface/alarm.c @@ -15,6 +15,7 @@ * limitations under the License. * */ +#include "src/core/lib/surface/alarm_internal.h" #include #include @@ -22,7 +23,13 @@ #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/surface/completion_queue.h" +#ifndef NDEBUG +grpc_tracer_flag grpc_trace_alarm_refcount = + GRPC_TRACER_INITIALIZER(false, "alarm_refcount"); +#endif + struct grpc_alarm { + gpr_refcount refs; grpc_timer alarm; grpc_closure on_alarm; grpc_cq_completion completion; @@ -32,13 +39,58 @@ struct grpc_alarm { void *tag; }; -static void do_nothing_end_completion(grpc_exec_ctx *exec_ctx, void *arg, - grpc_cq_completion *c) {} +static void alarm_ref(grpc_alarm *alarm) { gpr_ref(&alarm->refs); } + +static void alarm_unref(grpc_alarm *alarm) { + if (gpr_unref(&alarm->refs)) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, alarm->cq, "alarm"); + grpc_exec_ctx_finish(&exec_ctx); + gpr_free(alarm); + } +} + +#ifndef NDEBUG +static void alarm_ref_dbg(grpc_alarm *alarm, const char *reason, + const char *file, int line) { + if (GRPC_TRACER_ON(grpc_trace_alarm_refcount)) { + gpr_atm val = gpr_atm_no_barrier_load(&alarm->refs.count); + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, + "Alarm:%p ref %" PRIdPTR " -> %" PRIdPTR " %s", alarm, val, + val + 1, reason); + } + + alarm_ref(alarm); +} + +static void alarm_unref_dbg(grpc_alarm *alarm, const char *reason, + const char *file, int line) { + if (GRPC_TRACER_ON(grpc_trace_alarm_refcount)) { + gpr_atm val = gpr_atm_no_barrier_load(&alarm->refs.count); + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, + "Alarm:%p Unref %" PRIdPTR " -> %" PRIdPTR " %s", alarm, val, + val - 1, reason); + } + + alarm_unref(alarm); +} +#endif + +static void alarm_end_completion(grpc_exec_ctx *exec_ctx, void *arg, + grpc_cq_completion *c) { + grpc_alarm *alarm = arg; + GRPC_ALARM_UNREF(alarm, "dequeue-end-op"); +} static void alarm_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_alarm *alarm = arg; - grpc_cq_end_op(exec_ctx, alarm->cq, alarm->tag, error, - do_nothing_end_completion, NULL, &alarm->completion); + + /* We are queuing an op on completion queue. This means, the alarm's structure + cannot be destroyed until the op is dequeued. Adding an extra ref + here and unref'ing when the op is dequeued will achieve this */ + GRPC_ALARM_REF(alarm, "queue-end-op"); + grpc_cq_end_op(exec_ctx, alarm->cq, alarm->tag, error, alarm_end_completion, + (void *)alarm, &alarm->completion); } grpc_alarm *grpc_alarm_create(grpc_completion_queue *cq, gpr_timespec deadline, @@ -46,6 +98,14 @@ grpc_alarm *grpc_alarm_create(grpc_completion_queue *cq, gpr_timespec deadline, grpc_alarm *alarm = gpr_malloc(sizeof(grpc_alarm)); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_ref_init(&alarm->refs, 1); + +#ifndef NDEBUG + if (GRPC_TRACER_ON(grpc_trace_alarm_refcount)) { + gpr_log(GPR_DEBUG, "Alarm:%p created (ref: 1)", alarm); + } +#endif + GRPC_CQ_INTERNAL_REF(cq, "alarm"); alarm->cq = cq; alarm->tag = tag; @@ -67,9 +127,6 @@ void grpc_alarm_cancel(grpc_alarm *alarm) { } void grpc_alarm_destroy(grpc_alarm *alarm) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_alarm_cancel(alarm); - GRPC_CQ_INTERNAL_UNREF(&exec_ctx, alarm->cq, "alarm"); - gpr_free(alarm); - grpc_exec_ctx_finish(&exec_ctx); + GRPC_ALARM_UNREF(alarm, "alarm_destroy"); } diff --git a/src/core/lib/surface/alarm_internal.h b/src/core/lib/surface/alarm_internal.h new file mode 100644 index 00000000000..7f2126c5c91 --- /dev/null +++ b/src/core/lib/surface/alarm_internal.h @@ -0,0 +1,40 @@ +/* + * + * Copyright 2015-2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_SURFACE_ALARM_INTERNAL_H +#define GRPC_CORE_LIB_SURFACE_ALARM_INTERNAL_H + +#include +#include "src/core/lib/debug/trace.h" + +#ifndef NDEBUG + +extern grpc_tracer_flag grpc_trace_alarm_refcount; + +#define GRPC_ALARM_REF(a, reason) alarm_ref_dbg(a, reason, __FILE__, __LINE__) +#define GRPC_ALARM_UNREF(a, reason) \ + alarm_unref_dbg(a, reason, __FILE__, __LINE__) + +#else /* !defined(NDEBUG) */ + +#define GRPC_ALARM_REF(a, reason) alarm_ref(a) +#define GRPC_ALARM_UNREF(a, reason) alarm_unref(a) + +#endif /* defined(NDEBUG) */ + +#endif /* GRPC_CORE_LIB_SURFACE_ALARM_INTERNAL_H */ diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 3d82a32e82f..c20cfbc7403 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -235,7 +235,8 @@ typedef struct cq_next_data { /* Number of outstanding events (+1 if not shut down) */ gpr_atm pending_events; - int shutdown_called; + /** 0 initially. 1 once we initiated shutdown */ + bool shutdown_called; } cq_next_data; typedef struct cq_pluck_data { @@ -244,15 +245,20 @@ typedef struct cq_pluck_data { grpc_cq_completion *completed_tail; /** Number of pending events (+1 if we're not shutdown) */ - gpr_refcount pending_events; + gpr_atm pending_events; /** Counter of how many things have ever been queued on this completion queue useful for avoiding locks to check the queue */ gpr_atm things_queued_ever; - /** 0 initially, 1 once we've begun shutting down */ + /** 0 initially. 1 once we completed shutting */ + /* TODO: (sreek) This is not needed since (shutdown == 1) if and only if + * (pending_events == 0). So consider removing this in future and use + * pending_events */ gpr_atm shutdown; - int shutdown_called; + + /** 0 initially. 1 once we initiated shutdown */ + bool shutdown_called; int num_pluckers; plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS]; @@ -436,7 +442,7 @@ grpc_completion_queue *grpc_completion_queue_create_internal( static void cq_init_next(void *ptr) { cq_next_data *cqd = ptr; - /* Initial ref is dropped by grpc_completion_queue_shutdown */ + /* Initial count is dropped by grpc_completion_queue_shutdown */ gpr_atm_no_barrier_store(&cqd->pending_events, 1); cqd->shutdown_called = false; gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0); @@ -451,12 +457,12 @@ static void cq_destroy_next(void *ptr) { static void cq_init_pluck(void *ptr) { cq_pluck_data *cqd = ptr; - /* Initial ref is dropped by grpc_completion_queue_shutdown */ - gpr_ref_init(&cqd->pending_events, 1); + /* Initial count is dropped by grpc_completion_queue_shutdown */ + gpr_atm_no_barrier_store(&cqd->pending_events, 1); cqd->completed_tail = &cqd->completed_head; cqd->completed_head.next = (uintptr_t)cqd->completed_tail; gpr_atm_no_barrier_store(&cqd->shutdown, 0); - cqd->shutdown_called = 0; + cqd->shutdown_called = false; cqd->num_pluckers = 0; gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0); } @@ -549,24 +555,32 @@ static void cq_check_tag(grpc_completion_queue *cq, void *tag, bool lock_cq) { static void cq_check_tag(grpc_completion_queue *cq, void *tag, bool lock_cq) {} #endif -static bool cq_begin_op_for_next(grpc_completion_queue *cq, void *tag) { - cq_next_data *cqd = DATA_FROM_CQ(cq); +/* Atomically increments a counter only if the counter is not zero. Returns + * true if the increment was successful; false if the counter is zero */ +static bool atm_inc_if_nonzero(gpr_atm *counter) { while (true) { - gpr_atm count = gpr_atm_no_barrier_load(&cqd->pending_events); + gpr_atm count = gpr_atm_no_barrier_load(counter); + /* If zero, we are done. If not, we must to a CAS (instead of an atomic + * increment) to maintain the contract: do not increment the counter if it + * is zero. */ if (count == 0) { return false; - } else if (gpr_atm_no_barrier_cas(&cqd->pending_events, count, count + 1)) { + } else if (gpr_atm_no_barrier_cas(counter, count, count + 1)) { break; } } + return true; } +static bool cq_begin_op_for_next(grpc_completion_queue *cq, void *tag) { + cq_next_data *cqd = DATA_FROM_CQ(cq); + return atm_inc_if_nonzero(&cqd->pending_events); +} + static bool cq_begin_op_for_pluck(grpc_completion_queue *cq, void *tag) { cq_pluck_data *cqd = DATA_FROM_CQ(cq); - GPR_ASSERT(!cqd->shutdown_called); - gpr_ref(&cqd->pending_events); - return true; + return atm_inc_if_nonzero(&cqd->pending_events); } bool grpc_cq_begin_op(grpc_completion_queue *cq, void *tag) { @@ -704,8 +718,10 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx, ((uintptr_t)storage) | (1u & (uintptr_t)cqd->completed_tail->next); cqd->completed_tail = storage; - int shutdown = gpr_unref(&cqd->pending_events); - if (!shutdown) { + if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) { + cq_finish_shutdown_pluck(exec_ctx, cq); + gpr_mu_unlock(cq->mu); + } else { grpc_pollset_worker *pluck_worker = NULL; for (int i = 0; i < cqd->num_pluckers; i++) { if (cqd->pluckers[i].tag == tag) { @@ -725,9 +741,6 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(kick_error); } - } else { - cq_finish_shutdown_pluck(exec_ctx, cq); - gpr_mu_unlock(cq->mu); } GPR_TIMER_END("cq_end_op_for_pluck", 0); @@ -952,6 +965,12 @@ static void cq_shutdown_next(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq) { cq_next_data *cqd = DATA_FROM_CQ(cq); + /* Need an extra ref for cq here because: + * We call cq_finish_shutdown_next() below, that would call pollset shutdown. + * Pollset shutdown decrements the cq ref count which can potentially destroy + * the cq (if that happens to be the last ref). + * Creating an extra ref here prevents the cq from getting destroyed while + * this function is still active */ GRPC_CQ_INTERNAL_REF(cq, "shutting_down"); gpr_mu_lock(cq->mu); if (cqd->shutdown_called) { @@ -960,7 +979,7 @@ static void cq_shutdown_next(grpc_exec_ctx *exec_ctx, GPR_TIMER_END("grpc_completion_queue_shutdown", 0); return; } - cqd->shutdown_called = 1; + cqd->shutdown_called = true; if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) { cq_finish_shutdown_next(exec_ctx, cq); } @@ -1172,21 +1191,32 @@ static void cq_finish_shutdown_pluck(grpc_exec_ctx *exec_ctx, &cq->pollset_shutdown_done); } +/* NOTE: This function is almost exactly identical to cq_shutdown_next() but + * merging them is a bit tricky and probably not worth it */ static void cq_shutdown_pluck(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq) { cq_pluck_data *cqd = DATA_FROM_CQ(cq); + /* Need an extra ref for cq here because: + * We call cq_finish_shutdown_pluck() below, that would call pollset shutdown. + * Pollset shutdown decrements the cq ref count which can potentially destroy + * the cq (if that happens to be the last ref). + * Creating an extra ref here prevents the cq from getting destroyed while + * this function is still active */ + GRPC_CQ_INTERNAL_REF(cq, "shutting_down (pluck cq)"); gpr_mu_lock(cq->mu); if (cqd->shutdown_called) { gpr_mu_unlock(cq->mu); + GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "shutting_down (pluck cq)"); GPR_TIMER_END("grpc_completion_queue_shutdown", 0); return; } - cqd->shutdown_called = 1; - if (gpr_unref(&cqd->pending_events)) { + cqd->shutdown_called = true; + if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) { cq_finish_shutdown_pluck(exec_ctx, cq); } gpr_mu_unlock(cq->mu); + GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "shutting_down (pluck cq)"); } /* Shutdown simply drops a ref that we reserved at creation time; if we drop diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c index db111e597f6..d199ac060e6 100644 --- a/src/core/lib/surface/init.c +++ b/src/core/lib/surface/init.c @@ -36,6 +36,7 @@ #include "src/core/lib/iomgr/resource_quota.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/surface/alarm_internal.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel_init.h" @@ -135,6 +136,7 @@ void grpc_init(void) { grpc_register_tracer(&grpc_call_error_trace); #ifndef NDEBUG grpc_register_tracer(&grpc_trace_pending_tags); + grpc_register_tracer(&grpc_trace_alarm_refcount); grpc_register_tracer(&grpc_trace_cq_refcount); grpc_register_tracer(&grpc_trace_closure); grpc_register_tracer(&grpc_trace_error_refcount); diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c index 810447313c5..967126ecee7 100644 --- a/src/core/tsi/fake_transport_security.c +++ b/src/core/tsi/fake_transport_security.c @@ -407,8 +407,10 @@ static void fake_handshaker_result_destroy(tsi_handshaker_result *self) { static const tsi_handshaker_result_vtable handshaker_result_vtable = { fake_handshaker_result_extract_peer, + NULL, /* create_zero_copy_grpc_protector */ fake_handshaker_result_create_frame_protector, - fake_handshaker_result_get_unused_bytes, fake_handshaker_result_destroy, + fake_handshaker_result_get_unused_bytes, + fake_handshaker_result_destroy, }; static tsi_result fake_handshaker_result_create( @@ -530,7 +532,7 @@ static void fake_handshaker_destroy(tsi_handshaker *self) { static tsi_result fake_handshaker_next( tsi_handshaker *self, const unsigned char *received_bytes, - size_t received_bytes_size, unsigned char **bytes_to_send, + size_t received_bytes_size, const unsigned char **bytes_to_send, size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, tsi_handshaker_on_next_done_cb cb, void *user_data) { /* Sanity check the arguments. */ diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c index 2b1f4310c14..76213072a3c 100644 --- a/src/core/tsi/transport_security.c +++ b/src/core/tsi/transport_security.c @@ -74,14 +74,12 @@ tsi_result tsi_frame_protector_protect(tsi_frame_protector *self, size_t *unprotected_bytes_size, unsigned char *protected_output_frames, size_t *protected_output_frames_size) { - if (self == NULL || unprotected_bytes == NULL || + if (self == NULL || self->vtable == NULL || unprotected_bytes == NULL || unprotected_bytes_size == NULL || protected_output_frames == NULL || protected_output_frames_size == NULL) { return TSI_INVALID_ARGUMENT; } - if (self->vtable == NULL || self->vtable->protect == NULL) { - return TSI_UNIMPLEMENTED; - } + if (self->vtable->protect == NULL) return TSI_UNIMPLEMENTED; return self->vtable->protect(self, unprotected_bytes, unprotected_bytes_size, protected_output_frames, protected_output_frames_size); @@ -90,13 +88,11 @@ tsi_result tsi_frame_protector_protect(tsi_frame_protector *self, tsi_result tsi_frame_protector_protect_flush( tsi_frame_protector *self, unsigned char *protected_output_frames, size_t *protected_output_frames_size, size_t *still_pending_size) { - if (self == NULL || protected_output_frames == NULL || + if (self == NULL || self->vtable == NULL || protected_output_frames == NULL || protected_output_frames_size == NULL || still_pending_size == NULL) { return TSI_INVALID_ARGUMENT; } - if (self->vtable == NULL || self->vtable->protect_flush == NULL) { - return TSI_UNIMPLEMENTED; - } + if (self->vtable->protect_flush == NULL) return TSI_UNIMPLEMENTED; return self->vtable->protect_flush(self, protected_output_frames, protected_output_frames_size, still_pending_size); @@ -106,14 +102,12 @@ tsi_result tsi_frame_protector_unprotect( tsi_frame_protector *self, const unsigned char *protected_frames_bytes, size_t *protected_frames_bytes_size, unsigned char *unprotected_bytes, size_t *unprotected_bytes_size) { - if (self == NULL || protected_frames_bytes == NULL || + if (self == NULL || self->vtable == NULL || protected_frames_bytes == NULL || protected_frames_bytes_size == NULL || unprotected_bytes == NULL || unprotected_bytes_size == NULL) { return TSI_INVALID_ARGUMENT; } - if (self->vtable == NULL || self->vtable->unprotect == NULL) { - return TSI_UNIMPLEMENTED; - } + if (self->vtable->unprotect == NULL) return TSI_UNIMPLEMENTED; return self->vtable->unprotect(self, protected_frames_bytes, protected_frames_bytes_size, unprotected_bytes, unprotected_bytes_size); @@ -131,48 +125,44 @@ void tsi_frame_protector_destroy(tsi_frame_protector *self) { tsi_result tsi_handshaker_get_bytes_to_send_to_peer(tsi_handshaker *self, unsigned char *bytes, size_t *bytes_size) { - if (self == NULL || bytes == NULL || bytes_size == NULL) { + if (self == NULL || self->vtable == NULL || bytes == NULL || + bytes_size == NULL) { return TSI_INVALID_ARGUMENT; } if (self->frame_protector_created) return TSI_FAILED_PRECONDITION; - if (self->vtable == NULL || self->vtable->get_bytes_to_send_to_peer == NULL) { - return TSI_UNIMPLEMENTED; - } + if (self->vtable->get_bytes_to_send_to_peer == NULL) return TSI_UNIMPLEMENTED; return self->vtable->get_bytes_to_send_to_peer(self, bytes, bytes_size); } tsi_result tsi_handshaker_process_bytes_from_peer(tsi_handshaker *self, const unsigned char *bytes, size_t *bytes_size) { - if (self == NULL || bytes == NULL || bytes_size == NULL) { + if (self == NULL || self->vtable == NULL || bytes == NULL || + bytes_size == NULL) { return TSI_INVALID_ARGUMENT; } if (self->frame_protector_created) return TSI_FAILED_PRECONDITION; - if (self->vtable == NULL || self->vtable->process_bytes_from_peer == NULL) { - return TSI_UNIMPLEMENTED; - } + if (self->vtable->process_bytes_from_peer == NULL) return TSI_UNIMPLEMENTED; return self->vtable->process_bytes_from_peer(self, bytes, bytes_size); } tsi_result tsi_handshaker_get_result(tsi_handshaker *self) { - if (self == NULL) return TSI_INVALID_ARGUMENT; + if (self == NULL || self->vtable == NULL) return TSI_INVALID_ARGUMENT; if (self->frame_protector_created) return TSI_FAILED_PRECONDITION; - if (self->vtable == NULL || self->vtable->get_result == NULL) { - return TSI_UNIMPLEMENTED; - } + if (self->vtable->get_result == NULL) return TSI_UNIMPLEMENTED; return self->vtable->get_result(self); } tsi_result tsi_handshaker_extract_peer(tsi_handshaker *self, tsi_peer *peer) { - if (self == NULL || peer == NULL) return TSI_INVALID_ARGUMENT; + if (self == NULL || self->vtable == NULL || peer == NULL) { + return TSI_INVALID_ARGUMENT; + } memset(peer, 0, sizeof(tsi_peer)); if (self->frame_protector_created) return TSI_FAILED_PRECONDITION; if (tsi_handshaker_get_result(self) != TSI_OK) { return TSI_FAILED_PRECONDITION; } - if (self->vtable == NULL || self->vtable->extract_peer == NULL) { - return TSI_UNIMPLEMENTED; - } + if (self->vtable->extract_peer == NULL) return TSI_UNIMPLEMENTED; return self->vtable->extract_peer(self, peer); } @@ -180,14 +170,12 @@ tsi_result tsi_handshaker_create_frame_protector( tsi_handshaker *self, size_t *max_protected_frame_size, tsi_frame_protector **protector) { tsi_result result; - if (self == NULL || protector == NULL) return TSI_INVALID_ARGUMENT; - if (self->frame_protector_created) return TSI_FAILED_PRECONDITION; - if (tsi_handshaker_get_result(self) != TSI_OK) { - return TSI_FAILED_PRECONDITION; - } - if (self->vtable == NULL || self->vtable->create_frame_protector == NULL) { - return TSI_UNIMPLEMENTED; + if (self == NULL || self->vtable == NULL || protector == NULL) { + return TSI_INVALID_ARGUMENT; } + if (self->frame_protector_created) return TSI_FAILED_PRECONDITION; + if (tsi_handshaker_get_result(self) != TSI_OK) return TSI_FAILED_PRECONDITION; + if (self->vtable->create_frame_protector == NULL) return TSI_UNIMPLEMENTED; result = self->vtable->create_frame_protector(self, max_protected_frame_size, protector); if (result == TSI_OK) { @@ -198,14 +186,12 @@ tsi_result tsi_handshaker_create_frame_protector( tsi_result tsi_handshaker_next( tsi_handshaker *self, const unsigned char *received_bytes, - size_t received_bytes_size, unsigned char **bytes_to_send, + size_t received_bytes_size, const unsigned char **bytes_to_send, size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, tsi_handshaker_on_next_done_cb cb, void *user_data) { - if (self == NULL) return TSI_INVALID_ARGUMENT; + if (self == NULL || self->vtable == NULL) return TSI_INVALID_ARGUMENT; if (self->handshaker_result_created) return TSI_FAILED_PRECONDITION; - if (self->vtable == NULL || self->vtable->next == NULL) { - return TSI_UNIMPLEMENTED; - } + if (self->vtable->next == NULL) return TSI_UNIMPLEMENTED; return self->vtable->next(self, received_bytes, received_bytes_size, bytes_to_send, bytes_to_send_size, handshaker_result, cb, user_data); @@ -220,21 +206,21 @@ void tsi_handshaker_destroy(tsi_handshaker *self) { tsi_result tsi_handshaker_result_extract_peer(const tsi_handshaker_result *self, tsi_peer *peer) { - if (self == NULL || peer == NULL) return TSI_INVALID_ARGUMENT; - memset(peer, 0, sizeof(tsi_peer)); - if (self->vtable == NULL || self->vtable->extract_peer == NULL) { - return TSI_UNIMPLEMENTED; + if (self == NULL || self->vtable == NULL || peer == NULL) { + return TSI_INVALID_ARGUMENT; } + memset(peer, 0, sizeof(tsi_peer)); + if (self->vtable->extract_peer == NULL) return TSI_UNIMPLEMENTED; return self->vtable->extract_peer(self, peer); } tsi_result tsi_handshaker_result_create_frame_protector( const tsi_handshaker_result *self, size_t *max_protected_frame_size, tsi_frame_protector **protector) { - if (self == NULL || protector == NULL) return TSI_INVALID_ARGUMENT; - if (self->vtable == NULL || self->vtable->create_frame_protector == NULL) { - return TSI_UNIMPLEMENTED; + if (self == NULL || self->vtable == NULL || protector == NULL) { + return TSI_INVALID_ARGUMENT; } + if (self->vtable->create_frame_protector == NULL) return TSI_UNIMPLEMENTED; return self->vtable->create_frame_protector(self, max_protected_frame_size, protector); } @@ -242,12 +228,11 @@ tsi_result tsi_handshaker_result_create_frame_protector( tsi_result tsi_handshaker_result_get_unused_bytes( const tsi_handshaker_result *self, const unsigned char **bytes, size_t *bytes_size) { - if (self == NULL || bytes == NULL || bytes_size == NULL) { + if (self == NULL || self->vtable == NULL || bytes == NULL || + bytes_size == NULL) { return TSI_INVALID_ARGUMENT; } - if (self->vtable == NULL || self->vtable->get_unused_bytes == NULL) { - return TSI_UNIMPLEMENTED; - } + if (self->vtable->get_unused_bytes == NULL) return TSI_UNIMPLEMENTED; return self->vtable->get_unused_bytes(self, bytes, bytes_size); } diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h index 2c7db6bca92..b0d7039850d 100644 --- a/src/core/tsi/transport_security.h +++ b/src/core/tsi/transport_security.h @@ -70,7 +70,8 @@ typedef struct { tsi_frame_protector **protector); void (*destroy)(tsi_handshaker *self); tsi_result (*next)(tsi_handshaker *self, const unsigned char *received_bytes, - size_t received_bytes_size, unsigned char **bytes_to_send, + size_t received_bytes_size, + const unsigned char **bytes_to_send, size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, tsi_handshaker_on_next_done_cb cb, void *user_data); @@ -86,6 +87,10 @@ struct tsi_handshaker { See transport_security_interface.h for documentation. */ typedef struct { tsi_result (*extract_peer)(const tsi_handshaker_result *self, tsi_peer *peer); + tsi_result (*create_zero_copy_grpc_protector)( + const tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, + tsi_zero_copy_grpc_protector **protector); tsi_result (*create_frame_protector)(const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, tsi_frame_protector **protector); diff --git a/src/core/tsi/transport_security_adapter.c b/src/core/tsi/transport_security_adapter.c index b6dc660c475..1c2a57b3bd1 100644 --- a/src/core/tsi/transport_security_adapter.c +++ b/src/core/tsi/transport_security_adapter.c @@ -66,8 +66,11 @@ static void adapter_result_destroy(tsi_handshaker_result *self) { } static const tsi_handshaker_result_vtable result_vtable = { - adapter_result_extract_peer, adapter_result_create_frame_protector, - adapter_result_get_unused_bytes, adapter_result_destroy, + adapter_result_extract_peer, + NULL, /* create_zero_copy_grpc_protector */ + adapter_result_create_frame_protector, + adapter_result_get_unused_bytes, + adapter_result_destroy, }; /* Ownership of wrapped tsi_handshaker is transferred to the result object. */ @@ -140,7 +143,7 @@ static void adapter_destroy(tsi_handshaker *self) { static tsi_result adapter_next( tsi_handshaker *self, const unsigned char *received_bytes, - size_t received_bytes_size, unsigned char **bytes_to_send, + size_t received_bytes_size, const unsigned char **bytes_to_send, size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, tsi_handshaker_on_next_done_cb cb, void *user_data) { /* Input sanity check. */ diff --git a/src/core/tsi/transport_security_grpc.c b/src/core/tsi/transport_security_grpc.c new file mode 100644 index 00000000000..5bcfdfa61f8 --- /dev/null +++ b/src/core/tsi/transport_security_grpc.c @@ -0,0 +1,64 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "src/core/tsi/transport_security_grpc.h" + +/* This method creates a tsi_zero_copy_grpc_protector object. */ +tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector( + const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + tsi_zero_copy_grpc_protector **protector) { + if (self == NULL || self->vtable == NULL || protector == NULL) { + return TSI_INVALID_ARGUMENT; + } + if (self->vtable->create_zero_copy_grpc_protector == NULL) { + return TSI_UNIMPLEMENTED; + } + return self->vtable->create_zero_copy_grpc_protector( + self, max_output_protected_frame_size, protector); +} + +/* --- tsi_zero_copy_grpc_protector common implementation. --- + + Calls specific implementation after state/input validation. */ + +tsi_result tsi_zero_copy_grpc_protector_protect( + tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *unprotected_slices, + grpc_slice_buffer *protected_slices) { + if (self == NULL || self->vtable == NULL || unprotected_slices == NULL || + protected_slices == NULL) { + return TSI_INVALID_ARGUMENT; + } + if (self->vtable->protect == NULL) return TSI_UNIMPLEMENTED; + return self->vtable->protect(self, unprotected_slices, protected_slices); +} + +tsi_result tsi_zero_copy_grpc_protector_unprotect( + tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *protected_slices, + grpc_slice_buffer *unprotected_slices) { + if (self == NULL || self->vtable == NULL || protected_slices == NULL || + unprotected_slices == NULL) { + return TSI_INVALID_ARGUMENT; + } + if (self->vtable->unprotect == NULL) return TSI_UNIMPLEMENTED; + return self->vtable->unprotect(self, protected_slices, unprotected_slices); +} + +void tsi_zero_copy_grpc_protector_destroy(tsi_zero_copy_grpc_protector *self) { + if (self == NULL) return; + self->vtable->destroy(self); +} diff --git a/src/core/tsi/transport_security_grpc.h b/src/core/tsi/transport_security_grpc.h new file mode 100644 index 00000000000..5ab5297cc48 --- /dev/null +++ b/src/core/tsi/transport_security_grpc.h @@ -0,0 +1,80 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_TSI_TRANSPORT_SECURITY_GRPC_H +#define GRPC_CORE_TSI_TRANSPORT_SECURITY_GRPC_H + +#include +#include "src/core/tsi/transport_security.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* This method creates a tsi_zero_copy_grpc_protector object. It return TSI_OK + assuming there is no fatal error. + The caller is responsible for destroying the protector. */ +tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector( + const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + tsi_zero_copy_grpc_protector **protector); + +/* -- tsi_zero_copy_grpc_protector object -- */ + +/* Outputs protected frames. + - unprotected_slices is the unprotected data to be protected. + - protected_slices is the protected output frames. One or more frames + may be produced in this protect function. + - This method returns TSI_OK in case of success or a specific error code in + case of failure. */ +tsi_result tsi_zero_copy_grpc_protector_protect( + tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *unprotected_slices, + grpc_slice_buffer *protected_slices); + +/* Outputs unprotected bytes. + - protected_slices is the bytes of protected frames. + - unprotected_slices is the unprotected output data. + - This method returns TSI_OK in case of success. Success includes cases where + there is not enough data to output in which case unprotected_slices has 0 + bytes. */ +tsi_result tsi_zero_copy_grpc_protector_unprotect( + tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *protected_slices, + grpc_slice_buffer *unprotected_slices); + +/* Destroys the tsi_zero_copy_grpc_protector object. */ +void tsi_zero_copy_grpc_protector_destroy(tsi_zero_copy_grpc_protector *self); + +/* Base for tsi_zero_copy_grpc_protector implementations. */ +typedef struct { + tsi_result (*protect)(tsi_zero_copy_grpc_protector *self, + grpc_slice_buffer *unprotected_slices, + grpc_slice_buffer *protected_slices); + tsi_result (*unprotect)(tsi_zero_copy_grpc_protector *self, + grpc_slice_buffer *protected_slices, + grpc_slice_buffer *unprotected_slices); + void (*destroy)(tsi_zero_copy_grpc_protector *self); +} tsi_zero_copy_grpc_protector_vtable; + +struct tsi_zero_copy_grpc_protector { + const tsi_zero_copy_grpc_protector_vtable *vtable; +}; + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_CORE_TSI_TRANSPORT_SECURITY_GRPC_H */ diff --git a/src/core/tsi/transport_security_interface.h b/src/core/tsi/transport_security_interface.h index 39ba8addc49..80c426bbdbe 100644 --- a/src/core/tsi/transport_security_interface.h +++ b/src/core/tsi/transport_security_interface.h @@ -62,6 +62,15 @@ const char *tsi_result_to_string(tsi_result result); extern grpc_tracer_flag tsi_tracing_enabled; +/* -- tsi_zero_copy_grpc_protector object -- + + This object protects and unprotects grpc slice buffers with zero or minimized + memory copy once the handshake is done. Implementations of this object must be + thread compatible. This object depends on grpc and the details of this object + is defined in transport_security_grpc.h. */ + +typedef struct tsi_zero_copy_grpc_protector tsi_zero_copy_grpc_protector; + /* --- tsi_frame_protector object --- This object protects and unprotects buffers once the handshake is done. @@ -429,7 +438,7 @@ typedef void (*tsi_handshaker_on_next_done_cb)( tsi_handshaker object. */ tsi_result tsi_handshaker_next( tsi_handshaker *self, const unsigned char *received_bytes, - size_t received_bytes_size, unsigned char **bytes_to_send, + size_t received_bytes_size, const unsigned char **bytes_to_send, size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, tsi_handshaker_on_next_done_cb cb, void *user_data); diff --git a/src/cpp/util/slice_cc.cc b/src/cpp/util/slice_cc.cc index 56e0328b949..486d0cdf0ec 100644 --- a/src/cpp/util/slice_cc.cc +++ b/src/cpp/util/slice_cc.cc @@ -17,6 +17,7 @@ */ #include +#include namespace grpc { @@ -43,4 +44,10 @@ Slice::Slice(const void* buf, size_t len, StaticSlice) Slice::Slice(const Slice& other) : slice_(grpc_slice_ref(other.slice_)) {} +Slice::Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data) + : slice_(grpc_slice_new_with_user_data(buf, len, destroy, user_data)) {} + +Slice::Slice(void* buf, size_t len, void (*destroy)(void*, size_t)) + : slice_(grpc_slice_new_with_len(buf, len, destroy)) {} + } // namespace grpc diff --git a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs index c74d04c8294..0cb9288131d 100644 --- a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs +++ b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs @@ -92,9 +92,33 @@ namespace Grpc.Core.Tests var ex = Assert.Throws(() => Calls.BlockingUnaryCall(helper.CreateUnaryCall(), "abc")); Assert.AreEqual(StatusCode.Unauthenticated, ex.Status.StatusCode); + Assert.AreEqual(0, ex.Trailers.Count); var ex2 = Assert.ThrowsAsync(async () => await Calls.AsyncUnaryCall(helper.CreateUnaryCall(), "abc")); Assert.AreEqual(StatusCode.Unauthenticated, ex2.Status.StatusCode); + Assert.AreEqual(0, ex.Trailers.Count); + } + + [Test] + public void UnaryCall_ServerHandlerThrowsRpcExceptionWithTrailers() + { + helper.UnaryHandler = new UnaryServerMethod((request, context) => + { + var trailers = new Metadata { {"xyz", "xyz-value"} }; + throw new RpcException(new Status(StatusCode.Unauthenticated, ""), trailers); + }); + + var ex = Assert.Throws(() => Calls.BlockingUnaryCall(helper.CreateUnaryCall(), "abc")); + Assert.AreEqual(StatusCode.Unauthenticated, ex.Status.StatusCode); + Assert.AreEqual(1, ex.Trailers.Count); + Assert.AreEqual("xyz", ex.Trailers[0].Key); + Assert.AreEqual("xyz-value", ex.Trailers[0].Value); + + var ex2 = Assert.ThrowsAsync(async () => await Calls.AsyncUnaryCall(helper.CreateUnaryCall(), "abc")); + Assert.AreEqual(StatusCode.Unauthenticated, ex2.Status.StatusCode); + Assert.AreEqual(1, ex2.Trailers.Count); + Assert.AreEqual("xyz", ex2.Trailers[0].Key); + Assert.AreEqual("xyz-value", ex2.Trailers[0].Value); } [Test] @@ -108,9 +132,34 @@ namespace Grpc.Core.Tests var ex = Assert.Throws(() => Calls.BlockingUnaryCall(helper.CreateUnaryCall(), "abc")); Assert.AreEqual(StatusCode.Unauthenticated, ex.Status.StatusCode); + Assert.AreEqual(0, ex.Trailers.Count); + + var ex2 = Assert.ThrowsAsync(async () => await Calls.AsyncUnaryCall(helper.CreateUnaryCall(), "abc")); + Assert.AreEqual(StatusCode.Unauthenticated, ex2.Status.StatusCode); + Assert.AreEqual(0, ex2.Trailers.Count); + } + + [Test] + public void UnaryCall_ServerHandlerSetsStatusAndTrailers() + { + helper.UnaryHandler = new UnaryServerMethod(async (request, context) => + { + context.Status = new Status(StatusCode.Unauthenticated, ""); + context.ResponseTrailers.Add("xyz", "xyz-value"); + return ""; + }); + + var ex = Assert.Throws(() => Calls.BlockingUnaryCall(helper.CreateUnaryCall(), "abc")); + Assert.AreEqual(StatusCode.Unauthenticated, ex.Status.StatusCode); + Assert.AreEqual(1, ex.Trailers.Count); + Assert.AreEqual("xyz", ex.Trailers[0].Key); + Assert.AreEqual("xyz-value", ex.Trailers[0].Value); var ex2 = Assert.ThrowsAsync(async () => await Calls.AsyncUnaryCall(helper.CreateUnaryCall(), "abc")); Assert.AreEqual(StatusCode.Unauthenticated, ex2.Status.StatusCode); + Assert.AreEqual(1, ex2.Trailers.Count); + Assert.AreEqual("xyz", ex2.Trailers[0].Key); + Assert.AreEqual("xyz-value", ex2.Trailers[0].Value); } [Test] @@ -148,7 +197,7 @@ namespace Grpc.Core.Tests CollectionAssert.AreEqual(new string[] { "A", "B", "C" }, await call.ResponseStream.ToListAsync()); Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode); - Assert.IsNotNull("xyz", call.GetTrailers()[0].Key); + Assert.AreEqual("xyz", call.GetTrailers()[0].Key); } [Test] @@ -182,6 +231,27 @@ namespace Grpc.Core.Tests Assert.AreEqual(StatusCode.InvalidArgument, ex2.Status.StatusCode); } + [Test] + public async Task ServerStreamingCall_TrailersFromMultipleSourcesGetConcatenated() + { + helper.ServerStreamingHandler = new ServerStreamingServerMethod(async (request, responseStream, context) => + { + context.ResponseTrailers.Add("xyz", "xyz-value"); + throw new RpcException(new Status(StatusCode.InvalidArgument, ""), new Metadata { {"abc", "abc-value"} }); + }); + + var call = Calls.AsyncServerStreamingCall(helper.CreateServerStreamingCall(), ""); + + var ex = Assert.ThrowsAsync(async () => await call.ResponseStream.MoveNext()); + Assert.AreEqual(StatusCode.InvalidArgument, ex.Status.StatusCode); + Assert.AreEqual(2, call.GetTrailers().Count); + Assert.AreEqual(2, ex.Trailers.Count); + Assert.AreEqual("xyz", ex.Trailers[0].Key); + Assert.AreEqual("xyz-value", ex.Trailers[0].Value); + Assert.AreEqual("abc", ex.Trailers[1].Key); + Assert.AreEqual("abc-value", ex.Trailers[1].Value); + } + [Test] public async Task DuplexStreamingCall() { @@ -199,7 +269,7 @@ namespace Grpc.Core.Tests CollectionAssert.AreEqual(new string[] { "A", "B", "C" }, await call.ResponseStream.ToListAsync()); Assert.AreEqual(StatusCode.OK, call.GetStatus().StatusCode); - Assert.IsNotNull("xyz-value", call.GetTrailers()[0].Value); + Assert.AreEqual("xyz-value", call.GetTrailers()[0].Value); } [Test] diff --git a/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs b/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs index 0f3a82c605f..fc9d5599f21 100644 --- a/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs +++ b/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs @@ -18,6 +18,7 @@ using System; using System.Linq; +using System.Threading; using Grpc.Core; using NUnit.Framework; @@ -75,5 +76,19 @@ namespace Grpc.Core.Tests var parts = coreVersion.Split('.'); Assert.AreEqual(3, parts.Length); } + + [Test] + public void ShuttingDownEventIsFired() + { + var cts = new CancellationTokenSource(); + var handler = new EventHandler((sender, args) => { cts.Cancel(); }); + + GrpcEnvironment.ShuttingDown += handler; + var env = GrpcEnvironment.AddRef(); + GrpcEnvironment.ReleaseAsync().Wait(); + GrpcEnvironment.ShuttingDown -= handler; + + Assert.IsTrue(cts.Token.IsCancellationRequested); + } } } diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs index 0663ee92150..cbc7d2078c3 100644 --- a/src/csharp/Grpc.Core/GrpcEnvironment.cs +++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs @@ -49,7 +49,7 @@ namespace Grpc.Core readonly DebugStats debugStats = new DebugStats(); readonly AtomicCounter cqPickerCounter = new AtomicCounter(); - bool isClosed; + bool isShutdown; /// /// Returns a reference-counted instance of initialized gRPC environment. @@ -237,6 +237,12 @@ namespace Grpc.Core } } + /// + /// Occurs when GrpcEnvironment is about the start the shutdown logic. + /// If GrpcEnvironment is later initialized and shutdown, the event will be fired again (unless unregistered first). + /// + public static event EventHandler ShuttingDown; + /// /// Creates gRPC environment. /// @@ -311,13 +317,16 @@ namespace Grpc.Core /// private async Task ShutdownAsync() { - if (isClosed) + if (isShutdown) { - throw new InvalidOperationException("Close has already been called"); + throw new InvalidOperationException("ShutdownAsync has already been called"); } + + await Task.Run(() => ShuttingDown?.Invoke(this, null)).ConfigureAwait(false); + await threadPool.StopAsync().ConfigureAwait(false); GrpcNativeShutdown(); - isClosed = true; + isShutdown = true; debugStats.CheckOK(); } diff --git a/src/csharp/Grpc.Core/Internal/AsyncCall.cs b/src/csharp/Grpc.Core/Internal/AsyncCall.cs index 6e6ca7cd53c..17109de587b 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCall.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCall.cs @@ -329,7 +329,7 @@ namespace Grpc.Core.Internal protected override Exception GetRpcExceptionClientOnly() { - return new RpcException(finishedStatus.Value.Status); + return new RpcException(finishedStatus.Value.Status, finishedStatus.Value.Trailers); } protected override Task CheckSendAllowedOrEarlyResult() @@ -348,7 +348,7 @@ namespace Grpc.Core.Internal // Writing after the call has finished is not a programming error because server can close // the call anytime, so don't throw directly, but let the write task finish with an error. var tcs = new TaskCompletionSource(); - tcs.SetException(new RpcException(finishedStatus.Value.Status)); + tcs.SetException(new RpcException(finishedStatus.Value.Status, finishedStatus.Value.Trailers)); return tcs.Task; } @@ -468,7 +468,7 @@ namespace Grpc.Core.Internal var status = receivedStatus.Status; if (status.StatusCode != StatusCode.OK) { - unaryResponseTcs.SetException(new RpcException(status)); + unaryResponseTcs.SetException(new RpcException(status, receivedStatus.Trailers)); return; } @@ -506,7 +506,7 @@ namespace Grpc.Core.Internal var status = receivedStatus.Status; if (status.StatusCode != StatusCode.OK) { - streamingResponseCallFinishedTcs.SetException(new RpcException(status)); + streamingResponseCallFinishedTcs.SetException(new RpcException(status, receivedStatus.Trailers)); return; } diff --git a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs index 36702a3fab4..6019f8e7934 100644 --- a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs +++ b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs @@ -76,7 +76,7 @@ namespace Grpc.Core.Internal { Logger.Warning(e, "Exception occured in handler."); } - status = HandlerUtils.StatusFromException(e); + status = HandlerUtils.GetStatusFromExceptionAndMergeTrailers(e, context.ResponseTrailers); } try { @@ -133,7 +133,7 @@ namespace Grpc.Core.Internal { Logger.Warning(e, "Exception occured in handler."); } - status = HandlerUtils.StatusFromException(e); + status = HandlerUtils.GetStatusFromExceptionAndMergeTrailers(e, context.ResponseTrailers); } try @@ -191,7 +191,7 @@ namespace Grpc.Core.Internal { Logger.Warning(e, "Exception occured in handler."); } - status = HandlerUtils.StatusFromException(e); + status = HandlerUtils.GetStatusFromExceptionAndMergeTrailers(e, context.ResponseTrailers); } try @@ -247,7 +247,7 @@ namespace Grpc.Core.Internal { Logger.Warning(e, "Exception occured in handler."); } - status = HandlerUtils.StatusFromException(e); + status = HandlerUtils.GetStatusFromExceptionAndMergeTrailers(e, context.ResponseTrailers); } try { @@ -292,11 +292,20 @@ namespace Grpc.Core.Internal internal static class HandlerUtils { - public static Status StatusFromException(Exception e) + public static Status GetStatusFromExceptionAndMergeTrailers(Exception e, Metadata callContextResponseTrailers) { var rpcException = e as RpcException; if (rpcException != null) { + // There are two sources of metadata entries on the server-side: + // 1. serverCallContext.ResponseTrailers + // 2. trailers in RpcException thrown by user code in server side handler. + // As metadata allows duplicate keys, the logical thing to do is + // to just merge trailers from RpcException into serverCallContext.ResponseTrailers. + foreach (var entry in rpcException.Trailers) + { + callContextResponseTrailers.Add(entry); + } // use the status thrown by handler. return rpcException.Status; } diff --git a/src/csharp/Grpc.Core/RpcException.cs b/src/csharp/Grpc.Core/RpcException.cs index 01b9e4fb1a1..d2c912e73af 100644 --- a/src/csharp/Grpc.Core/RpcException.cs +++ b/src/csharp/Grpc.Core/RpcException.cs @@ -17,6 +17,7 @@ #endregion using System; +using Grpc.Core.Utils; namespace Grpc.Core { @@ -26,6 +27,7 @@ namespace Grpc.Core public class RpcException : Exception { private readonly Status status; + private readonly Metadata trailers; /// /// Creates a new RpcException associated with given status. @@ -34,6 +36,7 @@ namespace Grpc.Core public RpcException(Status status) : base(status.ToString()) { this.status = status; + this.trailers = Metadata.Empty; } /// @@ -44,6 +47,18 @@ namespace Grpc.Core public RpcException(Status status, string message) : base(message) { this.status = status; + this.trailers = Metadata.Empty; + } + + /// + /// Creates a new RpcException associated with given status and trailing response metadata. + /// + /// Resulting status of a call. + /// Response trailing metadata. + public RpcException(Status status, Metadata trailers) : base(status.ToString()) + { + this.status = status; + this.trailers = GrpcPreconditions.CheckNotNull(trailers); } /// @@ -56,5 +71,18 @@ namespace Grpc.Core return status; } } + + /// + /// Gets the call trailing metadata. + /// Trailers only have meaningful content for client-side calls (in which case they represent the trailing metadata sent by the server when closing the call). + /// Instances of RpcException thrown by the server-side part of the stack will have trailers always set to empty. + /// + public Metadata Trailers + { + get + { + return trailers; + } + } } } diff --git a/src/csharp/Grpc.IntegrationTesting/CustomErrorDetailsTest.cs b/src/csharp/Grpc.IntegrationTesting/CustomErrorDetailsTest.cs index be996f91e03..374c6fc23f0 100644 --- a/src/csharp/Grpc.IntegrationTesting/CustomErrorDetailsTest.cs +++ b/src/csharp/Grpc.IntegrationTesting/CustomErrorDetailsTest.cs @@ -65,7 +65,7 @@ namespace Grpc.IntegrationTesting } [Test] - public async Task UnaryCall() + public async Task ErrorDetailsFromCallObject() { var call = client.UnaryCallAsync(new SimpleRequest { ResponseSize = 10 }); @@ -83,7 +83,24 @@ namespace Grpc.IntegrationTesting } } - private DebugInfo GetDebugInfo(Metadata trailers) + [Test] + public async Task ErrorDetailsFromRpcException() + { + try + { + await client.UnaryCallAsync(new SimpleRequest { ResponseSize = 10 }); + Assert.Fail(); + } + catch (RpcException e) + { + Assert.AreEqual(StatusCode.Unknown, e.Status.StatusCode); + var debugInfo = GetDebugInfo(e.Trailers); + Assert.AreEqual(debugInfo.Detail, ExceptionDetail); + Assert.IsNotEmpty(debugInfo.StackEntries); + } + } + + private static DebugInfo GetDebugInfo(Metadata trailers) { var entry = trailers.First((e) => e.Key == DebugInfoTrailerName); return DebugInfo.Parser.ParseFrom(entry.ValueBytes); diff --git a/src/node/ext/call.cc b/src/node/ext/call.cc index 71e69040083..26095a78f9e 100644 --- a/src/node/ext/call.cc +++ b/src/node/ext/call.cc @@ -260,7 +260,10 @@ class SendClientCloseOp : public Op { class SendServerStatusOp : public Op { public: - SendServerStatusOp() { grpc_metadata_array_init(&status_metadata); } + SendServerStatusOp() { + details = grpc_empty_slice(); + grpc_metadata_array_init(&status_metadata); + } ~SendServerStatusOp() { grpc_slice_unref(details); DestroyMetadataArray(&status_metadata); @@ -381,7 +384,10 @@ class ReadMessageOp : public Op { class ClientStatusOp : public Op { public: - ClientStatusOp() { grpc_metadata_array_init(&metadata_array); } + ClientStatusOp() { + grpc_metadata_array_init(&metadata_array); + status_details = grpc_empty_slice(); + } ~ClientStatusOp() { grpc_metadata_array_destroy(&metadata_array); diff --git a/src/node/test/call_test.js b/src/node/test/call_test.js index aebd298e322..b5246c4f31a 100644 --- a/src/node/test/call_test.js +++ b/src/node/test/call_test.js @@ -188,6 +188,103 @@ describe('call', function() { }, TypeError); }); }); + describe('startBatch with message', function() { + it('should fail with null argument', function() { + var call = new grpc.Call(channel, 'method', getDeadline(1)); + assert.throws(function() { + var batch = {}; + batch[grpc.opType.SEND_MESSAGE] = null; + call.startBatch(batch, function(){}); + }, TypeError); + }); + it('should fail with numeric argument', function() { + var call = new grpc.Call(channel, 'method', getDeadline(1)); + assert.throws(function() { + var batch = {}; + batch[grpc.opType.SEND_MESSAGE] = 5; + call.startBatch(batch, function(){}); + }, TypeError); + }); + it('should fail with string argument', function() { + var call = new grpc.Call(channel, 'method', getDeadline(1)); + assert.throws(function() { + var batch = {}; + batch[grpc.opType.SEND_MESSAGE] = 'value'; + call.startBatch(batch, function(){}); + }, TypeError); + }); + }); + describe('startBatch with status', function() { + it('should fail without a code', function() { + var call = new grpc.Call(channel, 'method', getDeadline(1)); + assert.throws(function() { + var batch = {}; + batch[grpc.opType.SEND_STATUS_FROM_SERVER] = { + details: 'details string', + metadata: {} + }; + call.startBatch(batch, function(){}); + }, TypeError); + }); + it('should fail without details', function() { + var call = new grpc.Call(channel, 'method', getDeadline(1)); + assert.throws(function() { + var batch = {}; + batch[grpc.opType.SEND_STATUS_FROM_SERVER] = { + code: 0, + metadata: {} + }; + call.startBatch(batch, function(){}); + }, TypeError); + }); + it('should fail without metadata', function() { + var call = new grpc.Call(channel, 'method', getDeadline(1)); + assert.throws(function() { + var batch = {}; + batch[grpc.opType.SEND_STATUS_FROM_SERVER] = { + code: 0, + details: 'details string' + }; + call.startBatch(batch, function(){}); + }, TypeError); + }); + it('should fail with incorrectly typed code argument', function() { + var call = new grpc.Call(channel, 'method', getDeadline(1)); + assert.throws(function() { + var batch = {}; + batch[grpc.opType.SEND_STATUS_FROM_SERVER] = { + code: 'code string', + details: 'details string', + metadata: {} + }; + call.startBatch(batch, function(){}); + }, TypeError); + }); + it('should fail with incorrectly typed details argument', function() { + var call = new grpc.Call(channel, 'method', getDeadline(1)); + assert.throws(function() { + var batch = {}; + batch[grpc.opType.SEND_STATUS_FROM_SERVER] = { + code: 0, + details: 5, + metadata: {} + }; + call.startBatch(batch, function(){}); + }, TypeError); + }); + it('should fail with incorrectly typed metadata argument', function() { + var call = new grpc.Call(channel, 'method', getDeadline(1)); + assert.throws(function() { + var batch = {}; + batch[grpc.opType.SEND_STATUS_FROM_SERVER] = { + code: 0, + details: 'details string', + metadata: 'abc' + }; + call.startBatch(batch, function(){}); + }, TypeError); + }); + }); describe('cancel', function() { it('should succeed', function() { var call = new grpc.Call(channel, 'method', getDeadline(1)); diff --git a/src/objective-c/README.md b/src/objective-c/README.md index 3624475b9cc..e76ee173ea3 100644 --- a/src/objective-c/README.md +++ b/src/objective-c/README.md @@ -112,7 +112,7 @@ the sample Podspec above. For example, you could use: ```ruby s.prepare_command = <<-CMD ... - #{src}/*.proto #{src}/**/*.proto + `find . -name *.proto -print | xargs` CMD ... ms.source_files = "#{dir}/*.pbobjc.{h,m}", "#{dir}/**/*.pbobjc.{h,m}" diff --git a/src/php/ext/grpc/call.c b/src/php/ext/grpc/call.c index 2f67e5cee79..c4997f720d1 100644 --- a/src/php/ext/grpc/call.c +++ b/src/php/ext/grpc/call.c @@ -214,10 +214,12 @@ PHP_METHOD(Call, __construct) { return; } wrapped_grpc_channel *channel = Z_WRAPPED_GRPC_CHANNEL_P(channel_obj); - if (channel->wrapped == NULL) { + gpr_mu_lock(&channel->wrapper->mu); + if (channel->wrapper->wrapped == NULL) { zend_throw_exception(spl_ce_InvalidArgumentException, "Call cannot be constructed from a closed Channel", 1 TSRMLS_CC); + gpr_mu_unlock(&channel->wrapper->mu); return; } add_property_zval(getThis(), "channel", channel_obj); @@ -226,13 +228,15 @@ PHP_METHOD(Call, __construct) { grpc_slice host_slice = host_override != NULL ? grpc_slice_from_copied_string(host_override) : grpc_empty_slice(); call->wrapped = - grpc_channel_create_call(channel->wrapped, NULL, GRPC_PROPAGATE_DEFAULTS, + grpc_channel_create_call(channel->wrapper->wrapped, NULL, + GRPC_PROPAGATE_DEFAULTS, completion_queue, method_slice, host_override != NULL ? &host_slice : NULL, deadline->wrapped, NULL); grpc_slice_unref(method_slice); grpc_slice_unref(host_slice); call->owned = true; + gpr_mu_unlock(&channel->wrapper->mu); } /** diff --git a/src/php/ext/grpc/call_credentials.c b/src/php/ext/grpc/call_credentials.c index a990206c086..f46091d7098 100644 --- a/src/php/ext/grpc/call_credentials.c +++ b/src/php/ext/grpc/call_credentials.c @@ -109,8 +109,8 @@ PHP_METHOD(CallCredentials, createFromPlugin) { zend_fcall_info *fci; zend_fcall_info_cache *fci_cache; - fci = (zend_fcall_info *)emalloc(sizeof(zend_fcall_info)); - fci_cache = (zend_fcall_info_cache *)emalloc(sizeof(zend_fcall_info_cache)); + fci = (zend_fcall_info *)malloc(sizeof(zend_fcall_info)); + fci_cache = (zend_fcall_info_cache *)malloc(sizeof(zend_fcall_info_cache)); memset(fci, 0, sizeof(zend_fcall_info)); memset(fci_cache, 0, sizeof(zend_fcall_info_cache)); @@ -123,7 +123,7 @@ PHP_METHOD(CallCredentials, createFromPlugin) { } plugin_state *state; - state = (plugin_state *)emalloc(sizeof(plugin_state)); + state = (plugin_state *)malloc(sizeof(plugin_state)); memset(state, 0, sizeof(plugin_state)); /* save the user provided PHP callback function */ @@ -210,13 +210,13 @@ void plugin_get_metadata(void *ptr, grpc_auth_metadata_context context, /* Cleanup function for plugin creds API */ void plugin_destroy_state(void *ptr) { plugin_state *state = (plugin_state *)ptr; - efree(state->fci); - efree(state->fci_cache); + free(state->fci); + free(state->fci_cache); #if PHP_MAJOR_VERSION < 7 PHP_GRPC_FREE_STD_ZVAL(state->fci->params); PHP_GRPC_FREE_STD_ZVAL(state->fci->retval); #endif - efree(state); + free(state); } ZEND_BEGIN_ARG_INFO_EX(arginfo_createComposite, 0, 0, 2) diff --git a/src/php/ext/grpc/channel.c b/src/php/ext/grpc/channel.c index 6c432d28187..f1187e8722f 100644 --- a/src/php/ext/grpc/channel.c +++ b/src/php/ext/grpc/channel.c @@ -25,6 +25,13 @@ #include #include #include +#include +#include +#if PHP_MAJOR_VERSION < 7 +#include +#else +#include +#endif #include #include "php_grpc.h" @@ -44,11 +51,25 @@ zend_class_entry *grpc_ce_channel; #if PHP_MAJOR_VERSION >= 7 static zend_object_handlers channel_ce_handlers; #endif +static gpr_mu global_persistent_list_mu; +int le_plink; /* Frees and destroys an instance of wrapped_grpc_channel */ PHP_GRPC_FREE_WRAPPED_FUNC_START(wrapped_grpc_channel) - if (p->wrapped != NULL) { - grpc_channel_destroy(p->wrapped); + if (p->wrapper != NULL) { + gpr_mu_lock(&p->wrapper->mu); + if (p->wrapper->wrapped != NULL) { + php_grpc_zend_resource *rsrc; + php_grpc_int key_len = strlen(p->wrapper->key); + // only destroy the channel here if not found in the persistent list + gpr_mu_lock(&global_persistent_list_mu); + if (!(PHP_GRPC_PERSISTENT_LIST_FIND(&EG(persistent_list), p->wrapper->key, + key_len, rsrc))) { + grpc_channel_destroy(p->wrapper->wrapped); + } + gpr_mu_unlock(&global_persistent_list_mu); + } + gpr_mu_unlock(&p->wrapper->mu); } PHP_GRPC_FREE_WRAPPED_FUNC_END() @@ -62,15 +83,15 @@ php_grpc_zend_object create_wrapped_grpc_channel(zend_class_entry *class_type PHP_GRPC_FREE_CLASS_OBJECT(wrapped_grpc_channel, channel_ce_handlers); } -void php_grpc_read_args_array(zval *args_array, - grpc_channel_args *args TSRMLS_DC) { +int php_grpc_read_args_array(zval *args_array, + grpc_channel_args *args TSRMLS_DC) { HashTable *array_hash; int args_index; array_hash = Z_ARRVAL_P(args_array); if (!array_hash) { zend_throw_exception(spl_ce_InvalidArgumentException, "array_hash is NULL", 1 TSRMLS_CC); - return; + return FAILURE; } args->num_args = zend_hash_num_elements(array_hash); args->args = ecalloc(args->num_args, sizeof(grpc_arg)); @@ -84,7 +105,7 @@ void php_grpc_read_args_array(zval *args_array, if (key_type != HASH_KEY_IS_STRING) { zend_throw_exception(spl_ce_InvalidArgumentException, "args keys must be strings", 1 TSRMLS_CC); - return; + return FAILURE; } args->args[args_index].key = key; switch (Z_TYPE_P(data)) { @@ -99,16 +120,78 @@ void php_grpc_read_args_array(zval *args_array, default: zend_throw_exception(spl_ce_InvalidArgumentException, "args values must be int or string", 1 TSRMLS_CC); - return; + return FAILURE; } args_index++; PHP_GRPC_HASH_FOREACH_END() + return SUCCESS; +} + +void generate_sha1_str(char *sha1str, char *str, php_grpc_int len) { + PHP_SHA1_CTX context; + unsigned char digest[20]; + sha1str[0] = '\0'; + PHP_SHA1Init(&context); + PHP_GRPC_SHA1Update(&context, str, len); + PHP_SHA1Final(digest, &context); + make_sha1_digest(sha1str, digest); +} + +void create_channel( + wrapped_grpc_channel *channel, + char *target, + grpc_channel_args args, + wrapped_grpc_channel_credentials *creds) { + if (creds == NULL) { + channel->wrapper->wrapped = grpc_insecure_channel_create(target, &args, + NULL); + } else { + channel->wrapper->wrapped = + grpc_secure_channel_create(creds->wrapped, target, &args, NULL); + } + efree(args.args); +} + +void create_and_add_channel_to_persistent_list( + wrapped_grpc_channel *channel, + char *target, + grpc_channel_args args, + wrapped_grpc_channel_credentials *creds, + char *key, + php_grpc_int key_len) { + php_grpc_zend_resource new_rsrc; + channel_persistent_le_t *le; + // this links each persistent list entry to a destructor + new_rsrc.type = le_plink; + le = malloc(sizeof(channel_persistent_le_t)); + + create_channel(channel, target, args, creds); + + le->channel = channel->wrapper; + new_rsrc.ptr = le; + gpr_mu_lock(&global_persistent_list_mu); + PHP_GRPC_PERSISTENT_LIST_UPDATE(&EG(persistent_list), key, key_len, + (void *)&new_rsrc); + gpr_mu_unlock(&global_persistent_list_mu); } /** - * Construct an instance of the Channel class. If the $args array contains a - * "credentials" key mapping to a ChannelCredentials object, a secure channel - * will be created with those credentials. + * Construct an instance of the Channel class. + * + * By default, the underlying grpc_channel is "persistent". That is, given + * the same set of parameters passed to the constructor, the same underlying + * grpc_channel will be returned. + * + * If the $args array contains a "credentials" key mapping to a + * ChannelCredentials object, a secure channel will be created with those + * credentials. + * + * If the $args array contains a "force_new" key mapping to a boolean value + * of "true", a new underlying grpc_channel will be created regardless. If + * there are any opened channels on the same hostname, user must manually + * call close() on those dangling channels before the end of the PHP + * script. + * * @param string $target The hostname to associate with this channel * @param array $args_array The arguments to pass to the Channel */ @@ -121,6 +204,9 @@ PHP_METHOD(Channel, __construct) { grpc_channel_args args; HashTable *array_hash; wrapped_grpc_channel_credentials *creds = NULL; + php_grpc_zend_resource *rsrc; + bool force_new = false; + zval *force_new_obj = NULL; /* "sa" == 1 string, 1 array */ if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sa", &target, @@ -131,7 +217,7 @@ PHP_METHOD(Channel, __construct) { } array_hash = Z_ARRVAL_P(args_array); if (php_grpc_zend_hash_find(array_hash, "credentials", sizeof("credentials"), - (void **)&creds_obj) == SUCCESS) { + (void **)&creds_obj) == SUCCESS) { if (Z_TYPE_P(creds_obj) == IS_NULL) { creds = NULL; php_grpc_zend_hash_del(array_hash, "credentials", sizeof("credentials")); @@ -146,14 +232,82 @@ PHP_METHOD(Channel, __construct) { php_grpc_zend_hash_del(array_hash, "credentials", sizeof("credentials")); } } - php_grpc_read_args_array(args_array, &args TSRMLS_CC); - if (creds == NULL) { - channel->wrapped = grpc_insecure_channel_create(target, &args, NULL); + if (php_grpc_zend_hash_find(array_hash, "force_new", sizeof("force_new"), + (void **)&force_new_obj) == SUCCESS) { + if (PHP_GRPC_BVAL_IS_TRUE(force_new_obj)) { + force_new = true; + } + php_grpc_zend_hash_del(array_hash, "force_new", sizeof("force_new")); + } + + // parse the rest of the channel args array + if (php_grpc_read_args_array(args_array, &args TSRMLS_CC) == FAILURE) { + return; + } + + // Construct a hashkey for the persistent channel + // Currently, the hashkey contains 3 parts: + // 1. hostname + // 2. hash value of the channel args array (excluding "credentials" + // and "force_new") + // 3. (optional) hash value of the ChannelCredentials object + php_serialize_data_t var_hash; + smart_str buf = {0}; + PHP_VAR_SERIALIZE_INIT(var_hash); + PHP_GRPC_VAR_SERIALIZE(&buf, args_array, &var_hash); + PHP_VAR_SERIALIZE_DESTROY(var_hash); + + char sha1str[41]; + generate_sha1_str(sha1str, PHP_GRPC_SERIALIZED_BUF_STR(buf), + PHP_GRPC_SERIALIZED_BUF_LEN(buf)); + + php_grpc_int key_len = target_length + strlen(sha1str); + if (creds != NULL && creds->hashstr != NULL) { + key_len += strlen(creds->hashstr); + } + char *key = malloc(key_len + 1); + strcpy(key, target); + strcat(key, sha1str); + if (creds != NULL && creds->hashstr != NULL) { + strcat(key, creds->hashstr); + } + channel->wrapper = malloc(sizeof(grpc_channel_wrapper)); + channel->wrapper->key = key; + channel->wrapper->target = target; + channel->wrapper->args_hashstr = sha1str; + if (creds != NULL && creds->hashstr != NULL) { + channel->wrapper->creds_hashstr = creds->hashstr; + } + gpr_mu_init(&channel->wrapper->mu); + smart_str_free(&buf); + + if (force_new) { + php_grpc_delete_persistent_list_entry(key, key_len TSRMLS_CC); + } + + if (creds != NULL && creds->has_call_creds) { + // If the ChannelCredentials object was composed with a CallCredentials + // object, there is no way we can tell them apart. Do NOT persist + // them. They should be individually destroyed. + create_channel(channel, target, args, creds); + } else if (!(PHP_GRPC_PERSISTENT_LIST_FIND(&EG(persistent_list), key, + key_len, rsrc))) { + create_and_add_channel_to_persistent_list( + channel, target, args, creds, key, key_len); } else { - channel->wrapped = - grpc_secure_channel_create(creds->wrapped, target, &args, NULL); + // Found a previously stored channel in the persistent list + channel_persistent_le_t *le = (channel_persistent_le_t *)rsrc->ptr; + if (strcmp(target, le->channel->target) != 0 || + strcmp(sha1str, le->channel->args_hashstr) != 0 || + (creds != NULL && creds->hashstr != NULL && + strcmp(creds->hashstr, le->channel->creds_hashstr) != 0)) { + // somehow hash collision + create_and_add_channel_to_persistent_list( + channel, target, args, creds, key, key_len); + } else { + channel->wrapper = le->channel; + } } - efree(args.args); } /** @@ -162,7 +316,16 @@ PHP_METHOD(Channel, __construct) { */ PHP_METHOD(Channel, getTarget) { wrapped_grpc_channel *channel = Z_WRAPPED_GRPC_CHANNEL_P(getThis()); - PHP_GRPC_RETURN_STRING(grpc_channel_get_target(channel->wrapped), 1); + gpr_mu_lock(&channel->wrapper->mu); + if (channel->wrapper->wrapped == NULL) { + zend_throw_exception(spl_ce_RuntimeException, + "Channel already closed", 1 TSRMLS_CC); + gpr_mu_unlock(&channel->wrapper->mu); + return; + } + char *target = grpc_channel_get_target(channel->wrapper->wrapped); + gpr_mu_unlock(&channel->wrapper->mu); + PHP_GRPC_RETURN_STRING(target, 1); } /** @@ -172,6 +335,14 @@ PHP_METHOD(Channel, getTarget) { */ PHP_METHOD(Channel, getConnectivityState) { wrapped_grpc_channel *channel = Z_WRAPPED_GRPC_CHANNEL_P(getThis()); + gpr_mu_lock(&channel->wrapper->mu); + if (channel->wrapper->wrapped == NULL) { + zend_throw_exception(spl_ce_RuntimeException, + "Channel already closed", 1 TSRMLS_CC); + gpr_mu_unlock(&channel->wrapper->mu); + return; + } + bool try_to_connect = false; /* "|b" == 1 optional bool */ @@ -179,10 +350,18 @@ PHP_METHOD(Channel, getConnectivityState) { == FAILURE) { zend_throw_exception(spl_ce_InvalidArgumentException, "getConnectivityState expects a bool", 1 TSRMLS_CC); + gpr_mu_unlock(&channel->wrapper->mu); return; } - RETURN_LONG(grpc_channel_check_connectivity_state(channel->wrapped, - (int)try_to_connect)); + int state = grpc_channel_check_connectivity_state(channel->wrapper->wrapped, + (int)try_to_connect); + // this can happen if another shared Channel object close the underlying + // channel + if (state == GRPC_CHANNEL_SHUTDOWN) { + channel->wrapper->wrapped = NULL; + } + gpr_mu_unlock(&channel->wrapper->mu); + RETURN_LONG(state); } /** @@ -194,25 +373,37 @@ PHP_METHOD(Channel, getConnectivityState) { */ PHP_METHOD(Channel, watchConnectivityState) { wrapped_grpc_channel *channel = Z_WRAPPED_GRPC_CHANNEL_P(getThis()); + gpr_mu_lock(&channel->wrapper->mu); + if (channel->wrapper->wrapped == NULL) { + zend_throw_exception(spl_ce_RuntimeException, + "Channel already closed", 1 TSRMLS_CC); + gpr_mu_unlock(&channel->wrapper->mu); + return; + } + php_grpc_long last_state; zval *deadline_obj; /* "lO" == 1 long 1 object */ if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "lO", - &last_state, &deadline_obj, grpc_ce_timeval) == FAILURE) { + &last_state, &deadline_obj, + grpc_ce_timeval) == FAILURE) { zend_throw_exception(spl_ce_InvalidArgumentException, - "watchConnectivityState expects 1 long 1 timeval", 1 TSRMLS_CC); + "watchConnectivityState expects 1 long 1 timeval", + 1 TSRMLS_CC); + gpr_mu_unlock(&channel->wrapper->mu); return; } wrapped_grpc_timeval *deadline = Z_WRAPPED_GRPC_TIMEVAL_P(deadline_obj); - grpc_channel_watch_connectivity_state(channel->wrapped, + grpc_channel_watch_connectivity_state(channel->wrapper->wrapped, (grpc_connectivity_state)last_state, deadline->wrapped, completion_queue, NULL); grpc_event event = - grpc_completion_queue_pluck(completion_queue, NULL, - gpr_inf_future(GPR_CLOCK_REALTIME), NULL); + grpc_completion_queue_pluck(completion_queue, NULL, + gpr_inf_future(GPR_CLOCK_REALTIME), NULL); + gpr_mu_unlock(&channel->wrapper->mu); RETURN_BOOL(event.success); } @@ -222,10 +413,48 @@ PHP_METHOD(Channel, watchConnectivityState) { */ PHP_METHOD(Channel, close) { wrapped_grpc_channel *channel = Z_WRAPPED_GRPC_CHANNEL_P(getThis()); - if (channel->wrapped != NULL) { - grpc_channel_destroy(channel->wrapped); - channel->wrapped = NULL; + gpr_mu_lock(&channel->wrapper->mu); + if (channel->wrapper->wrapped != NULL) { + grpc_channel_destroy(channel->wrapper->wrapped); + channel->wrapper->wrapped = NULL; + } + + php_grpc_delete_persistent_list_entry(channel->wrapper->key, + strlen(channel->wrapper->key) + TSRMLS_CC); + gpr_mu_unlock(&channel->wrapper->mu); +} + +// Delete an entry from the persistent list +// Note: this does not destroy or close the underlying grpc_channel +void php_grpc_delete_persistent_list_entry(char *key, php_grpc_int key_len + TSRMLS_DC) { + php_grpc_zend_resource *rsrc; + gpr_mu_lock(&global_persistent_list_mu); + if (PHP_GRPC_PERSISTENT_LIST_FIND(&EG(persistent_list), key, + key_len, rsrc)) { + channel_persistent_le_t *le; + le = (channel_persistent_le_t *)rsrc->ptr; + le->channel = NULL; + php_grpc_zend_hash_del(&EG(persistent_list), key, key_len+1); + } + gpr_mu_unlock(&global_persistent_list_mu); +} + +// A destructor associated with each list entry from the persistent list +static void php_grpc_channel_plink_dtor(php_grpc_zend_resource *rsrc + TSRMLS_DC) { + channel_persistent_le_t *le = (channel_persistent_le_t *)rsrc->ptr; + if (le->channel != NULL) { + gpr_mu_lock(&le->channel->mu); + if (le->channel->wrapped != NULL) { + grpc_channel_destroy(le->channel->wrapped); + free(le->channel->key); + free(le->channel); + } + gpr_mu_unlock(&le->channel->mu); } + free(le); } ZEND_BEGIN_ARG_INFO_EX(arginfo_construct, 0, 0, 2) @@ -262,10 +491,13 @@ static zend_function_entry channel_methods[] = { PHP_FE_END }; -void grpc_init_channel(TSRMLS_D) { +GRPC_STARTUP_FUNCTION(channel) { zend_class_entry ce; INIT_CLASS_ENTRY(ce, "Grpc\\Channel", channel_methods); ce.create_object = create_wrapped_grpc_channel; grpc_ce_channel = zend_register_internal_class(&ce TSRMLS_CC); + le_plink = zend_register_list_destructors_ex( + NULL, php_grpc_channel_plink_dtor, "Persistent Channel", module_number); PHP_GRPC_INIT_HANDLER(wrapped_grpc_channel, channel_ce_handlers); + return SUCCESS; } diff --git a/src/php/ext/grpc/channel.h b/src/php/ext/grpc/channel.h index 45c97441350..69adc4782c5 100755 --- a/src/php/ext/grpc/channel.h +++ b/src/php/ext/grpc/channel.h @@ -33,9 +33,18 @@ /* Class entry for the PHP Channel class */ extern zend_class_entry *grpc_ce_channel; +typedef struct _grpc_channel_wrapper { + grpc_channel *wrapped; + char *key; + char *target; + char *args_hashstr; + char *creds_hashstr; + gpr_mu mu; +} grpc_channel_wrapper; + /* Wrapper struct for grpc_channel that can be associated with a PHP object */ PHP_GRPC_WRAP_OBJECT_START(wrapped_grpc_channel) - grpc_channel *wrapped; + grpc_channel_wrapper *wrapper; PHP_GRPC_WRAP_OBJECT_END(wrapped_grpc_channel) #if PHP_MAJOR_VERSION < 7 @@ -57,10 +66,20 @@ static inline wrapped_grpc_channel #endif /* PHP_MAJOR_VERSION */ /* Initializes the Channel class */ -void grpc_init_channel(TSRMLS_D); +GRPC_STARTUP_FUNCTION(channel); /* Iterates through a PHP array and populates args with the contents */ -void php_grpc_read_args_array(zval *args_array, grpc_channel_args *args - TSRMLS_DC); +int php_grpc_read_args_array(zval *args_array, grpc_channel_args *args + TSRMLS_DC); + +void generate_sha1_str(char *sha1str, char *str, php_grpc_int len); + +void php_grpc_delete_persistent_list_entry(char *key, php_grpc_int key_len + TSRMLS_DC); + +typedef struct _channel_persistent_le { + grpc_channel_wrapper *channel; +} channel_persistent_le_t; + #endif /* NET_GRPC_PHP_GRPC_CHANNEL_H_ */ diff --git a/src/php/ext/grpc/channel_credentials.c b/src/php/ext/grpc/channel_credentials.c index 40629c8b000..19e1cefb6f5 100644 --- a/src/php/ext/grpc/channel_credentials.c +++ b/src/php/ext/grpc/channel_credentials.c @@ -26,7 +26,9 @@ #include #include #include +#include #include +#include "channel.h" #include "php_grpc.h" #include @@ -69,14 +71,17 @@ php_grpc_zend_object create_wrapped_grpc_channel_credentials( channel_credentials_ce_handlers); } -zval *grpc_php_wrap_channel_credentials(grpc_channel_credentials - *wrapped TSRMLS_DC) { +zval *grpc_php_wrap_channel_credentials(grpc_channel_credentials *wrapped, + char *hashstr, + zend_bool has_call_creds TSRMLS_DC) { zval *credentials_object; PHP_GRPC_MAKE_STD_ZVAL(credentials_object); object_init_ex(credentials_object, grpc_ce_channel_credentials); wrapped_grpc_channel_credentials *credentials = Z_WRAPPED_GRPC_CHANNEL_CREDS_P(credentials_object); credentials->wrapped = wrapped; + credentials->hashstr = hashstr; + credentials->has_call_creds = has_call_creds; return credentials_object; } @@ -106,7 +111,8 @@ PHP_METHOD(ChannelCredentials, setDefaultRootsPem) { */ PHP_METHOD(ChannelCredentials, createDefault) { grpc_channel_credentials *creds = grpc_google_default_credentials_create(); - zval *creds_object = grpc_php_wrap_channel_credentials(creds TSRMLS_CC); + zval *creds_object = grpc_php_wrap_channel_credentials(creds, NULL, false + TSRMLS_CC); RETURN_DESTROY_ZVAL(creds_object); } @@ -140,10 +146,24 @@ PHP_METHOD(ChannelCredentials, createSsl) { "createSsl expects 3 optional strings", 1 TSRMLS_CC); return; } + + php_grpc_int hashkey_len = root_certs_length + cert_chain_length; + char hashkey[hashkey_len]; + if (root_certs_length > 0) { + strcpy(hashkey, pem_root_certs); + } + if (cert_chain_length > 0) { + strcpy(hashkey, pem_key_cert_pair.cert_chain); + } + + char *hashstr = malloc(41); + generate_sha1_str(hashstr, hashkey, hashkey_len); + grpc_channel_credentials *creds = grpc_ssl_credentials_create( pem_root_certs, pem_key_cert_pair.private_key == NULL ? NULL : &pem_key_cert_pair, NULL); - zval *creds_object = grpc_php_wrap_channel_credentials(creds TSRMLS_CC); + zval *creds_object = grpc_php_wrap_channel_credentials(creds, hashstr, false + TSRMLS_CC); RETURN_DESTROY_ZVAL(creds_object); } @@ -172,7 +192,9 @@ PHP_METHOD(ChannelCredentials, createComposite) { grpc_channel_credentials *creds = grpc_composite_channel_credentials_create(cred1->wrapped, cred2->wrapped, NULL); - zval *creds_object = grpc_php_wrap_channel_credentials(creds TSRMLS_CC); + zval *creds_object = + grpc_php_wrap_channel_credentials(creds, cred1->hashstr, true + TSRMLS_CC); RETURN_DESTROY_ZVAL(creds_object); } diff --git a/src/php/ext/grpc/channel_credentials.h b/src/php/ext/grpc/channel_credentials.h index 28c7f2c1d38..357d732642f 100755 --- a/src/php/ext/grpc/channel_credentials.h +++ b/src/php/ext/grpc/channel_credentials.h @@ -38,6 +38,8 @@ extern zend_class_entry *grpc_ce_channel_credentials; * with a PHP object */ PHP_GRPC_WRAP_OBJECT_START(wrapped_grpc_channel_credentials) grpc_channel_credentials *wrapped; + char *hashstr; + zend_bool has_call_creds; PHP_GRPC_WRAP_OBJECT_END(wrapped_grpc_channel_credentials) #if PHP_MAJOR_VERSION < 7 diff --git a/src/php/ext/grpc/php7_wrapper.h b/src/php/ext/grpc/php7_wrapper.h index d4b4c262a7e..96091f9dadb 100644 --- a/src/php/ext/grpc/php7_wrapper.h +++ b/src/php/ext/grpc/php7_wrapper.h @@ -113,6 +113,20 @@ static inline int php_grpc_zend_hash_find(HashTable *ht, char *key, int len, } #define php_grpc_zend_hash_del zend_hash_del +#define php_grpc_zend_resource zend_rsrc_list_entry + +#define PHP_GRPC_BVAL_IS_TRUE(zv) Z_LVAL_P(zv) +#define PHP_GRPC_VAR_SERIALIZE(buf, zv, hash) \ + php_var_serialize(buf, &zv, hash TSRMLS_CC) +#define PHP_GRPC_SERIALIZED_BUF_STR(buf) buf.c +#define PHP_GRPC_SERIALIZED_BUF_LEN(buf) buf.len +#define PHP_GRPC_SHA1Update(cxt, str, len) \ + PHP_SHA1Update(cxt, (const unsigned char *)str, len) +#define PHP_GRPC_PERSISTENT_LIST_FIND(plist, key, len, rsrc) \ + zend_hash_find(plist, key, len+1, (void **)&rsrc) != FAILURE +#define PHP_GRPC_PERSISTENT_LIST_UPDATE(plist, key, len, rsrc) \ + zend_hash_update(plist, key, len+1, rsrc, sizeof(php_grpc_zend_resource), \ + NULL) #define PHP_GRPC_GET_CLASS_ENTRY(object) zend_get_class_entry(object TSRMLS_CC) @@ -200,6 +214,20 @@ static inline int php_grpc_zend_hash_find(HashTable *ht, char *key, int len, static inline int php_grpc_zend_hash_del(HashTable *ht, char *key, int len) { return zend_hash_str_del(ht, key, len - 1); } +#define php_grpc_zend_resource zend_resource + +#define PHP_GRPC_BVAL_IS_TRUE(zv) Z_TYPE_P(zv) == IS_TRUE +#define PHP_GRPC_VAR_SERIALIZE(buf, zv, hash) \ + php_var_serialize(buf, zv, hash) +#define PHP_GRPC_SERIALIZED_BUF_STR(buf) ZSTR_VAL(buf.s) +#define PHP_GRPC_SERIALIZED_BUF_LEN(buf) ZSTR_LEN(buf.s) +#define PHP_GRPC_SHA1Update(cxt, str, len) \ + PHP_SHA1Update(cxt, (unsigned char *)str, len) +#define PHP_GRPC_PERSISTENT_LIST_FIND(plist, key, len, rsrc) \ + (rsrc = zend_hash_str_find_ptr(plist, key, len)) != NULL +#define PHP_GRPC_PERSISTENT_LIST_UPDATE(plist, key, len, rsrc) \ + zend_hash_str_update_mem(plist, key, len, rsrc, \ + sizeof(php_grpc_zend_resource)) #define PHP_GRPC_GET_CLASS_ENTRY(object) Z_OBJ_P(object)->ce diff --git a/src/php/ext/grpc/php_grpc.c b/src/php/ext/grpc/php_grpc.c index 281b9e6aba3..a96daf7d9b8 100644 --- a/src/php/ext/grpc/php_grpc.c +++ b/src/php/ext/grpc/php_grpc.c @@ -221,7 +221,7 @@ PHP_MINIT_FUNCTION(grpc) { CONST_CS | CONST_PERSISTENT); grpc_init_call(TSRMLS_C); - grpc_init_channel(TSRMLS_C); + GRPC_STARTUP(channel); grpc_init_server(TSRMLS_C); grpc_init_timeval(TSRMLS_C); grpc_init_channel_credentials(TSRMLS_C); diff --git a/src/php/ext/grpc/php_grpc.h b/src/php/ext/grpc/php_grpc.h index ed846fdba4b..32329178dc4 100644 --- a/src/php/ext/grpc/php_grpc.h +++ b/src/php/ext/grpc/php_grpc.h @@ -74,4 +74,8 @@ ZEND_END_MODULE_GLOBALS(grpc) #define GRPC_G(v) (grpc_globals.v) #endif +#define GRPC_STARTUP_FUNCTION(module) ZEND_MINIT_FUNCTION(grpc_##module) +#define GRPC_STARTUP(module) \ + ZEND_MODULE_STARTUP_N(grpc_##module)(INIT_FUNC_ARGS_PASSTHRU) + #endif /* PHP_GRPC_H */ diff --git a/src/php/tests/unit_tests/CallTest.php b/src/php/tests/unit_tests/CallTest.php index 3270e73f821..c5e1890a98b 100644 --- a/src/php/tests/unit_tests/CallTest.php +++ b/src/php/tests/unit_tests/CallTest.php @@ -37,8 +37,7 @@ class CallTest extends PHPUnit_Framework_TestCase public function tearDown() { - unset($this->call); - unset($this->channel); + $this->channel->close(); } public function testConstructor() diff --git a/src/php/tests/unit_tests/ChannelTest.php b/src/php/tests/unit_tests/ChannelTest.php index 34e6185031a..400df0fb66d 100644 --- a/src/php/tests/unit_tests/ChannelTest.php +++ b/src/php/tests/unit_tests/ChannelTest.php @@ -25,17 +25,15 @@ class ChannelTest extends PHPUnit_Framework_TestCase public function tearDown() { - unset($this->channel); + if (!empty($this->channel)) { + $this->channel->close(); + } } public function testInsecureCredentials() { - $this->channel = new Grpc\Channel( - 'localhost:0', - [ - 'credentials' => Grpc\ChannelCredentials::createInsecure(), - ] - ); + $this->channel = new Grpc\Channel('localhost:0', + ['credentials' => Grpc\ChannelCredentials::createInsecure()]); $this->assertSame('Grpc\Channel', get_class($this->channel)); } @@ -111,7 +109,7 @@ class ChannelTest extends PHPUnit_Framework_TestCase */ public function testInvalidConstructorWith() { - $this->channel = new Grpc\Channel('localhost', 'invalid'); + $this->channel = new Grpc\Channel('localhost:0', 'invalid'); $this->assertNull($this->channel); } @@ -120,12 +118,8 @@ class ChannelTest extends PHPUnit_Framework_TestCase */ public function testInvalidCredentials() { - $this->channel = new Grpc\Channel( - 'localhost:0', - [ - 'credentials' => new Grpc\Timeval(100), - ] - ); + $this->channel = new Grpc\Channel('localhost:0', + ['credentials' => new Grpc\Timeval(100)]); } /** @@ -133,12 +127,8 @@ class ChannelTest extends PHPUnit_Framework_TestCase */ public function testInvalidOptionsArray() { - $this->channel = new Grpc\Channel( - 'localhost:0', - [ - 'abc' => [], - ] - ); + $this->channel = new Grpc\Channel('localhost:0', + ['abc' => []]); } /** @@ -170,4 +160,431 @@ class ChannelTest extends PHPUnit_Framework_TestCase ['credentials' => Grpc\ChannelCredentials::createInsecure()]); $this->channel->watchConnectivityState(1, 'hi'); } + + + public function assertConnecting($state) { + $this->assertTrue($state == GRPC\CHANNEL_CONNECTING || + $state == GRPC\CHANNEL_TRANSIENT_FAILURE); + } + + public function waitUntilNotIdle($channel) { + for ($i = 0; $i < 10; $i++) { + $now = Grpc\Timeval::now(); + $deadline = $now->add(new Grpc\Timeval(1000)); + if ($channel->watchConnectivityState(GRPC\CHANNEL_IDLE, + $deadline)) { + return true; + } + } + $this->assertTrue(false); + } + + public function testPersistentChannelSameHost() + { + $this->channel1 = new Grpc\Channel('localhost:1', []); + // the underlying grpc channel is the same by default + // when connecting to the same host + $this->channel2 = new Grpc\Channel('localhost:1', []); + + // both channels should be IDLE + $state = $this->channel1->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + // both channels should now be in the CONNECTING state + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel2->getConnectivityState(); + $this->assertConnecting($state); + + $this->channel1->close(); + $this->channel2->close(); + } + + public function testPersistentChannelDifferentHost() + { + // two different underlying channels because different hostname + $this->channel1 = new Grpc\Channel('localhost:1', []); + $this->channel2 = new Grpc\Channel('localhost:2', []); + + // both channels should be IDLE + $state = $this->channel1->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + // channel1 should now be in the CONNECTING state + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + // channel2 should still be in the IDLE state + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + $this->channel1->close(); + $this->channel2->close(); + } + + public function testPersistentChannelSameArgs() + { + $this->channel1 = new Grpc\Channel('localhost:1', ["abc" => "def"]); + $this->channel2 = new Grpc\Channel('localhost:1', ["abc" => "def"]); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel2->getConnectivityState(); + $this->assertConnecting($state); + + $this->channel1->close(); + $this->channel2->close(); + } + + public function testPersistentChannelDifferentArgs() + { + $this->channel1 = new Grpc\Channel('localhost:1', []); + $this->channel2 = new Grpc\Channel('localhost:1', ["abc" => "def"]); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + $this->channel1->close(); + $this->channel2->close(); + } + + public function testPersistentChannelSameChannelCredentials() + { + $creds1 = Grpc\ChannelCredentials::createSsl(); + $creds2 = Grpc\ChannelCredentials::createSsl(); + + $this->channel1 = new Grpc\Channel('localhost:1', + ["credentials" => $creds1]); + $this->channel2 = new Grpc\Channel('localhost:1', + ["credentials" => $creds2]); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel2->getConnectivityState(); + $this->assertConnecting($state); + + $this->channel1->close(); + $this->channel2->close(); + } + + public function testPersistentChannelDifferentChannelCredentials() + { + $creds1 = Grpc\ChannelCredentials::createSsl(); + $creds2 = Grpc\ChannelCredentials::createSsl( + file_get_contents(dirname(__FILE__).'/../data/ca.pem')); + + $this->channel1 = new Grpc\Channel('localhost:1', + ["credentials" => $creds1]); + $this->channel2 = new Grpc\Channel('localhost:1', + ["credentials" => $creds2]); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + $this->channel1->close(); + $this->channel2->close(); + } + + public function testPersistentChannelSameChannelCredentialsRootCerts() + { + $creds1 = Grpc\ChannelCredentials::createSsl( + file_get_contents(dirname(__FILE__).'/../data/ca.pem')); + $creds2 = Grpc\ChannelCredentials::createSsl( + file_get_contents(dirname(__FILE__).'/../data/ca.pem')); + + $this->channel1 = new Grpc\Channel('localhost:1', + ["credentials" => $creds1]); + $this->channel2 = new Grpc\Channel('localhost:1', + ["credentials" => $creds2]); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel2->getConnectivityState(); + $this->assertConnecting($state); + + $this->channel1->close(); + $this->channel2->close(); + } + + public function testPersistentChannelDifferentSecureChannelCredentials() + { + $creds1 = Grpc\ChannelCredentials::createSsl(); + $creds2 = Grpc\ChannelCredentials::createInsecure(); + + $this->channel1 = new Grpc\Channel('localhost:1', + ["credentials" => $creds1]); + $this->channel2 = new Grpc\Channel('localhost:1', + ["credentials" => $creds2]); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + $this->channel1->close(); + $this->channel2->close(); + } + + /** + * @expectedException RuntimeException + */ + public function testPersistentChannelSharedChannelClose() + { + // same underlying channel + $this->channel1 = new Grpc\Channel('localhost:1', []); + $this->channel2 = new Grpc\Channel('localhost:1', []); + + // close channel1 + $this->channel1->close(); + + // channel2 is now in SHUTDOWN state + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_FATAL_FAILURE, $state); + + // calling it again will result in an exception because the + // channel is already closed + $state = $this->channel2->getConnectivityState(); + } + + public function testPersistentChannelCreateAfterClose() + { + $this->channel1 = new Grpc\Channel('localhost:1', []); + + $this->channel1->close(); + + $this->channel2 = new Grpc\Channel('localhost:1', []); + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + $this->channel2->close(); + } + + public function testPersistentChannelSharedMoreThanTwo() + { + $this->channel1 = new Grpc\Channel('localhost:1', []); + $this->channel2 = new Grpc\Channel('localhost:1', []); + $this->channel3 = new Grpc\Channel('localhost:1', []); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + // all 3 channels should be in CONNECTING state + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel2->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel3->getConnectivityState(); + $this->assertConnecting($state); + + $this->channel1->close(); + } + + public function callbackFunc($context) + { + return []; + } + + public function callbackFunc2($context) + { + return ["k1" => "v1"]; + } + + public function testPersistentChannelWithCallCredentials() + { + $creds = Grpc\ChannelCredentials::createSsl(); + $callCreds = Grpc\CallCredentials::createFromPlugin( + [$this, 'callbackFunc']); + $credsWithCallCreds = Grpc\ChannelCredentials::createComposite( + $creds, $callCreds); + + // If a ChannelCredentials object is composed with a + // CallCredentials object, the underlying grpc channel will + // always be created new and NOT persisted. + $this->channel1 = new Grpc\Channel('localhost:1', + ["credentials" => + $credsWithCallCreds]); + $this->channel2 = new Grpc\Channel('localhost:1', + ["credentials" => + $credsWithCallCreds]); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + $this->channel1->close(); + $this->channel2->close(); + } + + public function testPersistentChannelWithDifferentCallCredentials() + { + $callCreds1 = Grpc\CallCredentials::createFromPlugin( + [$this, 'callbackFunc']); + $callCreds2 = Grpc\CallCredentials::createFromPlugin( + [$this, 'callbackFunc2']); + + $creds1 = Grpc\ChannelCredentials::createSsl(); + $creds2 = Grpc\ChannelCredentials::createComposite( + $creds1, $callCreds1); + $creds3 = Grpc\ChannelCredentials::createComposite( + $creds1, $callCreds2); + + // Similar to the test above, anytime a ChannelCredentials + // object is composed with a CallCredentials object, the + // underlying grpc channel will always be separate and not + // persisted + $this->channel1 = new Grpc\Channel('localhost:1', + ["credentials" => $creds1]); + $this->channel2 = new Grpc\Channel('localhost:1', + ["credentials" => $creds2]); + $this->channel3 = new Grpc\Channel('localhost:1', + ["credentials" => $creds3]); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + $state = $this->channel3->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + $this->channel1->close(); + $this->channel2->close(); + $this->channel3->close(); + } + + public function testPersistentChannelForceNew() + { + $this->channel1 = new Grpc\Channel('localhost:1', []); + // even though all the channel params are the same, channel2 + // has a new and different underlying channel + $this->channel2 = new Grpc\Channel('localhost:1', + ["force_new" => true]); + + // try to connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + // any dangling old connection to the same host must be + // manually closed + $this->channel1->close(); + $this->channel2->close(); + } + + public function testPersistentChannelForceNewOldChannelIdle() + { + + $this->channel1 = new Grpc\Channel('localhost:1', []); + $this->channel2 = new Grpc\Channel('localhost:1', + ["force_new" => true]); + $this->channel3 = new Grpc\Channel('localhost:1', []); + + // try to connect on channel2 + $state = $this->channel2->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel2); + + $state = $this->channel1->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + $state = $this->channel2->getConnectivityState(); + $this->assertConnecting($state); + $state = $this->channel3->getConnectivityState(); + $this->assertConnecting($state); + + $this->channel1->close(); + $this->channel2->close(); + } + + public function testPersistentChannelForceNewOldChannelClose() + { + + $this->channel1 = new Grpc\Channel('localhost:1', []); + $this->channel2 = new Grpc\Channel('localhost:1', + ["force_new" => true]); + $this->channel3 = new Grpc\Channel('localhost:1', []); + + $this->channel1->close(); + + $state = $this->channel2->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + $state = $this->channel3->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + $this->channel2->close(); + $this->channel3->close(); + } + + public function testPersistentChannelForceNewNewChannelClose() + { + + $this->channel1 = new Grpc\Channel('localhost:1', []); + $this->channel2 = new Grpc\Channel('localhost:1', + ["force_new" => true]); + $this->channel3 = new Grpc\Channel('localhost:1', []); + + $this->channel2->close(); + + $state = $this->channel1->getConnectivityState(); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); + + // can still connect on channel1 + $state = $this->channel1->getConnectivityState(true); + $this->waitUntilNotIdle($this->channel1); + + $state = $this->channel1->getConnectivityState(); + $this->assertConnecting($state); + + $this->channel1->close(); + } } diff --git a/src/php/tests/unit_tests/EndToEndTest.php b/src/php/tests/unit_tests/EndToEndTest.php index 43d54d9ee60..b54f1d87c9f 100644 --- a/src/php/tests/unit_tests/EndToEndTest.php +++ b/src/php/tests/unit_tests/EndToEndTest.php @@ -28,8 +28,7 @@ class EndToEndTest extends PHPUnit_Framework_TestCase public function tearDown() { - unset($this->channel); - unset($this->server); + $this->channel->close(); } public function testSimpleRequestBody() @@ -516,7 +515,7 @@ class EndToEndTest extends PHPUnit_Framework_TestCase $this->assertTrue($idle_state == Grpc\CHANNEL_IDLE); $now = Grpc\Timeval::now(); - $delta = new Grpc\Timeval(500000); // should timeout + $delta = new Grpc\Timeval(50000); // should timeout $deadline = $now->add($delta); $this->assertFalse($this->channel->watchConnectivityState( @@ -545,7 +544,7 @@ class EndToEndTest extends PHPUnit_Framework_TestCase $this->assertTrue($idle_state == Grpc\CHANNEL_IDLE); $now = Grpc\Timeval::now(); - $delta = new Grpc\Timeval(100000); + $delta = new Grpc\Timeval(50000); $deadline = $now->add($delta); $this->assertFalse($this->channel->watchConnectivityState( diff --git a/src/php/tests/unit_tests/SecureEndToEndTest.php b/src/php/tests/unit_tests/SecureEndToEndTest.php index 0fecbfb3dd3..dff4e878ea1 100644 --- a/src/php/tests/unit_tests/SecureEndToEndTest.php +++ b/src/php/tests/unit_tests/SecureEndToEndTest.php @@ -43,8 +43,7 @@ class SecureEndToEndTest extends PHPUnit_Framework_TestCase public function tearDown() { - unset($this->channel); - unset($this->server); + $this->channel->close(); } public function testSimpleRequestBody() diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index aa074df8492..dc4d28f95bd 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -92,6 +92,9 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/ev_windows.c', 'src/core/lib/iomgr/exec_ctx.c', 'src/core/lib/iomgr/executor.c', + 'src/core/lib/iomgr/gethostname_fallback.c', + 'src/core/lib/iomgr/gethostname_host_name_max.c', + 'src/core/lib/iomgr/gethostname_sysconf.c', 'src/core/lib/iomgr/iocp_windows.c', 'src/core/lib/iomgr/iomgr.c', 'src/core/lib/iomgr/iomgr_posix.c', @@ -243,6 +246,7 @@ CORE_SOURCE_FILES = [ 'src/core/tsi/fake_transport_security.c', 'src/core/tsi/gts_transport_security.c', 'src/core/tsi/ssl_transport_security.c', + 'src/core/tsi/transport_security_grpc.c', 'src/core/tsi/transport_security.c', 'src/core/tsi/transport_security_adapter.c', 'src/core/ext/transport/chttp2/server/chttp2_server.c', diff --git a/src/python/grpcio_testing/grpc_testing/__init__.py b/src/python/grpcio_testing/grpc_testing/__init__.py index c5a17f457a4..14e25f09e23 100644 --- a/src/python/grpcio_testing/grpc_testing/__init__.py +++ b/src/python/grpcio_testing/grpc_testing/__init__.py @@ -15,11 +15,284 @@ import abc +from google.protobuf import descriptor import six import grpc +class UnaryUnaryChannelRpc(six.with_metaclass(abc.ABCMeta)): + """Fixture for a unary-unary RPC invoked by a system under test. + + Enables users to "play server" for the RPC. + """ + + @abc.abstractmethod + def send_initial_metadata(self, initial_metadata): + """Sends the RPC's initial metadata to the system under test. + + Args: + initial_metadata: The RPC's initial metadata to be "sent" to + the system under test. + """ + raise NotImplementedError() + + @abc.abstractmethod + def cancelled(self): + """Blocks until the system under test has cancelled the RPC.""" + raise NotImplementedError() + + @abc.abstractmethod + def terminate(self, response, trailing_metadata, code, details): + """Terminates the RPC. + + Args: + response: The response for the RPC. + trailing_metadata: The RPC's trailing metadata. + code: The RPC's status code. + details: The RPC's status details. + """ + raise NotImplementedError() + + +class UnaryStreamChannelRpc(six.with_metaclass(abc.ABCMeta)): + """Fixture for a unary-stream RPC invoked by a system under test. + + Enables users to "play server" for the RPC. + """ + + @abc.abstractmethod + def send_initial_metadata(self, initial_metadata): + """Sends the RPC's initial metadata to the system under test. + + Args: + initial_metadata: The RPC's initial metadata to be "sent" to + the system under test. + """ + raise NotImplementedError() + + @abc.abstractmethod + def send_response(self, response): + """Sends a response to the system under test. + + Args: + response: A response message to be "sent" to the system under test. + """ + raise NotImplementedError() + + @abc.abstractmethod + def cancelled(self): + """Blocks until the system under test has cancelled the RPC.""" + raise NotImplementedError() + + @abc.abstractmethod + def terminate(self, trailing_metadata, code, details): + """Terminates the RPC. + + Args: + trailing_metadata: The RPC's trailing metadata. + code: The RPC's status code. + details: The RPC's status details. + """ + raise NotImplementedError() + + +class StreamUnaryChannelRpc(six.with_metaclass(abc.ABCMeta)): + """Fixture for a stream-unary RPC invoked by a system under test. + + Enables users to "play server" for the RPC. + """ + + @abc.abstractmethod + def send_initial_metadata(self, initial_metadata): + """Sends the RPC's initial metadata to the system under test. + + Args: + initial_metadata: The RPC's initial metadata to be "sent" to + the system under test. + """ + raise NotImplementedError() + + @abc.abstractmethod + def take_request(self): + """Draws one of the requests added to the RPC by the system under test. + + This method blocks until the system under test has added to the RPC + the request to be returned. + + Successive calls to this method return requests in the same order in + which the system under test added them to the RPC. + + Returns: + A request message added to the RPC by the system under test. + """ + raise NotImplementedError() + + @abc.abstractmethod + def requests_closed(self): + """Blocks until the system under test has closed the request stream.""" + raise NotImplementedError() + + @abc.abstractmethod + def cancelled(self): + """Blocks until the system under test has cancelled the RPC.""" + raise NotImplementedError() + + @abc.abstractmethod + def terminate(self, response, trailing_metadata, code, details): + """Terminates the RPC. + + Args: + response: The response for the RPC. + trailing_metadata: The RPC's trailing metadata. + code: The RPC's status code. + details: The RPC's status details. + """ + raise NotImplementedError() + + +class StreamStreamChannelRpc(six.with_metaclass(abc.ABCMeta)): + """Fixture for a stream-stream RPC invoked by a system under test. + + Enables users to "play server" for the RPC. + """ + + @abc.abstractmethod + def send_initial_metadata(self, initial_metadata): + """Sends the RPC's initial metadata to the system under test. + + Args: + initial_metadata: The RPC's initial metadata to be "sent" to the + system under test. + """ + raise NotImplementedError() + + @abc.abstractmethod + def take_request(self): + """Draws one of the requests added to the RPC by the system under test. + + This method blocks until the system under test has added to the RPC + the request to be returned. + + Successive calls to this method return requests in the same order in + which the system under test added them to the RPC. + + Returns: + A request message added to the RPC by the system under test. + """ + raise NotImplementedError() + + @abc.abstractmethod + def send_response(self, response): + """Sends a response to the system under test. + + Args: + response: A response messages to be "sent" to the system under test. + """ + raise NotImplementedError() + + @abc.abstractmethod + def requests_closed(self): + """Blocks until the system under test has closed the request stream.""" + raise NotImplementedError() + + @abc.abstractmethod + def cancelled(self): + """Blocks until the system under test has cancelled the RPC.""" + raise NotImplementedError() + + @abc.abstractmethod + def terminate(self, trailing_metadata, code, details): + """Terminates the RPC. + + Args: + trailing_metadata: The RPC's trailing metadata. + code: The RPC's status code. + details: The RPC's status details. + """ + raise NotImplementedError() + + +class Channel(six.with_metaclass(abc.ABCMeta), grpc.Channel): + """A grpc.Channel double with which to test a system that invokes RPCs.""" + + @abc.abstractmethod + def take_unary_unary(self, method_descriptor): + """Draws an RPC currently being made by the system under test. + + If the given descriptor does not identify any RPC currently being made + by the system under test, this method blocks until the system under + test invokes such an RPC. + + Args: + method_descriptor: A descriptor.MethodDescriptor describing a + unary-unary RPC method. + + Returns: + A (invocation_metadata, request, unary_unary_channel_rpc) tuple of + the RPC's invocation metadata, its request, and a + UnaryUnaryChannelRpc with which to "play server" for the RPC. + """ + raise NotImplementedError() + + @abc.abstractmethod + def take_unary_stream(self, method_descriptor): + """Draws an RPC currently being made by the system under test. + + If the given descriptor does not identify any RPC currently being made + by the system under test, this method blocks until the system under + test invokes such an RPC. + + Args: + method_descriptor: A descriptor.MethodDescriptor describing a + unary-stream RPC method. + + Returns: + A (invocation_metadata, request, unary_stream_channel_rpc) tuple of + the RPC's invocation metadata, its request, and a + UnaryStreamChannelRpc with which to "play server" for the RPC. + """ + raise NotImplementedError() + + @abc.abstractmethod + def take_stream_unary(self, method_descriptor): + """Draws an RPC currently being made by the system under test. + + If the given descriptor does not identify any RPC currently being made + by the system under test, this method blocks until the system under + test invokes such an RPC. + + Args: + method_descriptor: A descriptor.MethodDescriptor describing a + stream-unary RPC method. + + Returns: + A (invocation_metadata, stream_unary_channel_rpc) tuple of the RPC's + invocation metadata and a StreamUnaryChannelRpc with which to "play + server" for the RPC. + """ + raise NotImplementedError() + + @abc.abstractmethod + def take_stream_stream(self, method_descriptor): + """Draws an RPC currently being made by the system under test. + + If the given descriptor does not identify any RPC currently being made + by the system under test, this method blocks until the system under + test invokes such an RPC. + + Args: + method_descriptor: A descriptor.MethodDescriptor describing a + stream-stream RPC method. + + Returns: + A (invocation_metadata, stream_stream_channel_rpc) tuple of the RPC's + invocation metadata and a StreamStreamChannelRpc with which to + "play server" for the RPC. + """ + raise NotImplementedError() + + class Time(six.with_metaclass(abc.ABCMeta)): """A simulation of time. @@ -117,3 +390,19 @@ def strict_fake_time(now): """ from grpc_testing import _time return _time.StrictFakeTime(now) + + +def channel(service_descriptors, time): + """Creates a Channel for use in tests of a gRPC Python-using system. + + Args: + service_descriptors: An iterable of descriptor.ServiceDescriptors + describing the RPCs that will be made on the returned Channel by the + system under test. + time: A Time to be used for tests. + + Returns: + A Channel for use in tests. + """ + from grpc_testing import _channel + return _channel.testing_channel(service_descriptors, time) diff --git a/src/python/grpcio_testing/grpc_testing/_channel/__init__.py b/src/python/grpcio_testing/grpc_testing/_channel/__init__.py new file mode 100644 index 00000000000..8011975d0aa --- /dev/null +++ b/src/python/grpcio_testing/grpc_testing/_channel/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from grpc_testing._channel import _channel +from grpc_testing._channel import _channel_state + + +# descriptors is reserved for later use. +# pylint: disable=unused-argument +def testing_channel(descriptors, time): + return _channel.TestingChannel(time, _channel_state.State()) +# pylint: enable=unused-argument diff --git a/src/python/grpcio_testing/grpc_testing/_channel/_channel.py b/src/python/grpcio_testing/grpc_testing/_channel/_channel.py new file mode 100644 index 00000000000..fbd064db885 --- /dev/null +++ b/src/python/grpcio_testing/grpc_testing/_channel/_channel.py @@ -0,0 +1,62 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc_testing +from grpc_testing._channel import _channel_rpc +from grpc_testing._channel import _multi_callable + + +# All serializer and deserializer parameters are not (yet) used by this +# test infrastructure. +# pylint: disable=unused-argument +class TestingChannel(grpc_testing.Channel): + + def __init__(self, time, state): + self._time = time + self._state = state + + def subscribe(self, callback, try_to_connect=False): + raise NotImplementedError() + + def unsubscribe(self, callback): + raise NotImplementedError() + + def unary_unary( + self, method, request_serializer=None, response_deserializer=None): + return _multi_callable.UnaryUnary(method, self._state) + + def unary_stream( + self, method, request_serializer=None, response_deserializer=None): + return _multi_callable.UnaryStream(method, self._state) + + def stream_unary( + self, method, request_serializer=None, response_deserializer=None): + return _multi_callable.StreamUnary(method, self._state) + + def stream_stream( + self, method, request_serializer=None, response_deserializer=None): + return _multi_callable.StreamStream(method, self._state) + + def take_unary_unary(self, method_descriptor): + return _channel_rpc.unary_unary(self._state, method_descriptor) + + def take_unary_stream(self, method_descriptor): + return _channel_rpc.unary_stream(self._state, method_descriptor) + + def take_stream_unary(self, method_descriptor): + return _channel_rpc.stream_unary(self._state, method_descriptor) + + def take_stream_stream(self, method_descriptor): + return _channel_rpc.stream_stream(self._state, method_descriptor) +# pylint: enable=unused-argument diff --git a/src/python/grpcio_testing/grpc_testing/_channel/_channel_rpc.py b/src/python/grpcio_testing/grpc_testing/_channel/_channel_rpc.py new file mode 100644 index 00000000000..762b6a035bf --- /dev/null +++ b/src/python/grpcio_testing/grpc_testing/_channel/_channel_rpc.py @@ -0,0 +1,119 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc_testing + + +class _UnaryUnary(grpc_testing.UnaryUnaryChannelRpc): + + def __init__(self, rpc_state): + self._rpc_state = rpc_state + + def send_initial_metadata(self, initial_metadata): + self._rpc_state.send_initial_metadata(initial_metadata) + + def cancelled(self): + self._rpc_state.cancelled() + + def terminate(self, response, trailing_metadata, code, details): + self._rpc_state.terminate_with_response( + response, trailing_metadata, code, details) + + +class _UnaryStream(grpc_testing.UnaryStreamChannelRpc): + + def __init__(self, rpc_state): + self._rpc_state = rpc_state + + def send_initial_metadata(self, initial_metadata): + self._rpc_state.send_initial_metadata(initial_metadata) + + def send_response(self, response): + self._rpc_state.send_response(response) + + def cancelled(self): + self._rpc_state.cancelled() + + def terminate(self, trailing_metadata, code, details): + self._rpc_state.terminate(trailing_metadata, code, details) + + +class _StreamUnary(grpc_testing.StreamUnaryChannelRpc): + + def __init__(self, rpc_state): + self._rpc_state = rpc_state + + def send_initial_metadata(self, initial_metadata): + self._rpc_state.send_initial_metadata(initial_metadata) + + def take_request(self): + return self._rpc_state.take_request() + + def requests_closed(self): + return self._rpc_state.requests_closed() + + def cancelled(self): + self._rpc_state.cancelled() + + def terminate(self, response, trailing_metadata, code, details): + self._rpc_state.terminate_with_response( + response, trailing_metadata, code, details) + + +class _StreamStream(grpc_testing.StreamStreamChannelRpc): + + def __init__(self, rpc_state): + self._rpc_state = rpc_state + + def send_initial_metadata(self, initial_metadata): + self._rpc_state.send_initial_metadata(initial_metadata) + + def take_request(self): + return self._rpc_state.take_request() + + def send_response(self, response): + self._rpc_state.send_response(response) + + def requests_closed(self): + return self._rpc_state.requests_closed() + + def cancelled(self): + self._rpc_state.cancelled() + + def terminate(self, trailing_metadata, code, details): + self._rpc_state.terminate(trailing_metadata, code, details) + + +def unary_unary(channel_state, method_descriptor): + rpc_state = channel_state.take_rpc_state(method_descriptor) + invocation_metadata, request = ( + rpc_state.take_invocation_metadata_and_request()) + return invocation_metadata, request, _UnaryUnary(rpc_state) + + +def unary_stream(channel_state, method_descriptor): + rpc_state = channel_state.take_rpc_state(method_descriptor) + invocation_metadata, request = ( + rpc_state.take_invocation_metadata_and_request()) + return invocation_metadata, request, _UnaryStream(rpc_state) + + +def stream_unary(channel_state, method_descriptor): + rpc_state = channel_state.take_rpc_state(method_descriptor) + return rpc_state.take_invocation_metadata(), _StreamUnary(rpc_state) + + +def stream_stream(channel_state, method_descriptor): + rpc_state = channel_state.take_rpc_state(method_descriptor) + return rpc_state.take_invocation_metadata(), _StreamStream(rpc_state) diff --git a/src/python/grpcio_testing/grpc_testing/_channel/_channel_state.py b/src/python/grpcio_testing/grpc_testing/_channel/_channel_state.py new file mode 100644 index 00000000000..569c41d79d7 --- /dev/null +++ b/src/python/grpcio_testing/grpc_testing/_channel/_channel_state.py @@ -0,0 +1,48 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import threading + +from grpc_testing import _common +from grpc_testing._channel import _rpc_state + + +class State(_common.ChannelHandler): + + def __init__(self): + self._condition = threading.Condition() + self._rpc_states = collections.defaultdict(list) + + def invoke_rpc( + self, method_full_rpc_name, invocation_metadata, requests, + requests_closed, timeout): + rpc_state = _rpc_state.State( + invocation_metadata, requests, requests_closed) + with self._condition: + self._rpc_states[method_full_rpc_name].append(rpc_state) + self._condition.notify_all() + return rpc_state + + def take_rpc_state(self, method_descriptor): + method_full_rpc_name = '/{}/{}'.format( + method_descriptor.containing_service.full_name, + method_descriptor.name) + with self._condition: + while True: + method_rpc_states = self._rpc_states[method_full_rpc_name] + if method_rpc_states: + return method_rpc_states.pop(0) + else: + self._condition.wait() diff --git a/src/python/grpcio_testing/grpc_testing/_channel/_invocation.py b/src/python/grpcio_testing/grpc_testing/_channel/_invocation.py new file mode 100644 index 00000000000..ebce652eeb0 --- /dev/null +++ b/src/python/grpcio_testing/grpc_testing/_channel/_invocation.py @@ -0,0 +1,322 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import threading + +import grpc + +_NOT_YET_OBSERVED = object() + + +def _cancel(handler): + return handler.cancel(grpc.StatusCode.CANCELLED, 'Locally cancelled!') + + +def _is_active(handler): + return handler.is_active() + + +def _time_remaining(unused_handler): + raise NotImplementedError() + + +def _add_callback(handler, callback): + return handler.add_callback(callback) + + +def _initial_metadata(handler): + return handler.initial_metadata() + + +def _trailing_metadata(handler): + trailing_metadata, unused_code, unused_details = handler.termination() + return trailing_metadata + + +def _code(handler): + unused_trailing_metadata, code, unused_details = handler.termination() + return code + + +def _details(handler): + unused_trailing_metadata, unused_code, details = handler.termination() + return details + + +class _Call(grpc.Call): + + def __init__(self, handler): + self._handler = handler + + def cancel(self): + _cancel(self._handler) + + def is_active(self): + return _is_active(self._handler) + + def time_remaining(self): + return _time_remaining(self._handler) + + def add_callback(self, callback): + return _add_callback(self._handler, callback) + + def initial_metadata(self): + return _initial_metadata(self._handler) + + def trailing_metadata(self): + return _trailing_metadata(self._handler) + + def code(self): + return _code(self._handler) + + def details(self): + return _details(self._handler) + + +class _RpcErrorCall(grpc.RpcError, grpc.Call): + + def __init__(self, handler): + self._handler = handler + + def cancel(self): + _cancel(self._handler) + + def is_active(self): + return _is_active(self._handler) + + def time_remaining(self): + return _time_remaining(self._handler) + + def add_callback(self, callback): + return _add_callback(self._handler, callback) + + def initial_metadata(self): + return _initial_metadata(self._handler) + + def trailing_metadata(self): + return _trailing_metadata(self._handler) + + def code(self): + return _code(self._handler) + + def details(self): + return _details(self._handler) + + +def _next(handler): + read = handler.take_response() + if read.code is None: + return read.response + elif read.code is grpc.StatusCode.OK: + raise StopIteration() + else: + raise _RpcErrorCall(handler) + + +class _HandlerExtras(object): + + def __init__(self): + self.condition = threading.Condition() + self.unary_response = _NOT_YET_OBSERVED + self.cancelled = False + + +def _with_extras_cancel(handler, extras): + with extras.condition: + if handler.cancel(grpc.StatusCode.CANCELLED, 'Locally cancelled!'): + extras.cancelled = True + return True + else: + return False + + +def _extras_without_cancelled(extras): + with extras.condition: + return extras.cancelled + + +def _running(handler): + return handler.is_active() + + +def _done(handler): + return not handler.is_active() + + +def _with_extras_unary_response(handler, extras): + with extras.condition: + if extras.unary_response is _NOT_YET_OBSERVED: + read = handler.take_response() + if read.code is None: + extras.unary_response = read.response + return read.response + else: + raise _RpcErrorCall(handler) + else: + return extras.unary_response + + +def _exception(unused_handler): + raise NotImplementedError('TODO!') + + +def _traceback(unused_handler): + raise NotImplementedError('TODO!') + + +def _add_done_callback(handler, callback, future): + adapted_callback = lambda: callback(future) + if not handler.add_callback(adapted_callback): + callback(future) + + +class _FutureCall(grpc.Future, grpc.Call): + + def __init__(self, handler, extras): + self._handler = handler + self._extras = extras + + def cancel(self): + return _with_extras_cancel(self._handler, self._extras) + + def cancelled(self): + return _extras_without_cancelled(self._extras) + + def running(self): + return _running(self._handler) + + def done(self): + return _done(self._handler) + + def result(self): + return _with_extras_unary_response(self._handler, self._extras) + + def exception(self): + return _exception(self._handler) + + def traceback(self): + return _traceback(self._handler) + + def add_done_callback(self, fn): + _add_done_callback(self._handler, fn, self) + + def is_active(self): + return _is_active(self._handler) + + def time_remaining(self): + return _time_remaining(self._handler) + + def add_callback(self, callback): + return _add_callback(self._handler, callback) + + def initial_metadata(self): + return _initial_metadata(self._handler) + + def trailing_metadata(self): + return _trailing_metadata(self._handler) + + def code(self): + return _code(self._handler) + + def details(self): + return _details(self._handler) + + +def consume_requests(request_iterator, handler): + + def _consume(): + while True: + try: + request = next(request_iterator) + added = handler.add_request(request) + if not added: + break + except StopIteration: + handler.close_requests() + break + except Exception: # pylint: disable=broad-except + details = 'Exception iterating requests!' + logging.exception(details) + handler.cancel(grpc.StatusCode.UNKNOWN, details) + + consumption = threading.Thread(target=_consume) + consumption.start() + + +def blocking_unary_response(handler): + read = handler.take_response() + if read.code is None: + unused_trailing_metadata, code, unused_details = handler.termination() + if code is grpc.StatusCode.OK: + return read.response + else: + raise _RpcErrorCall(handler) + else: + raise _RpcErrorCall(handler) + + +def blocking_unary_response_with_call(handler): + read = handler.take_response() + if read.code is None: + unused_trailing_metadata, code, unused_details = handler.termination() + if code is grpc.StatusCode.OK: + return read.response, _Call(handler) + else: + raise _RpcErrorCall(handler) + else: + raise _RpcErrorCall(handler) + + +def future_call(handler): + return _FutureCall(handler, _HandlerExtras()) + + +class ResponseIteratorCall(grpc.Call): + + def __init__(self, handler): + self._handler = handler + + def __iter__(self): + return self + + def __next__(self): + return _next(self._handler) + + def next(self): + return _next(self._handler) + + def cancel(self): + _cancel(self._handler) + + def is_active(self): + return _is_active(self._handler) + + def time_remaining(self): + return _time_remaining(self._handler) + + def add_callback(self, callback): + return _add_callback(self._handler, callback) + + def initial_metadata(self): + return _initial_metadata(self._handler) + + def trailing_metadata(self): + return _trailing_metadata(self._handler) + + def code(self): + return _code(self._handler) + + def details(self): + return _details(self._handler) diff --git a/src/python/grpcio_testing/grpc_testing/_channel/_multi_callable.py b/src/python/grpcio_testing/grpc_testing/_channel/_multi_callable.py new file mode 100644 index 00000000000..fe69257f5be --- /dev/null +++ b/src/python/grpcio_testing/grpc_testing/_channel/_multi_callable.py @@ -0,0 +1,115 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc +from grpc_testing import _common +from grpc_testing._channel import _invocation + +# All per-call credentials parameters are unused by this test infrastructure. +# pylint: disable=unused-argument +class UnaryUnary(grpc.UnaryUnaryMultiCallable): + + def __init__(self, method_full_rpc_name, channel_handler): + self._method_full_rpc_name = method_full_rpc_name + self._channel_handler = channel_handler + + def __call__(self, request, timeout=None, metadata=None, credentials=None): + rpc_handler = self._channel_handler.invoke_rpc( + self._method_full_rpc_name, _common.fuss_with_metadata(metadata), + [request], True, timeout) + return _invocation.blocking_unary_response(rpc_handler) + + def with_call(self, request, timeout=None, metadata=None, credentials=None): + rpc_handler = self._channel_handler.invoke_rpc( + self._method_full_rpc_name, _common.fuss_with_metadata(metadata), + [request], True, timeout) + return _invocation.blocking_unary_response_with_call(rpc_handler) + + def future(self, request, timeout=None, metadata=None, credentials=None): + rpc_handler = self._channel_handler.invoke_rpc( + self._method_full_rpc_name, _common.fuss_with_metadata(metadata), + [request], True, timeout) + return _invocation.future_call(rpc_handler) + + +class UnaryStream(grpc.StreamStreamMultiCallable): + + def __init__(self, method_full_rpc_name, channel_handler): + self._method_full_rpc_name = method_full_rpc_name + self._channel_handler = channel_handler + + def __call__(self, request, timeout=None, metadata=None, credentials=None): + rpc_handler = self._channel_handler.invoke_rpc( + self._method_full_rpc_name, + _common.fuss_with_metadata(metadata), [request], True, timeout) + return _invocation.ResponseIteratorCall(rpc_handler) + + +class StreamUnary(grpc.StreamUnaryMultiCallable): + + def __init__(self, method_full_rpc_name, channel_handler): + self._method_full_rpc_name = method_full_rpc_name + self._channel_handler = channel_handler + + def __call__(self, + request_iterator, + timeout=None, + metadata=None, + credentials=None): + rpc_handler = self._channel_handler.invoke_rpc( + self._method_full_rpc_name, + _common.fuss_with_metadata(metadata), [], False, timeout) + _invocation.consume_requests(request_iterator, rpc_handler) + return _invocation.blocking_unary_response(rpc_handler) + + def with_call(self, + request_iterator, + timeout=None, + metadata=None, + credentials=None): + rpc_handler = self._channel_handler.invoke_rpc( + self._method_full_rpc_name, + _common.fuss_with_metadata(metadata), [], False, timeout) + _invocation.consume_requests(request_iterator, rpc_handler) + return _invocation.blocking_unary_response_with_call(rpc_handler) + + def future(self, + request_iterator, + timeout=None, + metadata=None, + credentials=None): + rpc_handler = self._channel_handler.invoke_rpc( + self._method_full_rpc_name, + _common.fuss_with_metadata(metadata), [], False, timeout) + _invocation.consume_requests(request_iterator, rpc_handler) + return _invocation.future_call(rpc_handler) + + +class StreamStream(grpc.StreamStreamMultiCallable): + + def __init__(self, method_full_rpc_name, channel_handler): + self._method_full_rpc_name = method_full_rpc_name + self._channel_handler = channel_handler + + def __call__(self, + request_iterator, + timeout=None, + metadata=None, + credentials=None): + rpc_handler = self._channel_handler.invoke_rpc( + self._method_full_rpc_name, + _common.fuss_with_metadata(metadata), [], False, timeout) + _invocation.consume_requests(request_iterator, rpc_handler) + return _invocation.ResponseIteratorCall(rpc_handler) +# pylint: enable=unused-argument diff --git a/src/python/grpcio_testing/grpc_testing/_channel/_rpc_state.py b/src/python/grpcio_testing/grpc_testing/_channel/_rpc_state.py new file mode 100644 index 00000000000..e1fa49a2a80 --- /dev/null +++ b/src/python/grpcio_testing/grpc_testing/_channel/_rpc_state.py @@ -0,0 +1,193 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading + +import grpc +from grpc_testing import _common + + +class State(_common.ChannelRpcHandler): + + def __init__(self, invocation_metadata, requests, requests_closed): + self._condition = threading.Condition() + self._invocation_metadata = invocation_metadata + self._requests = requests + self._requests_closed = requests_closed + self._initial_metadata = None + self._responses = [] + self._trailing_metadata = None + self._code = None + self._details = None + + def initial_metadata(self): + with self._condition: + while True: + if self._initial_metadata is None: + if self._code is None: + self._condition.wait() + else: + return _common.FUSSED_EMPTY_METADATA + else: + return self._initial_metadata + + def add_request(self, request): + with self._condition: + if self._code is None and not self._requests_closed: + self._requests.append(request) + self._condition.notify_all() + return True + else: + return False + + def close_requests(self): + with self._condition: + if self._code is None and not self._requests_closed: + self._requests_closed = True + self._condition.notify_all() + + def take_response(self): + with self._condition: + while True: + if self._code is grpc.StatusCode.OK: + if self._responses: + response = self._responses.pop(0) + return _common.ChannelRpcRead( + response, None, None, None) + else: + return _common.ChannelRpcRead( + None, self._trailing_metadata, + grpc.StatusCode.OK, self._details) + elif self._code is None: + if self._responses: + response = self._responses.pop(0) + return _common.ChannelRpcRead( + response, None, None, None) + else: + self._condition.wait() + else: + return _common.ChannelRpcRead( + None, self._trailing_metadata, self._code, + self._details) + + def termination(self): + with self._condition: + while True: + if self._code is None: + self._condition.wait() + else: + return self._trailing_metadata, self._code, self._details + + def cancel(self, code, details): + with self._condition: + if self._code is None: + if self._initial_metadata is None: + self._initial_metadata = _common.FUSSED_EMPTY_METADATA + self._trailing_metadata = _common.FUSSED_EMPTY_METADATA + self._code = code + self._details = details + self._condition.notify_all() + return True + else: + return False + + def take_invocation_metadata(self): + with self._condition: + if self._invocation_metadata is None: + raise ValueError('Expected invocation metadata!') + else: + invocation_metadata = self._invocation_metadata + self._invocation_metadata = None + return invocation_metadata + + def take_invocation_metadata_and_request(self): + with self._condition: + if self._invocation_metadata is None: + raise ValueError('Expected invocation metadata!') + elif not self._requests: + raise ValueError('Expected at least one request!') + else: + invocation_metadata = self._invocation_metadata + self._invocation_metadata = None + return invocation_metadata, self._requests.pop(0) + + def send_initial_metadata(self, initial_metadata): + with self._condition: + self._initial_metadata = _common.fuss_with_metadata( + initial_metadata) + self._condition.notify_all() + + def take_request(self): + with self._condition: + while True: + if self._requests: + return self._requests.pop(0) + else: + self._condition.wait() + + def requests_closed(self): + with self._condition: + while True: + if self._requests_closed: + return + else: + self._condition.wait() + + def send_response(self, response): + with self._condition: + if self._code is None: + self._responses.append(response) + self._condition.notify_all() + + def terminate_with_response( + self, response, trailing_metadata, code, details): + with self._condition: + if self._initial_metadata is None: + self._initial_metadata = _common.FUSSED_EMPTY_METADATA + self._responses.append(response) + self._trailing_metadata = _common.fuss_with_metadata( + trailing_metadata) + self._code = code + self._details = details + self._condition.notify_all() + + def terminate(self, trailing_metadata, code, details): + with self._condition: + if self._initial_metadata is None: + self._initial_metadata = _common.FUSSED_EMPTY_METADATA + self._trailing_metadata = _common.fuss_with_metadata( + trailing_metadata) + self._code = code + self._details = details + self._condition.notify_all() + + def cancelled(self): + with self._condition: + while True: + if self._code is grpc.StatusCode.CANCELLED: + return + elif self._code is None: + self._condition.wait() + else: + raise ValueError( + 'Status code unexpectedly {}!'.format(self._code)) + + def is_active(self): + raise NotImplementedError() + + def time_remaining(self): + raise NotImplementedError() + + def add_callback(self, callback): + raise NotImplementedError() diff --git a/src/python/grpcio_testing/grpc_testing/_common.py b/src/python/grpcio_testing/grpc_testing/_common.py new file mode 100644 index 00000000000..cb4a7f5fa2f --- /dev/null +++ b/src/python/grpcio_testing/grpc_testing/_common.py @@ -0,0 +1,92 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common interfaces and implementation.""" + +import abc +import collections + +import six + + +def _fuss(tuplified_metadata): + return tuplified_metadata + ( + ( + 'grpc.metadata_added_by_runtime', + 'gRPC is allowed to add metadata in transmission and does so.', + ), + ) + +FUSSED_EMPTY_METADATA = _fuss(()) + + +def fuss_with_metadata(metadata): + if metadata is None: + return FUSSED_EMPTY_METADATA + else: + return _fuss(tuple(metadata)) + + +class ChannelRpcRead( + collections.namedtuple( + 'ChannelRpcRead', + ('response', 'trailing_metadata', 'code', 'details',))): + pass + + +class ChannelRpcHandler(six.with_metaclass(abc.ABCMeta)): + + @abc.abstractmethod + def initial_metadata(self): + raise NotImplementedError() + + @abc.abstractmethod + def add_request(self, request): + raise NotImplementedError() + + @abc.abstractmethod + def close_requests(self): + raise NotImplementedError() + + @abc.abstractmethod + def take_response(self): + raise NotImplementedError() + + @abc.abstractmethod + def cancel(self, code, details): + raise NotImplementedError() + + @abc.abstractmethod + def termination(self): + raise NotImplementedError() + + @abc.abstractmethod + def is_active(self): + raise NotImplementedError() + + @abc.abstractmethod + def time_remaining(self): + raise NotImplementedError() + + @abc.abstractmethod + def add_callback(self, callback): + raise NotImplementedError() + + +class ChannelHandler(six.with_metaclass(abc.ABCMeta)): + + @abc.abstractmethod + def invoke_rpc( + self, method_full_rpc_name, invocation_metadata, requests, + requests_closed, timeout): + raise NotImplementedError() diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index adc909ccdc7..debe14c40e5 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -68,6 +68,10 @@ PACKAGE_DATA = { 'tests.protoc_plugin.protos.invocation_testing.split_services': [ 'services.proto', ], + 'tests.testing.proto': [ + 'requests.proto', + 'services.proto', + ], 'tests.unit': [ 'credentials/ca.pem', 'credentials/server1.key', diff --git a/src/python/grpcio_tests/tests/testing/_application_common.py b/src/python/grpcio_tests/tests/testing/_application_common.py new file mode 100644 index 00000000000..4e98879607a --- /dev/null +++ b/src/python/grpcio_tests/tests/testing/_application_common.py @@ -0,0 +1,36 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""An example gRPC Python-using application's common code elements.""" + +from tests.testing.proto import requests_pb2 +from tests.testing.proto import services_pb2 + +SERVICE_NAME = 'tests_of_grpc_testing.FirstService' +UNARY_UNARY_METHOD_NAME = 'UnUn' +UNARY_STREAM_METHOD_NAME = 'UnStre' +STREAM_UNARY_METHOD_NAME = 'StreUn' +STREAM_STREAM_METHOD_NAME = 'StreStre' + +UNARY_UNARY_REQUEST = requests_pb2.Up(first_up_field=2) +ERRONEOUS_UNARY_UNARY_REQUEST = requests_pb2.Up(first_up_field=3) +UNARY_UNARY_RESPONSE = services_pb2.Down(first_down_field=5) +ERRONEOUS_UNARY_UNARY_RESPONSE = services_pb2.Down(first_down_field=7) +UNARY_STREAM_REQUEST = requests_pb2.Charm(first_charm_field=11) +STREAM_UNARY_REQUEST = requests_pb2.Charm(first_charm_field=13) +STREAM_UNARY_RESPONSE = services_pb2.Strange(first_strange_field=17) +STREAM_STREAM_REQUEST = requests_pb2.Top(first_top_field=19) +STREAM_STREAM_RESPONSE = services_pb2.Bottom(first_bottom_field=23) +TWO_STREAM_STREAM_RESPONSES = (STREAM_STREAM_RESPONSE,) * 2 + +INFINITE_REQUEST_STREAM_TIMEOUT = 0.2 diff --git a/src/python/grpcio_tests/tests/testing/_application_testing_common.py b/src/python/grpcio_tests/tests/testing/_application_testing_common.py new file mode 100644 index 00000000000..9c9e485a783 --- /dev/null +++ b/src/python/grpcio_tests/tests/testing/_application_testing_common.py @@ -0,0 +1,33 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc_testing + +from tests.testing.proto import requests_pb2 +from tests.testing.proto import services_pb2 + +# TODO(https://github.com/grpc/grpc/issues/11657): Eliminate this entirely. +# TODO(https://github.com/google/protobuf/issues/3452): Eliminate this if/else. +if services_pb2.DESCRIPTOR.services_by_name.get('FirstService') is None: + FIRST_SERVICE = 'Fix protobuf issue 3452!' + FIRST_SERVICE_UNUN = 'Fix protobuf issue 3452!' + FIRST_SERVICE_UNSTRE = 'Fix protobuf issue 3452!' + FIRST_SERVICE_STREUN = 'Fix protobuf issue 3452!' + FIRST_SERVICE_STRESTRE = 'Fix protobuf issue 3452!' +else: + FIRST_SERVICE = services_pb2.DESCRIPTOR.services_by_name['FirstService'] + FIRST_SERVICE_UNUN = FIRST_SERVICE.methods_by_name['UnUn'] + FIRST_SERVICE_UNSTRE = FIRST_SERVICE.methods_by_name['UnStre'] + FIRST_SERVICE_STREUN = FIRST_SERVICE.methods_by_name['StreUn'] + FIRST_SERVICE_STRESTRE = FIRST_SERVICE.methods_by_name['StreStre'] diff --git a/src/python/grpcio_tests/tests/testing/_client_application.py b/src/python/grpcio_tests/tests/testing/_client_application.py new file mode 100644 index 00000000000..aff32fb4dc5 --- /dev/null +++ b/src/python/grpcio_tests/tests/testing/_client_application.py @@ -0,0 +1,260 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""An example gRPC Python-using client-side application.""" + +import collections +import enum +import threading +import time + +import grpc +from tests.unit.framework.common import test_constants + +from tests.testing.proto import requests_pb2 +from tests.testing.proto import services_pb2 +from tests.testing.proto import services_pb2_grpc + +from tests.testing import _application_common + + +@enum.unique +class Scenario(enum.Enum): + UNARY_UNARY = 'unary unary' + UNARY_STREAM = 'unary stream' + STREAM_UNARY = 'stream unary' + STREAM_STREAM = 'stream stream' + CONCURRENT_STREAM_UNARY = 'concurrent stream unary' + CONCURRENT_STREAM_STREAM = 'concurrent stream stream' + CANCEL_UNARY_UNARY = 'cancel unary unary' + CANCEL_UNARY_STREAM = 'cancel unary stream' + INFINITE_REQUEST_STREAM = 'infinite request stream' + + +class Outcome(collections.namedtuple('Outcome', ('kind', 'code', 'details'))): + """Outcome of a client application scenario. + + Attributes: + kind: A Kind value describing the overall kind of scenario execution. + code: A grpc.StatusCode value. Only valid if kind is Kind.RPC_ERROR. + details: A status details string. Only valid if kind is Kind.RPC_ERROR. + """ + + @enum.unique + class Kind(enum.Enum): + SATISFACTORY = 'satisfactory' + UNSATISFACTORY = 'unsatisfactory' + RPC_ERROR = 'rpc error' + + +_SATISFACTORY_OUTCOME = Outcome(Outcome.Kind.SATISFACTORY, None, None) +_UNSATISFACTORY_OUTCOME = Outcome(Outcome.Kind.UNSATISFACTORY, None, None) + + +class _Pipe(object): + + def __init__(self): + self._condition = threading.Condition() + self._values = [] + self._open = True + + def __iter__(self): + return self + + def _next(self): + with self._condition: + while True: + if self._values: + return self._values.pop(0) + elif not self._open: + raise StopIteration() + else: + self._condition.wait() + + def __next__(self): # (Python 3 Iterator Protocol) + return self._next() + + def next(self): # (Python 2 Iterator Protocol) + return self._next() + + def add(self, value): + with self._condition: + self._values.append(value) + self._condition.notify_all() + + def close(self): + with self._condition: + self._open = False + self._condition.notify_all() + + +def _run_unary_unary(stub): + response = stub.UnUn(_application_common.UNARY_UNARY_REQUEST) + if _application_common.UNARY_UNARY_RESPONSE == response: + return _SATISFACTORY_OUTCOME + else: + return _UNSATISFACTORY_OUTCOME + + +def _run_unary_stream(stub): + response_iterator = stub.UnStre(_application_common.UNARY_STREAM_REQUEST) + try: + next(response_iterator) + except StopIteration: + return _SATISFACTORY_OUTCOME + else: + return _UNSATISFACTORY_OUTCOME + + +def _run_stream_unary(stub): + response, call = stub.StreUn.with_call( + iter((_application_common.STREAM_UNARY_REQUEST,) * 3)) + if (_application_common.STREAM_UNARY_RESPONSE == response and + call.code() is grpc.StatusCode.OK): + return _SATISFACTORY_OUTCOME + else: + return _UNSATISFACTORY_OUTCOME + + +def _run_stream_stream(stub): + request_pipe = _Pipe() + response_iterator = stub.StreStre(iter(request_pipe)) + request_pipe.add(_application_common.STREAM_STREAM_REQUEST) + first_responses = next(response_iterator), next(response_iterator), + request_pipe.add(_application_common.STREAM_STREAM_REQUEST) + second_responses = next(response_iterator), next(response_iterator), + request_pipe.close() + try: + next(response_iterator) + except StopIteration: + unexpected_extra_response = False + else: + unexpected_extra_response = True + if (first_responses == _application_common.TWO_STREAM_STREAM_RESPONSES and + second_responses == _application_common.TWO_STREAM_STREAM_RESPONSES + and not unexpected_extra_response): + return _SATISFACTORY_OUTCOME + else: + return _UNSATISFACTORY_OUTCOME + + +def _run_concurrent_stream_unary(stub): + future_calls = tuple( + stub.StreUn.future( + iter((_application_common.STREAM_UNARY_REQUEST,) * 3)) + for _ in range(test_constants.THREAD_CONCURRENCY)) + for future_call in future_calls: + if future_call.code() is grpc.StatusCode.OK: + response = future_call.result() + if _application_common.STREAM_UNARY_RESPONSE != response: + return _UNSATISFACTORY_OUTCOME + else: + return _UNSATISFACTORY_OUTCOME + else: + return _SATISFACTORY_OUTCOME + + +def _run_concurrent_stream_stream(stub): + condition = threading.Condition() + outcomes = [None] * test_constants.RPC_CONCURRENCY + + def run_stream_stream(index): + outcome = _run_stream_stream(stub) + with condition: + outcomes[index] = outcome + condition.notify() + + for index in range(test_constants.RPC_CONCURRENCY): + thread = threading.Thread(target=run_stream_stream, args=(index,)) + thread.start() + with condition: + while True: + if all(outcomes): + for outcome in outcomes: + if outcome.kind is not Outcome.Kind.SATISFACTORY: + return _UNSATISFACTORY_OUTCOME + else: + return _SATISFACTORY_OUTCOME + else: + condition.wait() + + +def _run_cancel_unary_unary(stub): + response_future_call = stub.UnUn.future( + _application_common.UNARY_UNARY_REQUEST) + initial_metadata = response_future_call.initial_metadata() + cancelled = response_future_call.cancel() + if initial_metadata is not None and cancelled: + return _SATISFACTORY_OUTCOME + else: + return _UNSATISFACTORY_OUTCOME + + +def _run_infinite_request_stream(stub): + + def infinite_request_iterator(): + while True: + yield _application_common.STREAM_UNARY_REQUEST + + response_future_call = stub.StreUn.future( + infinite_request_iterator(), + timeout=_application_common.INFINITE_REQUEST_STREAM_TIMEOUT) + if response_future_call.code() is grpc.StatusCode.DEADLINE_EXCEEDED: + return _SATISFACTORY_OUTCOME + else: + return _UNSATISFACTORY_OUTCOME + + +def run(scenario, channel): + stub = services_pb2_grpc.FirstServiceStub(channel) + try: + if scenario is Scenario.UNARY_UNARY: + return _run_unary_unary(stub) + elif scenario is Scenario.UNARY_STREAM: + return _run_unary_stream(stub) + elif scenario is Scenario.STREAM_UNARY: + return _run_stream_unary(stub) + elif scenario is Scenario.STREAM_STREAM: + return _run_stream_stream(stub) + elif scenario is Scenario.CONCURRENT_STREAM_UNARY: + return _run_concurrent_stream_unary(stub) + elif scenario is Scenario.CONCURRENT_STREAM_STREAM: + return _run_concurrent_stream_stream(stub) + elif scenario is Scenario.CANCEL_UNARY_UNARY: + return _run_cancel_unary_unary(stub) + elif scenario is Scenario.INFINITE_REQUEST_STREAM: + return _run_infinite_request_stream(stub) + except grpc.RpcError as rpc_error: + return Outcome(Outcome.Kind.RPC_ERROR, + rpc_error.code(), rpc_error.details()) + + +_IMPLEMENTATIONS = { + Scenario.UNARY_UNARY: _run_unary_unary, + Scenario.UNARY_STREAM: _run_unary_stream, + Scenario.STREAM_UNARY: _run_stream_unary, + Scenario.STREAM_STREAM: _run_stream_stream, + Scenario.CONCURRENT_STREAM_UNARY: _run_concurrent_stream_unary, + Scenario.CONCURRENT_STREAM_STREAM: _run_concurrent_stream_stream, + Scenario.CANCEL_UNARY_UNARY: _run_cancel_unary_unary, + Scenario.INFINITE_REQUEST_STREAM: _run_infinite_request_stream, +} + + +def run(scenario, channel): + stub = services_pb2_grpc.FirstServiceStub(channel) + try: + return _IMPLEMENTATIONS[scenario](stub) + except grpc.RpcError as rpc_error: + return Outcome(Outcome.Kind.RPC_ERROR, + rpc_error.code(), rpc_error.details()) diff --git a/src/python/grpcio_tests/tests/testing/_client_test.py b/src/python/grpcio_tests/tests/testing/_client_test.py new file mode 100644 index 00000000000..172f386d7b1 --- /dev/null +++ b/src/python/grpcio_tests/tests/testing/_client_test.py @@ -0,0 +1,306 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from concurrent import futures +import time +import unittest + +import grpc +from grpc.framework.foundation import logging_pool +from tests.unit.framework.common import test_constants +import grpc_testing + +from tests.testing import _application_common +from tests.testing import _application_testing_common +from tests.testing import _client_application +from tests.testing.proto import requests_pb2 +from tests.testing.proto import services_pb2 + + +# TODO(https://github.com/google/protobuf/issues/3452): Drop this skip. +@unittest.skipIf( + services_pb2.DESCRIPTOR.services_by_name.get('FirstService') is None, + 'Fix protobuf issue 3452!') +class ClientTest(unittest.TestCase): + + def setUp(self): + # In this test the client-side application under test executes in + # a separate thread while we retain use of the test thread to "play + # server". + self._client_execution_thread_pool = logging_pool.pool(1) + + self._fake_time = grpc_testing.strict_fake_time(time.time()) + self._real_time = grpc_testing.strict_real_time() + self._fake_time_channel = grpc_testing.channel( + services_pb2.DESCRIPTOR.services_by_name.values(), self._fake_time) + self._real_time_channel = grpc_testing.channel( + services_pb2.DESCRIPTOR.services_by_name.values(), self._real_time) + + def tearDown(self): + self._client_execution_thread_pool.shutdown(wait=True) + + def test_successful_unary_unary(self): + application_future = self._client_execution_thread_pool.submit( + _client_application.run, _client_application.Scenario.UNARY_UNARY, + self._real_time_channel) + invocation_metadata, request, rpc = ( + self._real_time_channel.take_unary_unary( + _application_testing_common.FIRST_SERVICE_UNUN)) + rpc.send_initial_metadata(()) + rpc.terminate(_application_common.UNARY_UNARY_RESPONSE, (), + grpc.StatusCode.OK, '') + application_return_value = application_future.result() + + self.assertEqual(_application_common.UNARY_UNARY_REQUEST, request) + self.assertIs(application_return_value.kind, + _client_application.Outcome.Kind.SATISFACTORY) + + def test_successful_unary_stream(self): + application_future = self._client_execution_thread_pool.submit( + _client_application.run, _client_application.Scenario.UNARY_STREAM, + self._fake_time_channel) + invocation_metadata, request, rpc = ( + self._fake_time_channel.take_unary_stream( + _application_testing_common.FIRST_SERVICE_UNSTRE)) + rpc.send_initial_metadata(()) + rpc.terminate((), grpc.StatusCode.OK, '') + application_return_value = application_future.result() + + self.assertEqual(_application_common.UNARY_STREAM_REQUEST, request) + self.assertIs(application_return_value.kind, + _client_application.Outcome.Kind.SATISFACTORY) + + def test_successful_stream_unary(self): + application_future = self._client_execution_thread_pool.submit( + _client_application.run, _client_application.Scenario.STREAM_UNARY, + self._real_time_channel) + invocation_metadata, rpc = self._real_time_channel.take_stream_unary( + _application_testing_common.FIRST_SERVICE_STREUN) + rpc.send_initial_metadata(()) + first_request = rpc.take_request() + second_request = rpc.take_request() + third_request = rpc.take_request() + rpc.requests_closed() + rpc.terminate(_application_common.STREAM_UNARY_RESPONSE, (), + grpc.StatusCode.OK, '') + application_return_value = application_future.result() + + self.assertEqual(_application_common.STREAM_UNARY_REQUEST, + first_request) + self.assertEqual(_application_common.STREAM_UNARY_REQUEST, + second_request) + self.assertEqual(_application_common.STREAM_UNARY_REQUEST, + third_request) + self.assertIs(application_return_value.kind, + _client_application.Outcome.Kind.SATISFACTORY) + + def test_successful_stream_stream(self): + application_future = self._client_execution_thread_pool.submit( + _client_application.run, _client_application.Scenario.STREAM_STREAM, + self._fake_time_channel) + invocation_metadata, rpc = self._fake_time_channel.take_stream_stream( + _application_testing_common.FIRST_SERVICE_STRESTRE) + first_request = rpc.take_request() + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + second_request = rpc.take_request() + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.requests_closed() + rpc.terminate((), grpc.StatusCode.OK, '') + application_return_value = application_future.result() + + self.assertEqual(_application_common.STREAM_STREAM_REQUEST, + first_request) + self.assertEqual(_application_common.STREAM_STREAM_REQUEST, + second_request) + self.assertIs(application_return_value.kind, + _client_application.Outcome.Kind.SATISFACTORY) + + def test_concurrent_stream_stream(self): + application_future = self._client_execution_thread_pool.submit( + _client_application.run, + _client_application.Scenario.CONCURRENT_STREAM_STREAM, + self._real_time_channel) + rpcs = [] + for _ in range(test_constants.RPC_CONCURRENCY): + invocation_metadata, rpc = ( + self._real_time_channel.take_stream_stream( + _application_testing_common.FIRST_SERVICE_STRESTRE)) + rpcs.append(rpc) + requests = {} + for rpc in rpcs: + requests[rpc] = [rpc.take_request()] + for rpc in rpcs: + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + for rpc in rpcs: + requests[rpc].append(rpc.take_request()) + for rpc in rpcs: + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + for rpc in rpcs: + rpc.requests_closed() + for rpc in rpcs: + rpc.terminate((), grpc.StatusCode.OK, '') + application_return_value = application_future.result() + + for requests_of_one_rpc in requests.values(): + for request in requests_of_one_rpc: + self.assertEqual(_application_common.STREAM_STREAM_REQUEST, + request) + self.assertIs(application_return_value.kind, + _client_application.Outcome.Kind.SATISFACTORY) + + def test_cancelled_unary_unary(self): + application_future = self._client_execution_thread_pool.submit( + _client_application.run, + _client_application.Scenario.CANCEL_UNARY_UNARY, + self._fake_time_channel) + invocation_metadata, request, rpc = ( + self._fake_time_channel.take_unary_unary( + _application_testing_common.FIRST_SERVICE_UNUN)) + rpc.send_initial_metadata(()) + rpc.cancelled() + application_return_value = application_future.result() + + self.assertEqual(_application_common.UNARY_UNARY_REQUEST, request) + self.assertIs(application_return_value.kind, + _client_application.Outcome.Kind.SATISFACTORY) + + def test_status_stream_unary(self): + application_future = self._client_execution_thread_pool.submit( + _client_application.run, + _client_application.Scenario.CONCURRENT_STREAM_UNARY, + self._fake_time_channel) + rpcs = tuple( + self._fake_time_channel.take_stream_unary( + _application_testing_common.FIRST_SERVICE_STREUN)[1] + for _ in range(test_constants.THREAD_CONCURRENCY)) + for rpc in rpcs: + rpc.take_request() + rpc.take_request() + rpc.take_request() + rpc.requests_closed() + rpc.send_initial_metadata(( + ('my_metadata_key', 'My Metadata Value!',),)) + for rpc in rpcs[:-1]: + rpc.terminate(_application_common.STREAM_UNARY_RESPONSE, (), + grpc.StatusCode.OK, '') + rpcs[-1].terminate(_application_common.STREAM_UNARY_RESPONSE, (), + grpc.StatusCode.RESOURCE_EXHAUSTED, + 'nope; not able to handle all those RPCs!') + application_return_value = application_future.result() + + self.assertIs(application_return_value.kind, + _client_application.Outcome.Kind.UNSATISFACTORY) + + def test_status_stream_stream(self): + code = grpc.StatusCode.DEADLINE_EXCEEDED + details = 'test deadline exceeded!' + + application_future = self._client_execution_thread_pool.submit( + _client_application.run, _client_application.Scenario.STREAM_STREAM, + self._real_time_channel) + invocation_metadata, rpc = self._real_time_channel.take_stream_stream( + _application_testing_common.FIRST_SERVICE_STRESTRE) + first_request = rpc.take_request() + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + second_request = rpc.take_request() + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.requests_closed() + rpc.terminate((), code, details) + application_return_value = application_future.result() + + self.assertEqual(_application_common.STREAM_STREAM_REQUEST, + first_request) + self.assertEqual(_application_common.STREAM_STREAM_REQUEST, + second_request) + self.assertIs(application_return_value.kind, + _client_application.Outcome.Kind.RPC_ERROR) + self.assertIs(application_return_value.code, code) + self.assertEqual(application_return_value.details, details) + + def test_misbehaving_server_unary_unary(self): + application_future = self._client_execution_thread_pool.submit( + _client_application.run, _client_application.Scenario.UNARY_UNARY, + self._fake_time_channel) + invocation_metadata, request, rpc = ( + self._fake_time_channel.take_unary_unary( + _application_testing_common.FIRST_SERVICE_UNUN)) + rpc.send_initial_metadata(()) + rpc.terminate(_application_common.ERRONEOUS_UNARY_UNARY_RESPONSE, (), + grpc.StatusCode.OK, '') + application_return_value = application_future.result() + + self.assertEqual(_application_common.UNARY_UNARY_REQUEST, request) + self.assertIs(application_return_value.kind, + _client_application.Outcome.Kind.UNSATISFACTORY) + + def test_misbehaving_server_stream_stream(self): + application_future = self._client_execution_thread_pool.submit( + _client_application.run, _client_application.Scenario.STREAM_STREAM, + self._real_time_channel) + invocation_metadata, rpc = self._real_time_channel.take_stream_stream( + _application_testing_common.FIRST_SERVICE_STRESTRE) + first_request = rpc.take_request() + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + second_request = rpc.take_request() + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.send_response(_application_common.STREAM_STREAM_RESPONSE) + rpc.requests_closed() + rpc.terminate((), grpc.StatusCode.OK, '') + application_return_value = application_future.result() + + self.assertEqual(_application_common.STREAM_STREAM_REQUEST, + first_request) + self.assertEqual(_application_common.STREAM_STREAM_REQUEST, + second_request) + self.assertIs(application_return_value.kind, + _client_application.Outcome.Kind.UNSATISFACTORY) + + def test_infinite_request_stream_real_time(self): + application_future = self._client_execution_thread_pool.submit( + _client_application.run, + _client_application.Scenario.INFINITE_REQUEST_STREAM, + self._real_time_channel) + invocation_metadata, rpc = self._real_time_channel.take_stream_unary( + _application_testing_common.FIRST_SERVICE_STREUN) + rpc.send_initial_metadata(()) + first_request = rpc.take_request() + second_request = rpc.take_request() + third_request = rpc.take_request() + self._real_time.sleep_for( + _application_common.INFINITE_REQUEST_STREAM_TIMEOUT) + rpc.terminate(_application_common.STREAM_UNARY_RESPONSE, (), + grpc.StatusCode.DEADLINE_EXCEEDED, '') + application_return_value = application_future.result() + + self.assertEqual(_application_common.STREAM_UNARY_REQUEST, + first_request) + self.assertEqual(_application_common.STREAM_UNARY_REQUEST, + second_request) + self.assertEqual(_application_common.STREAM_UNARY_REQUEST, + third_request) + self.assertIs(application_return_value.kind, + _client_application.Outcome.Kind.SATISFACTORY) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/testing/proto/__init__.py b/src/python/grpcio_tests/tests/testing/proto/__init__.py new file mode 100644 index 00000000000..1e120359cf9 --- /dev/null +++ b/src/python/grpcio_tests/tests/testing/proto/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/python/grpcio_tests/tests/testing/proto/requests.proto b/src/python/grpcio_tests/tests/testing/proto/requests.proto new file mode 100644 index 00000000000..54a60bff869 --- /dev/null +++ b/src/python/grpcio_tests/tests/testing/proto/requests.proto @@ -0,0 +1,29 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package tests_of_grpc_testing; + +message Up { + int32 first_up_field = 1; +} + +message Charm { + int32 first_charm_field = 1; +} + +message Top { + int32 first_top_field = 1; +} diff --git a/src/python/grpcio_tests/tests/testing/proto/services.proto b/src/python/grpcio_tests/tests/testing/proto/services.proto new file mode 100644 index 00000000000..cb15c0d1ce3 --- /dev/null +++ b/src/python/grpcio_tests/tests/testing/proto/services.proto @@ -0,0 +1,42 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +import "tests/testing/proto/requests.proto"; + +package tests_of_grpc_testing; + +message Down { + int32 first_down_field = 1; +} + +message Strange { + int32 first_strange_field = 1; +} + +message Bottom { + int32 first_bottom_field = 1; +} + +service FirstService { + rpc UnUn(Up) returns (Down); + rpc UnStre(Charm) returns (stream Strange); + rpc StreUn(stream Charm) returns (Strange); + rpc StreStre(stream Top) returns (stream Bottom); +} + +service SecondService { + rpc UnStre(Strange) returns (stream Charm); +} diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index f86eeb76c70..c10719b86f1 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -9,6 +9,7 @@ "protoc_plugin._split_definitions_test.SplitSeparateTest", "protoc_plugin.beta_python_plugin_test.PythonPluginTest", "reflection._reflection_servicer_test.ReflectionServicerTest", + "testing._client_test.ClientTest", "testing._time_test.StrictFakeTimeTest", "testing._time_test.StrictRealTimeTest", "unit._api_test.AllTest", diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c index b99954883f3..74f189e1e0d 100644 --- a/src/ruby/ext/grpc/rb_call.c +++ b/src/ruby/ext/grpc/rb_call.c @@ -179,6 +179,38 @@ static VALUE grpc_rb_call_cancel(VALUE self) { return Qnil; } +/* TODO: expose this as part of the surface API if needed. + * This is meant for internal usage by the "write thread" of grpc-ruby + * client-side bidi calls. It provides a way for the background write-thread + * to propogate failures to the main read-thread and give the user an error + * message. */ +static VALUE grpc_rb_call_cancel_with_status(VALUE self, VALUE status_code, + VALUE details) { + grpc_rb_call *call = NULL; + grpc_call_error err; + if (RTYPEDDATA_DATA(self) == NULL) { + // This call has been closed + return Qnil; + } + + if (TYPE(details) != T_STRING || TYPE(status_code) != T_FIXNUM) { + rb_raise(rb_eTypeError, + "Bad parameter type error for cancel with status. Want Fixnum, " + "String."); + return Qnil; + } + + TypedData_Get_Struct(self, grpc_rb_call, &grpc_call_data_type, call); + err = grpc_call_cancel_with_status(call->wrapped, NUM2LONG(status_code), + StringValueCStr(details), NULL); + if (err != GRPC_CALL_OK) { + rb_raise(grpc_rb_eCallError, "cancel with status failed: %s (code=%d)", + grpc_call_error_detail_of(err), err); + } + + return Qnil; +} + /* Releases the c-level resources associated with a call Once a call has been closed, no further requests can be processed. @@ -949,6 +981,8 @@ void Init_grpc_call() { /* Add ruby analogues of the Call methods. */ rb_define_method(grpc_rb_cCall, "run_batch", grpc_rb_call_run_batch, 1); rb_define_method(grpc_rb_cCall, "cancel", grpc_rb_call_cancel, 0); + rb_define_method(grpc_rb_cCall, "cancel_with_status", + grpc_rb_call_cancel_with_status, 2); rb_define_method(grpc_rb_cCall, "close", grpc_rb_call_close, 0); rb_define_method(grpc_rb_cCall, "peer", grpc_rb_call_get_peer, 0); rb_define_method(grpc_rb_cCall, "peer_cert", grpc_rb_call_get_peer_cert, 0); diff --git a/src/ruby/lib/grpc/generic/bidi_call.rb b/src/ruby/lib/grpc/generic/bidi_call.rb index 9e125cd986b..c2239d0178e 100644 --- a/src/ruby/lib/grpc/generic/bidi_call.rb +++ b/src/ruby/lib/grpc/generic/bidi_call.rb @@ -153,7 +153,12 @@ module GRPC rescue StandardError => e GRPC.logger.warn('bidi-write-loop: failed') GRPC.logger.warn(e) - raise e + if is_client + @call.cancel_with_status(GRPC::Core::StatusCodes::UNKNOWN, + "GRPC bidi call error: #{e.inspect}") + else + raise e + end ensure set_output_stream_done.call if is_client end @@ -180,8 +185,8 @@ module GRPC batch_result = @call.run_batch(RECV_STATUS_ON_CLIENT => nil) @call.status = batch_result.status @call.trailing_metadata = @call.status.metadata if @call.status - batch_result.check_status GRPC.logger.debug("bidi-read-loop: done status #{@call.status}") + batch_result.check_status end GRPC.logger.debug('bidi-read-loop: done reading!') diff --git a/src/ruby/spec/call_spec.rb b/src/ruby/spec/call_spec.rb index 473ff4a8bd1..1cc05002429 100644 --- a/src/ruby/spec/call_spec.rb +++ b/src/ruby/spec/call_spec.rb @@ -137,6 +137,39 @@ describe GRPC::Core::Call do end end + describe '#cancel' do + it 'completes ok' do + call = make_test_call + expect { call.cancel }.not_to raise_error + end + + it 'completes ok when the call is closed' do + call = make_test_call + call.close + expect { call.cancel }.not_to raise_error + end + end + + describe '#cancel_with_status' do + it 'completes ok' do + call = make_test_call + expect do + call.cancel_with_status(0, 'test status') + end.not_to raise_error + expect do + call.cancel_with_status(0, nil) + end.to raise_error(TypeError) + end + + it 'completes ok when the call is closed' do + call = make_test_call + call.close + expect do + call.cancel_with_status(0, 'test status') + end.not_to raise_error + end + end + def make_test_call @ch.create_call(nil, nil, 'dummy_method', nil, deadline) end diff --git a/src/ruby/spec/client_server_spec.rb b/src/ruby/spec/client_server_spec.rb index b48b4179ced..1a9b47e2c3b 100644 --- a/src/ruby/spec/client_server_spec.rb +++ b/src/ruby/spec/client_server_spec.rb @@ -226,6 +226,62 @@ shared_examples 'basic GRPC message delivery is OK' do svr_batch = server_call.run_batch(server_ops) expect(svr_batch.send_close).to be true end + + def client_cancel_test(cancel_proc, expected_code, + expected_details) + call = new_client_call + server_call = nil + + server_thread = Thread.new do + server_call = server_allows_client_to_proceed + end + + client_ops = { + CallOps::SEND_INITIAL_METADATA => {}, + CallOps::RECV_INITIAL_METADATA => nil + } + batch_result = call.run_batch(client_ops) + expect(batch_result.send_metadata).to be true + expect(batch_result.metadata).to eq({}) + + cancel_proc.call(call) + + server_thread.join + server_ops = { + CallOps::RECV_CLOSE_ON_SERVER => nil + } + svr_batch = server_call.run_batch(server_ops) + expect(svr_batch.send_close).to be true + + client_ops = { + CallOps::RECV_STATUS_ON_CLIENT => {} + } + batch_result = call.run_batch(client_ops) + + expect(batch_result.status.code).to be expected_code + expect(batch_result.status.details).to eq expected_details + end + + it 'clients can cancel a call on the server' do + expected_code = StatusCodes::CANCELLED + expected_details = 'Cancelled' + cancel_proc = proc { |call| call.cancel } + client_cancel_test(cancel_proc, expected_code, expected_details) + end + + it 'cancel_with_status unknown status' do + code = StatusCodes::UNKNOWN + details = 'test unknown reason' + cancel_proc = proc { |call| call.cancel_with_status(code, details) } + client_cancel_test(cancel_proc, code, details) + end + + it 'cancel_with_status unknown status' do + code = StatusCodes::FAILED_PRECONDITION + details = 'test failed precondition reason' + cancel_proc = proc { |call| call.cancel_with_status(code, details) } + client_cancel_test(cancel_proc, code, details) + end end shared_examples 'GRPC metadata delivery works OK' do diff --git a/src/ruby/spec/generic/client_stub_spec.rb b/src/ruby/spec/generic/client_stub_spec.rb index e1e7a535fb6..9539e56c0ff 100644 --- a/src/ruby/spec/generic/client_stub_spec.rb +++ b/src/ruby/spec/generic/client_stub_spec.rb @@ -472,7 +472,7 @@ describe 'ClientStub' do host = "localhost:#{server_port}" stub = GRPC::ClientStub.new(host, :this_channel_is_insecure) expect do - get_responses(stub) + get_responses(stub).collect { |r| r } end.to raise_error(ArgumentError, /Header values must be of type string or array/) end @@ -641,11 +641,101 @@ describe 'ClientStub' do expect(e.collect { |r| r }).to eq(@sent_msgs) th.join end + + # Prompted by grpc/github #10526 + describe 'surfacing of errors when sending requests' do + def run_server_bidi_send_one_then_read_indefinitely + @server.start + recvd_rpc = @server.request_call + recvd_call = recvd_rpc.call + server_call = GRPC::ActiveCall.new( + recvd_call, noop, noop, INFINITE_FUTURE, + metadata_received: true, started: false) + server_call.send_initial_metadata + server_call.remote_send('server response') + loop do + m = server_call.remote_read + break if m.nil? + end + # can't fail since initial metadata already sent + server_call.send_status(@pass, 'OK', true) + end + + def verify_error_from_write_thread(stub, requests_to_push, + request_queue, expected_description) + # TODO: an improvement might be to raise the original exception from + # bidi call write loops instead of only cancelling the call + failing_marshal_proc = proc do |req| + fail req if req.is_a?(StandardError) + req + end + begin + e = get_responses(stub, marshal_proc: failing_marshal_proc) + first_response = e.next + expect(first_response).to eq('server response') + requests_to_push.each { |req| request_queue.push(req) } + e.collect { |r| r } + rescue GRPC::Unknown => e + exception = e + end + expect(exception.message.include?(expected_description)).to be(true) + end + + # Provides an Enumerable view of a Queue + class BidiErrorTestingEnumerateForeverQueue + def initialize(queue) + @queue = queue + end + + def each + loop do + msg = @queue.pop + yield msg + end + end + end + + def run_error_in_client_request_stream_test(requests_to_push, + expected_error_message) + # start a server that waits on a read indefinitely - it should + # see a cancellation and be able to break out + th = Thread.new { run_server_bidi_send_one_then_read_indefinitely } + stub = GRPC::ClientStub.new(@host, :this_channel_is_insecure) + + request_queue = Queue.new + @sent_msgs = BidiErrorTestingEnumerateForeverQueue.new(request_queue) + + verify_error_from_write_thread(stub, + requests_to_push, + request_queue, + expected_error_message) + # the write loop errror should cancel the call and end the + # server's request stream + th.join + end + + it 'non-GRPC errors from the write loop surface when raised ' \ + 'at the start of a request stream' do + expected_error_message = 'expect error on first request' + requests_to_push = [StandardError.new(expected_error_message)] + run_error_in_client_request_stream_test(requests_to_push, + expected_error_message) + end + + it 'non-GRPC errors from the write loop surface when raised ' \ + 'during the middle of a request stream' do + expected_error_message = 'expect error on last request' + requests_to_push = %w( one two ) + requests_to_push << StandardError.new(expected_error_message) + run_error_in_client_request_stream_test(requests_to_push, + expected_error_message) + end + end end describe 'without a call operation' do - def get_responses(stub, deadline: nil) - e = stub.bidi_streamer(@method, @sent_msgs, noop, noop, + def get_responses(stub, deadline: nil, marshal_proc: noop) + e = stub.bidi_streamer(@method, @sent_msgs, marshal_proc, noop, metadata: @metadata, deadline: deadline) expect(e).to be_a(Enumerator) e @@ -658,8 +748,9 @@ describe 'ClientStub' do after(:each) do @op.wait # make sure wait doesn't hang end - def get_responses(stub, run_start_call_first: false, deadline: nil) - @op = stub.bidi_streamer(@method, @sent_msgs, noop, noop, + def get_responses(stub, run_start_call_first: false, deadline: nil, + marshal_proc: noop) + @op = stub.bidi_streamer(@method, @sent_msgs, marshal_proc, noop, return_op: true, metadata: @metadata, deadline: deadline) expect(@op).to be_a(GRPC::ActiveCall::Operation) diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index e4fe594e220..b887eaaf4e9 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -178,6 +178,18 @@ end CheckCallAfterFinishedServiceStub = CheckCallAfterFinishedService.rpc_stub_class +# A service with a bidi streaming method. +class BidiService + include GRPC::GenericService + rpc :server_sends_bad_input, stream(EchoMsg), stream(EchoMsg) + + def server_sends_bad_input(_, _) + 'bad response. (not an enumerable, client sees an error)' + end +end + +BidiStub = BidiService.rpc_stub_class + describe GRPC::RpcServer do RpcServer = GRPC::RpcServer StatusCodes = GRPC::Core::StatusCodes @@ -520,6 +532,29 @@ describe GRPC::RpcServer do t.join expect(one_failed_as_unavailable).to be(true) end + + it 'should send a status UNKNOWN with a relevant message when the' \ + 'servers response stream is not an enumerable' do + @srv.handle(BidiService) + t = Thread.new { @srv.run } + @srv.wait_till_running + stub = BidiStub.new(@host, :this_channel_is_insecure, **client_opts) + responses = stub.server_sends_bad_input([]) + exception = nil + begin + responses.each { |r| r } + rescue GRPC::Unknown => e + exception = e + end + # Erroneous responses sent from the server handler should cause an + # exception on the client with relevant info. + expected_details = 'NoMethodError: undefined method `each\' for '\ + '"bad response. (not an enumerable, client sees an error)"' + + expect(exception.inspect.include?(expected_details)).to be true + @srv.stop + t.join + end end context 'with connect metadata' do diff --git a/test/core/client_channel/resolvers/dns_resolver_connectivity_test.c b/test/core/client_channel/resolvers/dns_resolver_connectivity_test.c index 6e3d69c2653..364e1809633 100644 --- a/test/core/client_channel/resolvers/dns_resolver_connectivity_test.c +++ b/test/core/client_channel/resolvers/dns_resolver_connectivity_test.c @@ -60,7 +60,8 @@ static void my_resolve_address(grpc_exec_ctx *exec_ctx, const char *addr, static grpc_ares_request *my_dns_lookup_ares( grpc_exec_ctx *exec_ctx, const char *dns_server, const char *addr, const char *default_port, grpc_pollset_set *interested_parties, - grpc_closure *on_done, grpc_lb_addresses **lb_addrs, bool check_grpclb) { + grpc_closure *on_done, grpc_lb_addresses **lb_addrs, bool check_grpclb, + char **service_config_json) { gpr_mu_lock(&g_mu); GPR_ASSERT(0 == strcmp("test", addr)); grpc_error *error = GRPC_ERROR_NONE; diff --git a/test/core/end2end/fuzzers/api_fuzzer.c b/test/core/end2end/fuzzers/api_fuzzer.c index 01fa4f748f5..1228c9fe9ae 100644 --- a/test/core/end2end/fuzzers/api_fuzzer.c +++ b/test/core/end2end/fuzzers/api_fuzzer.c @@ -416,7 +416,8 @@ void my_resolve_address(grpc_exec_ctx *exec_ctx, const char *addr, grpc_ares_request *my_dns_lookup_ares( grpc_exec_ctx *exec_ctx, const char *dns_server, const char *addr, const char *default_port, grpc_pollset_set *interested_parties, - grpc_closure *on_done, grpc_lb_addresses **lb_addrs, bool check_grpclb) { + grpc_closure *on_done, grpc_lb_addresses **lb_addrs, bool check_grpclb, + char **service_config_json) { addr_req *r = gpr_malloc(sizeof(*r)); r->addr = gpr_strdup(addr); r->on_done = on_done; diff --git a/test/core/end2end/goaway_server_test.c b/test/core/end2end/goaway_server_test.c index bf90e2525d4..c3aca132495 100644 --- a/test/core/end2end/goaway_server_test.c +++ b/test/core/end2end/goaway_server_test.c @@ -48,7 +48,8 @@ static void (*iomgr_resolve_address)(grpc_exec_ctx *exec_ctx, const char *addr, static grpc_ares_request *(*iomgr_dns_lookup_ares)( grpc_exec_ctx *exec_ctx, const char *dns_server, const char *addr, const char *default_port, grpc_pollset_set *interested_parties, - grpc_closure *on_done, grpc_lb_addresses **addresses, bool check_grpclb); + grpc_closure *on_done, grpc_lb_addresses **addresses, bool check_grpclb, + char **service_config_json); static void set_resolve_port(int port) { gpr_mu_lock(&g_mu); @@ -90,11 +91,12 @@ static void my_resolve_address(grpc_exec_ctx *exec_ctx, const char *addr, static grpc_ares_request *my_dns_lookup_ares( grpc_exec_ctx *exec_ctx, const char *dns_server, const char *addr, const char *default_port, grpc_pollset_set *interested_parties, - grpc_closure *on_done, grpc_lb_addresses **lb_addrs, bool check_grpclb) { + grpc_closure *on_done, grpc_lb_addresses **lb_addrs, bool check_grpclb, + char **service_config_json) { if (0 != strcmp(addr, "test")) { return iomgr_dns_lookup_ares(exec_ctx, dns_server, addr, default_port, interested_parties, on_done, lb_addrs, - check_grpclb); + check_grpclb, service_config_json); } grpc_error *error = GRPC_ERROR_NONE; diff --git a/test/core/surface/alarm_test.c b/test/core/surface/alarm_test.c index baaa05928e0..6971d920745 100644 --- a/test/core/surface/alarm_test.c +++ b/test/core/surface/alarm_test.c @@ -73,6 +73,21 @@ static void test_alarm(void) { GPR_ASSERT(ev.success == 0); grpc_alarm_destroy(alarm); } + { + /* alarm_destroy before cq_next */ + grpc_event ev; + void *tag = create_test_tag(); + grpc_alarm *alarm = + grpc_alarm_create(cc, grpc_timeout_seconds_to_deadline(2), tag); + + grpc_alarm_destroy(alarm); + ev = grpc_completion_queue_next(cc, grpc_timeout_seconds_to_deadline(1), + NULL); + GPR_ASSERT(ev.type == GRPC_OP_COMPLETE); + GPR_ASSERT(ev.tag == tag); + GPR_ASSERT(ev.success == 0); + } + shutdown_and_destroy(cc); } diff --git a/test/core/util/BUILD b/test/core/util/BUILD index 9d899bc7beb..1268c2cd103 100644 --- a/test/core/util/BUILD +++ b/test/core/util/BUILD @@ -38,7 +38,7 @@ grpc_cc_library( ) grpc_cc_library( - name = "grpc_test_util", + name = "grpc_test_util_base", srcs = [ "debugger_macros.c", "grpc_profiler.c", @@ -68,10 +68,32 @@ grpc_cc_library( language = "C", deps = [ ":gpr_test_util", + "//:grpc_common", + ], +) + +grpc_cc_library( + name = "grpc_test_util", + srcs = [], + hdrs = [], + language = "C", + deps = [ + ":grpc_test_util_base", "//:grpc", ], ) +grpc_cc_library( + name = "grpc_test_util_unsecure", + srcs = [], + hdrs = [], + language = "C", + deps = [ + ":grpc_test_util_base", + "//:grpc_unsecure", + ], +) + grpc_cc_library( name = "one_corpus_entry_fuzzer", srcs = ["one_corpus_entry_fuzzer.c"], diff --git a/test/core/util/port.c b/test/core/util/port.c index f430c543bd1..b1fc722858e 100644 --- a/test/core/util/port.c +++ b/test/core/util/port.c @@ -79,7 +79,7 @@ static void chose_port(int port) { chosen_ports[num_chosen_ports - 1] = port; } -int grpc_pick_unused_port(void) { +static int grpc_pick_unused_port_impl(void) { int port = grpc_pick_port_using_server(); if (port != 0) { chose_port(port); @@ -88,7 +88,7 @@ int grpc_pick_unused_port(void) { return port; } -int grpc_pick_unused_port_or_die(void) { +static int grpc_pick_unused_port_or_die_impl(void) { int port = grpc_pick_unused_port(); if (port == 0) { fprintf(stderr, @@ -101,6 +101,31 @@ int grpc_pick_unused_port_or_die(void) { return port; } -void grpc_recycle_unused_port(int port) { GPR_ASSERT(free_chosen_port(port)); } +static void grpc_recycle_unused_port_impl(int port) { + GPR_ASSERT(free_chosen_port(port)); +} + +static grpc_pick_port_functions g_pick_port_functions = { + grpc_pick_unused_port_impl, grpc_pick_unused_port_or_die_impl, + grpc_recycle_unused_port_impl}; + +int grpc_pick_unused_port(void) { + return g_pick_port_functions.pick_unused_port_fn(); +} + +int grpc_pick_unused_port_or_die(void) { + return g_pick_port_functions.pick_unused_port_or_die_fn(); +} + +void grpc_recycle_unused_port(int port) { + g_pick_port_functions.recycle_unused_port_fn(port); +} + +void grpc_set_pick_port_functions(grpc_pick_port_functions functions) { + GPR_ASSERT(functions.pick_unused_port_fn != NULL); + GPR_ASSERT(functions.pick_unused_port_or_die_fn != NULL); + GPR_ASSERT(functions.recycle_unused_port_fn != NULL); + g_pick_port_functions = functions; +} #endif /* GRPC_TEST_PICK_PORT */ diff --git a/test/core/util/port.h b/test/core/util/port.h index 154e8f830c6..602099dea6d 100644 --- a/test/core/util/port.h +++ b/test/core/util/port.h @@ -23,6 +23,12 @@ extern "C" { #endif +typedef struct grpc_pick_port_functions { + int (*pick_unused_port_fn)(void); + int (*pick_unused_port_or_die_fn)(void); + void (*recycle_unused_port_fn)(int port); +} grpc_pick_port_functions; + /* pick a port number that is currently unused by either tcp or udp. return 0 on failure. */ int grpc_pick_unused_port(void); @@ -36,6 +42,9 @@ int grpc_pick_unused_port_or_die(void); * ports back to the server if it is going to allocate a large number. */ void grpc_recycle_unused_port(int port); +/** Request the family of pick_port functions in \a functions be used. */ +void grpc_set_pick_port_functions(grpc_pick_port_functions functions); + #ifdef __cplusplus } #endif diff --git a/test/cpp/common/BUILD b/test/cpp/common/BUILD index bd1173322a7..be9c279f13e 100644 --- a/test/cpp/common/BUILD +++ b/test/cpp/common/BUILD @@ -27,7 +27,7 @@ grpc_cc_test( name = "alarm_cpp_test", srcs = ["alarm_cpp_test.cc"], deps = [ - "//:grpc++", + "//:grpc++_unsecure", "//test/core/util:gpr_test_util", ], external_deps = [ diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 4fef5355069..b5cff664f63 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -215,7 +215,8 @@ class BalancerServiceImpl : public BalancerService { { std::unique_lock lock(mu_); if (shutdown_) goto done; - serverlist_cond_.wait(lock); + serverlist_cond_.wait(lock, [this] { return serverlist_ready_; }); + serverlist_ready_ = false; } if (client_load_reporting_interval_seconds_ > 0) { @@ -242,6 +243,7 @@ class BalancerServiceImpl : public BalancerService { .drop_token_counts[drop_token_count.load_balance_token()] += drop_token_count.num_calls(); } + load_report_ready_ = true; load_report_cond_.notify_one(); } done: @@ -285,12 +287,14 @@ class BalancerServiceImpl : public BalancerService { const ClientStats& WaitForLoadReport() { std::unique_lock lock(mu_); - load_report_cond_.wait(lock); + load_report_cond_.wait(lock, [this] { return load_report_ready_; }); + load_report_ready_ = false; return client_stats_; } void NotifyDoneWithServerlists() { std::lock_guard lock(mu_); + serverlist_ready_ = true; serverlist_cond_.notify_one(); } @@ -313,7 +317,9 @@ class BalancerServiceImpl : public BalancerService { std::vector responses_and_delays_; std::mutex mu_; std::condition_variable load_report_cond_; + bool load_report_ready_ = false; std::condition_variable serverlist_cond_; + bool serverlist_ready_ = false; ClientStats client_stats_; bool shutdown_; }; diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD index 5e1bcee89ff..442da38426d 100644 --- a/test/cpp/microbenchmarks/BUILD +++ b/test/cpp/microbenchmarks/BUILD @@ -40,9 +40,9 @@ grpc_cc_library( "helpers.h", ], deps = [ - "//:grpc++", + "//:grpc++_unsecure", "//src/proto/grpc/testing:echo_proto", - "//test/core/util:grpc_test_util", + "//test/core/util:grpc_test_util_unsecure", ], external_deps = [ "benchmark", diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index f7dda0f7589..265f174cc5c 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -120,7 +120,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { BenchmarkService::Stub* stub_; CompletionQueue* cq_; std::unique_ptr alarm_; - RequestType req_; + const RequestType& req_; ResponseType response_; enum State { INVALID, READY, RESP_DONE }; State next_state_; @@ -415,7 +415,7 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { BenchmarkService::Stub* stub_; CompletionQueue* cq_; std::unique_ptr alarm_; - RequestType req_; + const RequestType& req_; ResponseType response_; enum State { INVALID, @@ -554,7 +554,7 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { BenchmarkService::Stub* stub_; CompletionQueue* cq_; std::unique_ptr alarm_; - RequestType req_; + const RequestType& req_; ResponseType response_; enum State { INVALID, @@ -672,7 +672,7 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { BenchmarkService::Stub* stub_; CompletionQueue* cq_; std::unique_ptr alarm_; - RequestType req_; + const RequestType& req_; ResponseType response_; enum State { INVALID, STREAM_IDLE, READ_DONE }; State next_state_; diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index 122976c397f..8b00bcfeaf5 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -550,8 +550,7 @@ static Status ProcessGenericRPC(const PayloadConfig &payload_config, ByteBuffer *response) { int resp_size = payload_config.bytebuf_params().resp_size(); std::unique_ptr buf(new char[resp_size]); - grpc_slice s = grpc_slice_from_copied_buffer(buf.get(), resp_size); - Slice slice(s, Slice::STEAL_REF); + Slice slice(buf.get(), resp_size); *response = ByteBuffer(&slice, 1); return Status::OK; } diff --git a/test/cpp/server/BUILD b/test/cpp/server/BUILD index 512241e3504..3f63be2aa33 100644 --- a/test/cpp/server/BUILD +++ b/test/cpp/server/BUILD @@ -20,9 +20,9 @@ grpc_cc_test( name = "server_builder_test", srcs = ["server_builder_test.cc"], deps = [ - "//:grpc++", + "//:grpc++_unsecure", "//src/proto/grpc/testing:echo_proto", - "//test/core/util:grpc_test_util", + "//test/core/util:grpc_test_util_unsecure", ], external_deps = [ "gtest", @@ -33,9 +33,9 @@ grpc_cc_test( name = "server_request_call_test", srcs = ["server_request_call_test.cc"], deps = [ - "//:grpc++", + "//:grpc++_unsecure", "//src/proto/grpc/testing:echo_proto", - "//test/core/util:grpc_test_util", + "//test/core/util:grpc_test_util_unsecure", ], external_deps = [ "gtest", diff --git a/test/cpp/util/BUILD b/test/cpp/util/BUILD index c9b0d6c1df7..fbdec056989 100644 --- a/test/cpp/util/BUILD +++ b/test/cpp/util/BUILD @@ -24,16 +24,6 @@ package( ], ) -# The following builds a shared-object to confirm that grpc++_unsecure -# builds properly. Build-only is sufficient here -grpc_cc_binary( - name = "testso.so", - srcs = [], - linkshared = 1, - linkopts = ['-Wl,--no-undefined'], - deps = ["//:grpc++_unsecure"], -) - grpc_cc_library( name = "test_config", srcs = [ @@ -64,26 +54,43 @@ grpc_cc_library( ], ) +GRPCXX_TESTUTIL_SRCS = [ + "byte_buffer_proto_helper.cc", + "string_ref_helper.cc", + "subprocess.cc", +] + +GRPCXX_TESTUTIL_HDRS = [ + "byte_buffer_proto_helper.h", + "string_ref_helper.h", + "subprocess.h", +] + grpc_cc_library( name = "test_util", - srcs = [ - "byte_buffer_proto_helper.cc", + srcs = GRPCXX_TESTUTIL_SRCS + [ "create_test_channel.cc", - "string_ref_helper.cc", - "subprocess.cc", "test_credentials_provider.cc", ], - hdrs = [ - "byte_buffer_proto_helper.h", + hdrs = GRPCXX_TESTUTIL_HDRS + [ "create_test_channel.h", - "string_ref_helper.h", - "subprocess.h", "test_credentials_provider.h", ], deps = [ "//:grpc++", "//test/core/end2end:ssl_test_data", - "//test/core/util:gpr_test_util", + ], + external_deps = [ + "protobuf", + ], +) + +grpc_cc_library( + name = "test_util_unsecure", + srcs = GRPCXX_TESTUTIL_SRCS, + hdrs = GRPCXX_TESTUTIL_HDRS, + deps = [ + "//:grpc++_unsecure", ], external_deps = [ "protobuf", diff --git a/test/cpp/util/slice_test.cc b/test/cpp/util/slice_test.cc index 9e3329fab02..8a8962d7ee0 100644 --- a/test/cpp/util/slice_test.cc +++ b/test/cpp/util/slice_test.cc @@ -63,6 +63,42 @@ TEST_F(SliceTest, StaticBuf) { CheckSlice(spp, kContent); } +TEST_F(SliceTest, SliceNew) { + char* x = new char[strlen(kContent) + 1]; + strcpy(x, kContent); + Slice spp(x, strlen(x), [](void* p) { delete[] reinterpret_cast(p); }); + CheckSlice(spp, kContent); +} + +TEST_F(SliceTest, SliceNewDoNothing) { + Slice spp(const_cast(kContent), strlen(kContent), [](void* p) {}); + CheckSlice(spp, kContent); +} + +TEST_F(SliceTest, SliceNewWithUserData) { + struct stest { + char* x; + int y; + }; + auto* t = new stest; + t->x = new char[strlen(kContent) + 1]; + strcpy(t->x, kContent); + Slice spp(t->x, strlen(t->x), + [](void* p) { + auto* t = reinterpret_cast(p); + delete[] t->x; + delete t; + }, + t); + CheckSlice(spp, kContent); +} + +TEST_F(SliceTest, SliceNewLen) { + Slice spp(const_cast(kContent), strlen(kContent), + [](void* p, size_t l) { EXPECT_EQ(l, strlen(kContent)); }); + CheckSlice(spp, kContent); +} + TEST_F(SliceTest, Steal) { grpc_slice s = grpc_slice_from_copied_string(kContent); Slice spp(s, Slice::STEAL_REF); diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 33524311493..7290f20c86b 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -959,6 +959,7 @@ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.h \ src/core/lib/iomgr/exec_ctx.h \ src/core/lib/iomgr/executor.h \ +src/core/lib/iomgr/gethostname.h \ src/core/lib/iomgr/iocp_windows.h \ src/core/lib/iomgr/iomgr.h \ src/core/lib/iomgr/iomgr_internal.h \ @@ -1033,6 +1034,7 @@ src/core/lib/support/string_windows.h \ src/core/lib/support/thd_internal.h \ src/core/lib/support/time_precise.h \ src/core/lib/support/tmpfile.h \ +src/core/lib/surface/alarm_internal.h \ src/core/lib/surface/api_trace.h \ src/core/lib/surface/call.h \ src/core/lib/surface/call_test_only.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 9951edc6782..4901fc92182 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1116,6 +1116,10 @@ src/core/lib/iomgr/exec_ctx.c \ src/core/lib/iomgr/exec_ctx.h \ src/core/lib/iomgr/executor.c \ src/core/lib/iomgr/executor.h \ +src/core/lib/iomgr/gethostname.h \ +src/core/lib/iomgr/gethostname_fallback.c \ +src/core/lib/iomgr/gethostname_host_name_max.c \ +src/core/lib/iomgr/gethostname_sysconf.c \ src/core/lib/iomgr/iocp_windows.c \ src/core/lib/iomgr/iocp_windows.h \ src/core/lib/iomgr/iomgr.c \ @@ -1342,6 +1346,7 @@ src/core/lib/support/tmpfile_windows.c \ src/core/lib/support/wrap_memcpy.c \ src/core/lib/surface/README.md \ src/core/lib/surface/alarm.c \ +src/core/lib/surface/alarm_internal.h \ src/core/lib/surface/api_trace.c \ src/core/lib/surface/api_trace.h \ src/core/lib/surface/byte_buffer.c \ @@ -1416,6 +1421,8 @@ src/core/tsi/transport_security.c \ src/core/tsi/transport_security.h \ src/core/tsi/transport_security_adapter.c \ src/core/tsi/transport_security_adapter.h \ +src/core/tsi/transport_security_grpc.c \ +src/core/tsi/transport_security_grpc.h \ src/core/tsi/transport_security_interface.h \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_decode.c \ diff --git a/tools/flakes/detect_flakes.py b/tools/flakes/detect_flakes.py new file mode 100644 index 00000000000..7c2f0129648 --- /dev/null +++ b/tools/flakes/detect_flakes.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Detect new flakes introduced in the last 24h hours with respect to the +previous six days""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import logging +logging.basicConfig(format='%(asctime)s %(message)s') + +gcp_utils_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../gcp/utils')) +sys.path.append(gcp_utils_dir) + +import big_query_utils + + +def get_flaky_tests(days_lower_bound, days_upper_bound, limit=None): + """ period is one of "WEEK", "DAY", etc. + (see https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-and-operators#date_add). """ + + bq = big_query_utils.create_big_query() + query = """ +SELECT + filtered_test_name, + FIRST(timestamp), + FIRST(build_url), +FROM ( + SELECT + REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name, + result, + build_url, + timestamp + FROM + [grpc-testing:jenkins_test_results.aggregate_results] + WHERE + timestamp >= DATE_ADD(CURRENT_DATE(), {days_lower_bound}, "DAY") + AND timestamp <= DATE_ADD(CURRENT_DATE(), {days_upper_bound}, "DAY") + AND NOT REGEXP_MATCH(job_name, '.*portability.*')) +GROUP BY + filtered_test_name, + timestamp, + build_url +HAVING + SUM(result != 'PASSED' + AND result != 'SKIPPED') > 0 +ORDER BY + timestamp ASC +""".format(days_lower_bound=days_lower_bound, days_upper_bound=days_upper_bound) + if limit: + query += '\n LIMIT {}'.format(limit) + query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query) + page = bq.jobs().getQueryResults( + pageToken=None, **query_job['jobReference']).execute(num_retries=3) + testname_to_ts_url_pair = {row['f'][0]['v']: (row['f'][1]['v'], row['f'][2]['v']) for row in page['rows']} + return testname_to_ts_url_pair + + +def get_new_flakes(): + last_week_sans_yesterday = get_flaky_tests(-7, -1) + last_24 = get_flaky_tests(-1, +1) + last_week_sans_yesterday_names = set(last_week_sans_yesterday.keys()) + last_24_names = set(last_24.keys()) + logging.debug('|last_week_sans_yesterday| =', len(last_week_sans_yesterday_names)) + logging.debug('|last_24_names| =', len(last_24_names)) + new_flakes = last_24_names - last_week_sans_yesterday_names + logging.debug('|new_flakes| = ', len(new_flakes)) + return {k: last_24[k] for k in new_flakes} + + +def main(): + import datetime + new_flakes = get_new_flakes() + if new_flakes: + print("Found {} new flakes:".format(len(new_flakes))) + for k, v in new_flakes.items(): + ts = int(float(v[0])) + url = v[1] + human_ts = datetime.datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S UTC') + print("Test: {}, Timestamp: {}, URL: {}\n".format(k, human_ts, url)) + + +if __name__ == '__main__': + main() diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc new file mode 100644 index 00000000000..f467ac006af --- /dev/null +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Source this rc script to prepare the environment for MacOS interop +# builds. This rc script must be used in the root directory of gRPC +# and is expected to be used before prepare_build_macos_rc + +export CONFIG=opt + +# Move gRPC repo to directory that Docker for Mac has drive access to +mkdir /Users/kbuilder/workspace +cp -R ./ /Users/kbuilder/workspace/grpc +cd /Users/kbuilder/workspace/grpc + +# Needed for identifying Docker image sha1 +brew install md5sha1sum + +# Set up gRPC-Go and gRPC-Java to test +git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go +git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java + +# Set up Docker for Mac +docker-machine create -d virtualbox --virtualbox-share-folder "/Users/kbuilder/workspace:" default +docker-machine env default +eval $(docker-machine env default) + diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index 00105d48383..b48040f5e26 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -31,7 +31,6 @@ ulimit -a # pip does not install google-api-python-client properly, so use easy_install sudo easy_install --upgrade google-api-python-client export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json -gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS # required to build protobuf brew install gflags diff --git a/tools/internal_ci/macos/grpc_basictests.cfg b/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg similarity index 74% rename from tools/internal_ci/macos/grpc_basictests.cfg rename to tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg index 3faba2fc36c..11c211fd2b4 100644 --- a/tools/internal_ci/macos/grpc_basictests.cfg +++ b/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg @@ -15,9 +15,8 @@ # Config file for the internal CI (in protobuf text format) # Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh" -gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" -timeout_mins: 240 +build_file: "grpc/tools/internal_ci/linux/grpc_build_submodule_at_head.sh" +timeout_mins: 180 action { define_artifacts { regex: "**/*sponge_log.xml" @@ -25,7 +24,8 @@ action { } } +# Tiny hack: misusing an already whitelisted env var to pass submodule name env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" + value: "boringssl" } diff --git a/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg b/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg new file mode 100644 index 00000000000..2f08e15e638 --- /dev/null +++ b/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg @@ -0,0 +1,31 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_build_submodule_at_head.sh" +timeout_mins: 180 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +# Tiny hack: misusing an already whitelisted env var to pass submodule name +env_vars { + key: "RUN_TESTS_FLAGS" + value: "protobuf" +} diff --git a/tools/internal_ci/linux/grpc_build_submodule_at_head.sh b/tools/internal_ci/linux/grpc_build_submodule_at_head.sh new file mode 100755 index 00000000000..8189e6876b0 --- /dev/null +++ b/tools/internal_ci/linux/grpc_build_submodule_at_head.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build portability tests with an updated submodule + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +# Update submodule and commit it so changes are passed to Docker +(cd third_party/$RUN_TESTS_FLAGS && git pull origin master) +git -c user.name='foo' -c user.email='foo@google.com' commit -a -m 'Update submodule' + +tools/run_tests/run_tests_matrix.py -f linux --internal_ci --build_only + diff --git a/tools/internal_ci/macos/grpc_interop.sh b/tools/internal_ci/macos/grpc_interop.sh index 07601a67e4a..b03401bd962 100755 --- a/tools/internal_ci/macos/grpc_interop.sh +++ b/tools/internal_ci/macos/grpc_interop.sh @@ -18,6 +18,7 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -source tools/internal_ci/helper_scripts/prepare_build_interop_rc +source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc +source tools/internal_ci/helper_scripts/prepare_build_macos_rc tools/run_tests/run_interop_tests.py -l objc -s all --use_docker -t -j 1 diff --git a/tools/internal_ci/windows/grpc_portability_master.cfg b/tools/internal_ci/windows/grpc_portability_build_only.cfg similarity index 93% rename from tools/internal_ci/windows/grpc_portability_master.cfg rename to tools/internal_ci/windows/grpc_portability_build_only.cfg index c395cb4a949..b2b58ece2d7 100644 --- a/tools/internal_ci/windows/grpc_portability_master.cfg +++ b/tools/internal_ci/windows/grpc_portability_build_only.cfg @@ -26,5 +26,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f portability windows -j 1 --inner_jobs 8 --internal_ci" + value: "-f portability windows --internal_ci --build_only" } diff --git a/tools/run_tests/dockerize/build_interop_stress_image.sh b/tools/run_tests/dockerize/build_interop_stress_image.sh deleted file mode 100755 index acb566f5d54..00000000000 --- a/tools/run_tests/dockerize/build_interop_stress_image.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This script is invoked by run_interop_tests.py to build the docker image -# for interop testing. You should never need to call this script on your own. - -set -x - -# Params: -# INTEROP_IMAGE - Name of tag of the final interop image -# INTEROP_IMAGE_REPOSITORY_TAG - Optional. If set, the created image will be tagged using -# the command: 'docker tag $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG' -# BASE_NAME - Base name used to locate the base Dockerfile and build script -# BUILD_TYPE - The 'CONFIG' variable passed to the 'make' command (example: -# asan, tsan. Default value: opt). -# TTY_FLAG - optional -t flag to make docker allocate tty -# BUILD_INTEROP_DOCKER_EXTRA_ARGS - optional args to be passed to the -# docker run command - -cd `dirname $0`/../../.. -GRPC_ROOT=`pwd` -MOUNT_ARGS="-v $GRPC_ROOT:/var/local/jenkins/grpc:ro" - -GRPC_JAVA_ROOT=`cd ../grpc-java && pwd` -if [ "$GRPC_JAVA_ROOT" != "" ] -then - MOUNT_ARGS+=" -v $GRPC_JAVA_ROOT:/var/local/jenkins/grpc-java:ro" -else - echo "WARNING: grpc-java not found, it won't be mounted to the docker container." -fi - -GRPC_GO_ROOT=`cd ../grpc-go && pwd` -if [ "$GRPC_GO_ROOT" != "" ] -then - MOUNT_ARGS+=" -v $GRPC_GO_ROOT:/var/local/jenkins/grpc-go:ro" -else - echo "WARNING: grpc-go not found, it won't be mounted to the docker container." -fi - -mkdir -p /tmp/ccache - -# Mount service account dir if available. -# If service_directory does not contain the service account JSON file, -# some of the tests will fail. -if [ -e $HOME/service_account ] -then - MOUNT_ARGS+=" -v $HOME/service_account:/var/local/jenkins/service_account:ro" -fi - -# Use image name based on Dockerfile checksum -BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/stress_test/$BASE_NAME/Dockerfile | cut -f1 -d\ ` - -# Make sure base docker image has been built. Should be instantaneous if so. -docker build -t $BASE_IMAGE --force-rm=true tools/dockerfile/stress_test/$BASE_NAME || exit $? - -# Create a local branch so the child Docker script won't complain -git branch -f jenkins-docker - -CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" - -# Prepare image for interop tests, commit it on success. -(docker run \ - -e CCACHE_DIR=/tmp/ccache \ - -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ - -e BUILD_TYPE=${BUILD_TYPE:=opt} \ - -i $TTY_FLAG \ - $MOUNT_ARGS \ - $BUILD_INTEROP_DOCKER_EXTRA_ARGS \ - -v /tmp/ccache:/tmp/ccache \ - --name=$CONTAINER_NAME \ - $BASE_IMAGE \ - bash -l /var/local/jenkins/grpc/tools/dockerfile/stress_test/$BASE_NAME/build_interop_stress.sh \ - && docker commit $CONTAINER_NAME $INTEROP_IMAGE \ - && ( if [ -n "$INTEROP_IMAGE_REPOSITORY_TAG" ]; then docker tag $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG ; fi ) \ - && echo "Successfully built image $INTEROP_IMAGE") -EXITCODE=$? - -# remove intermediate container, possibly killing it first -docker rm -f $CONTAINER_NAME - -exit $EXITCODE diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index d5eba6e82d1..30fef6fc0a5 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7833,6 +7833,9 @@ "src/core/lib/iomgr/ev_windows.c", "src/core/lib/iomgr/exec_ctx.c", "src/core/lib/iomgr/executor.c", + "src/core/lib/iomgr/gethostname_fallback.c", + "src/core/lib/iomgr/gethostname_host_name_max.c", + "src/core/lib/iomgr/gethostname_sysconf.c", "src/core/lib/iomgr/iocp_windows.c", "src/core/lib/iomgr/iomgr.c", "src/core/lib/iomgr/iomgr_posix.c", @@ -7979,6 +7982,7 @@ "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", "src/core/lib/iomgr/executor.h", + "src/core/lib/iomgr/gethostname.h", "src/core/lib/iomgr/iocp_windows.h", "src/core/lib/iomgr/iomgr.h", "src/core/lib/iomgr/iomgr_internal.h", @@ -8035,6 +8039,7 @@ "src/core/lib/slice/slice_hash_table.h", "src/core/lib/slice/slice_internal.h", "src/core/lib/slice/slice_string_helpers.h", + "src/core/lib/surface/alarm_internal.h", "src/core/lib/surface/api_trace.h", "src/core/lib/surface/call.h", "src/core/lib/surface/call_test_only.h", @@ -8107,6 +8112,7 @@ "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", "src/core/lib/iomgr/executor.h", + "src/core/lib/iomgr/gethostname.h", "src/core/lib/iomgr/iocp_windows.h", "src/core/lib/iomgr/iomgr.h", "src/core/lib/iomgr/iomgr_internal.h", @@ -8163,6 +8169,7 @@ "src/core/lib/slice/slice_hash_table.h", "src/core/lib/slice/slice_internal.h", "src/core/lib/slice/slice_string_helpers.h", + "src/core/lib/surface/alarm_internal.h", "src/core/lib/surface/api_trace.h", "src/core/lib/surface/call.h", "src/core/lib/surface/call_test_only.h", @@ -9088,16 +9095,15 @@ "deps": [ "gpr", "grpc_base", - "grpc_trace" + "grpc_trace", + "tsi_interface" ], "headers": [ "src/core/tsi/fake_transport_security.h", "src/core/tsi/gts_transport_security.h", "src/core/tsi/ssl_transport_security.h", "src/core/tsi/ssl_types.h", - "src/core/tsi/transport_security.h", - "src/core/tsi/transport_security_adapter.h", - "src/core/tsi/transport_security_interface.h" + "src/core/tsi/transport_security_grpc.h" ], "is_filegroup": true, "language": "c", @@ -9110,6 +9116,26 @@ "src/core/tsi/ssl_transport_security.c", "src/core/tsi/ssl_transport_security.h", "src/core/tsi/ssl_types.h", + "src/core/tsi/transport_security_grpc.c", + "src/core/tsi/transport_security_grpc.h" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ + "gpr", + "grpc_trace" + ], + "headers": [ + "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_adapter.h", + "src/core/tsi/transport_security_interface.h" + ], + "is_filegroup": true, + "language": "c", + "name": "tsi_interface", + "src": [ "src/core/tsi/transport_security.c", "src/core/tsi/transport_security.h", "src/core/tsi/transport_security_adapter.c", diff --git a/tools/run_tests/run_build_statistics.py b/tools/run_tests/run_build_statistics.py index d63dc3e86fe..0ac6fc50aab 100755 --- a/tools/run_tests/run_build_statistics.py +++ b/tools/run_tests/run_build_statistics.py @@ -152,16 +152,13 @@ def _process_build(json_url, console_url): failure_count = test_result['failCount'] build_result['pass_count'] = test_result['passCount'] build_result['failure_count'] = failure_count + # This means Jenkins failure occurred. build_result['no_report_files_found'] = _no_report_files_found(html) - if failure_count > 0: + # Only check errors if Jenkins failure occurred. + if build_result['no_report_files_found']: error_list, known_error_count = _scrape_for_known_errors(html) - unknown_error_count = failure_count - known_error_count - # This can happen if the same error occurs multiple times in one test. - if failure_count < known_error_count: - print('====> Some errors are duplicates.') - unknown_error_count = 0 - error_list.append({'description': _UNKNOWN_ERROR, - 'count': unknown_error_count}) + if not error_list: + error_list.append({'description': _UNKNOWN_ERROR, 'count': 1}) except Exception as e: print('====> Got exception for %s: %s.' % (json_url, str(e))) print('====> Parsing errors from %s.' % console_url) @@ -176,6 +173,8 @@ def _process_build(json_url, console_url): if error_list: build_result['error'] = error_list + else: + build_result['error'] = [{'description': '', 'count': 0}] return build_result diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index dbbf2adb225..1537641aee9 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -731,7 +731,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, if manual_cmd_log is not None: if manual_cmd_log == []: manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % docker_image) - manual_cmd_log.append(manual_cmdline(cmdline, docker_iamge)) + manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) cwd = None test_job = jobset.JobSpec( @@ -793,7 +793,7 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): if manual_cmd_log is not None: if manual_cmd_log == []: manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % docker_image) - manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_iamge)) + manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image)) server_job = jobset.JobSpec( cmdline=docker_cmdline, environ=environ, diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index c953f904794..461928bcefc 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -74,19 +74,21 @@ def get_flaky_tests(limit=None): bq = big_query_utils.create_big_query() query = """ - SELECT - test_name, - SUM(result != 'PASSED' - AND result != 'SKIPPED') AS count_failed, - FROM - [grpc-testing:jenkins_test_results.aggregate_results] - WHERE - timestamp >= DATE_ADD(CURRENT_DATE(), -1, "WEEK") - AND NOT REGEXP_MATCH(job_name, '.*portability.*') - GROUP BY - test_name - HAVING - count_failed > 0""" +SELECT + filtered_test_name, + FROM ( + SELECT + REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name, + result + FROM + [grpc-testing:jenkins_test_results.aggregate_results] + WHERE + timestamp >= DATE_ADD(CURRENT_DATE(), -1, "WEEK") + AND NOT REGEXP_MATCH(job_name, '.*portability.*') ) +GROUP BY + filtered_test_name +HAVING + SUM(result != 'PASSED' AND result != 'SKIPPED') > 0""" if limit: query += " limit {}".format(limit) query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query) diff --git a/tools/run_tests/sanity/core_untyped_structs.sh b/tools/run_tests/sanity/core_untyped_structs.sh new file mode 100755 index 00000000000..792dd68fdd0 --- /dev/null +++ b/tools/run_tests/sanity/core_untyped_structs.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +cd `dirname $0`/../../.. + +# +# Make sure that all core struct/unions have a name or are typedef'ed +# + +egrep -Irn '(struct|union) *{' include/grpc | + egrep -v typedef | + diff - /dev/null + diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index a86ebee7b41..7e582bc40bf 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -6,6 +6,7 @@ - script: tools/run_tests/sanity/check_test_filtering.py - script: tools/run_tests/sanity/check_tracer_sanity.py - script: tools/run_tests/sanity/core_banned_functions.py +- script: tools/run_tests/sanity/core_untyped_structs.sh - script: tools/buildgen/generate_projects.sh -j 3 cpu_cost: 3 - script: tools/distrib/check_copyright.py diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 625c62adcd6..416adb4efdd 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -450,6 +450,7 @@ + @@ -506,6 +507,7 @@ + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index ff9913a362a..0e33351e055 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -701,6 +701,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -869,6 +872,9 @@ src\core\lib\slice + + src\core\lib\surface + src\core\lib\surface diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index baeb6e35b2d..8ea35c18daa 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -444,6 +444,7 @@ + @@ -500,6 +501,7 @@ + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 5820ce04141..edc3b81121a 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -668,6 +668,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -836,6 +839,9 @@ src\core\lib\slice + + src\core\lib\surface + src\core\lib\surface diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 3f711f690f8..8a659280a46 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -347,6 +347,7 @@ + @@ -401,6 +402,7 @@ + @@ -457,6 +459,7 @@ + @@ -582,6 +585,12 @@ + + + + + + @@ -884,6 +893,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 662eddfc525..6879047f55b 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -94,6 +94,15 @@ src\core\lib\iomgr + + src\core\lib\iomgr + + + src\core\lib\iomgr + + + src\core\lib\iomgr + src\core\lib\iomgr @@ -547,6 +556,9 @@ src\core\tsi + + src\core\tsi + src\core\tsi @@ -1001,6 +1013,9 @@ src\core\tsi + + src\core\tsi + src\core\tsi @@ -1163,6 +1178,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -1331,6 +1349,9 @@ src\core\lib\slice + + src\core\lib\surface + src\core\lib\surface diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index fa31dbe3e19..1766e2445d7 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -329,6 +329,12 @@ + + + + + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index fc9f64a6146..8516a07196d 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -151,6 +151,15 @@ src\core\lib\iomgr + + src\core\lib\iomgr + + + src\core\lib\iomgr + + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj index 457dbd57c74..c92f0d12378 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj @@ -317,6 +317,12 @@ + + + + + + diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters index 49e886de3de..df34bdba88d 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters @@ -136,6 +136,15 @@ src\core\lib\iomgr + + src\core\lib\iomgr + + + src\core\lib\iomgr + + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index c0ce8f0a4be..945ca2f3d72 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -365,6 +365,7 @@ + @@ -421,6 +422,7 @@ + @@ -548,6 +550,12 @@ + + + + + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 2d2a8200e0a..deb3433168d 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -97,6 +97,15 @@ src\core\lib\iomgr + + src\core\lib\iomgr + + + src\core\lib\iomgr + + + src\core\lib\iomgr + src\core\lib\iomgr @@ -992,6 +1001,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -1160,6 +1172,9 @@ src\core\lib\slice + + src\core\lib\surface + src\core\lib\surface