diff --git a/BUILD b/BUILD index de0382d0bc2..3a148cb41ec 100644 --- a/BUILD +++ b/BUILD @@ -2384,23 +2384,6 @@ grpc_cc_library( ], ) -grpc_cc_library( - name = "bdp_estimator", - srcs = [ - "src/core/lib/transport/bdp_estimator.cc", - ], - hdrs = ["src/core/lib/transport/bdp_estimator.h"], - tags = ["grpc-autodeps"], - deps = [ - "exec_ctx", - "gpr_base", - "gpr_codegen", - "gpr_platform", - "grpc_trace", - "time", - ], -) - grpc_cc_library( name = "percent_encoding", srcs = [ @@ -2523,6 +2506,8 @@ grpc_cc_library( "src/core/lib/surface/server.cc", "src/core/lib/surface/validate_metadata.cc", "src/core/lib/surface/version.cc", + "src/core/lib/transport/bdp_estimator.cc", + "src/core/lib/transport/byte_stream.cc", "src/core/lib/transport/connectivity_state.cc", "src/core/lib/transport/error_utils.cc", "src/core/lib/transport/metadata_batch.cc", @@ -2618,6 +2603,8 @@ grpc_cc_library( "src/core/lib/surface/lame_client.h", "src/core/lib/surface/server.h", "src/core/lib/surface/validate_metadata.h", + "src/core/lib/transport/bdp_estimator.h", + "src/core/lib/transport/byte_stream.h", "src/core/lib/transport/connectivity_state.h", "src/core/lib/transport/metadata_batch.h", "src/core/lib/transport/parsed_metadata.h", @@ -3071,7 +3058,6 @@ grpc_cc_library( "absl/types:variant", "absl/status", "absl/status:statusor", - "absl/utility", "upb_lib", ], language = "c++", @@ -3083,7 +3069,6 @@ grpc_cc_library( "channel_stack_type", "chunked_vector", "config", - "construct_destruct", "debug_location", "dual_ref_counted", "error", @@ -3115,7 +3100,6 @@ grpc_cc_library( "server_address", "service_config_parser", "slice", - "slice_buffer", "slice_refcount", "sockaddr_utils", "time", @@ -3348,8 +3332,8 @@ grpc_cc_library( "grpc_public_hdrs", "grpc_service_config", "json", + "orphanable", "service_config_parser", - "slice_buffer", ], ) @@ -3477,7 +3461,6 @@ grpc_cc_library( "promise", "seq", "slice", - "slice_buffer", "transport_fwd", ], ) @@ -5721,33 +5704,6 @@ grpc_cc_library( ], ) -grpc_cc_library( - name = "chttp2_flow_control", - srcs = [ - "src/core/ext/transport/chttp2/transport/flow_control.cc", - ], - hdrs = [ - "src/core/ext/transport/chttp2/transport/flow_control.h", - ], - external_deps = [ - "absl/status", - "absl/strings", - "absl/strings:str_format", - ], - tags = ["grpc-autodeps"], - deps = [ - "bdp_estimator", - "exec_ctx", - "gpr_base", - "gpr_platform", - "grpc_trace", - "memory_quota", - "pid_controller", - "time", - "useful", - ], -) - grpc_cc_library( name = "grpc_transport_chttp2", srcs = [ @@ -5755,6 +5711,7 @@ grpc_cc_library( "src/core/ext/transport/chttp2/transport/bin_encoder.cc", "src/core/ext/transport/chttp2/transport/chttp2_transport.cc", "src/core/ext/transport/chttp2/transport/context_list.cc", + "src/core/ext/transport/chttp2/transport/flow_control.cc", "src/core/ext/transport/chttp2/transport/frame_data.cc", "src/core/ext/transport/chttp2/transport/frame_goaway.cc", "src/core/ext/transport/chttp2/transport/frame_ping.cc", @@ -5777,6 +5734,7 @@ grpc_cc_library( "src/core/ext/transport/chttp2/transport/bin_encoder.h", "src/core/ext/transport/chttp2/transport/chttp2_transport.h", "src/core/ext/transport/chttp2/transport/context_list.h", + "src/core/ext/transport/chttp2/transport/flow_control.h", "src/core/ext/transport/chttp2/transport/frame.h", "src/core/ext/transport/chttp2/transport/frame_data.h", "src/core/ext/transport/chttp2/transport/frame_goaway.h", @@ -5809,9 +5767,7 @@ grpc_cc_library( visibility = ["@grpc:grpclb"], deps = [ "arena", - "bdp_estimator", "bitset", - "chttp2_flow_control", "chunked_vector", "debug_location", "gpr_base", @@ -5829,13 +5785,11 @@ grpc_cc_library( "memory_quota", "orphanable", "pid_controller", - "poll", "ref_counted", "ref_counted_ptr", "resource_quota", "resource_quota_trace", "slice", - "slice_buffer", "slice_refcount", "time", "uri_parser", @@ -5962,10 +5916,8 @@ grpc_cc_library( "absl/status:statusor", "absl/strings", "absl/types:optional", - "absl/utility", ], language = "c++", - tags = ["grpc-autodeps"], deps = [ "arena", "channel_args_preconditioning", @@ -5975,12 +5927,11 @@ grpc_cc_library( "gpr_base", "grpc_base", "grpc_codegen", - "grpc_public_hdrs", "grpc_trace", "iomgr_fwd", + "orphanable", "ref_counted_ptr", "slice", - "slice_buffer", "time", "transport_fwd", "useful", @@ -6565,7 +6516,6 @@ grpc_cc_library( "grpc++_base", "grpc_base", "slice", - "slice_buffer", "slice_refcount", ], ) diff --git a/CMakeLists.txt b/CMakeLists.txt index e00d6894dd4..33db3df1441 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -851,6 +851,7 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_c jwt_verifier_test) add_dependencies(buildtests_c lame_client_test) add_dependencies(buildtests_c load_file_test) + add_dependencies(buildtests_c manual_constructor_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c memory_quota_stress_test) endif() @@ -953,6 +954,7 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_cxx binder_transport_test) add_dependencies(buildtests_cxx bitset_test) add_dependencies(buildtests_cxx byte_buffer_test) + add_dependencies(buildtests_cxx byte_stream_test) add_dependencies(buildtests_cxx call_finalization_test) add_dependencies(buildtests_cxx call_push_pull_test) add_dependencies(buildtests_cxx cancel_ares_query_test) @@ -1014,7 +1016,6 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_cxx file_watcher_certificate_provider_factory_test) add_dependencies(buildtests_cxx filter_end2end_test) add_dependencies(buildtests_cxx flaky_network_test) - add_dependencies(buildtests_cxx flow_control_end2end_test) add_dependencies(buildtests_cxx flow_control_test) add_dependencies(buildtests_cxx for_each_test) add_dependencies(buildtests_cxx generic_end2end_test) @@ -2242,6 +2243,7 @@ add_library(grpc src/core/lib/surface/validate_metadata.cc src/core/lib/surface/version.cc src/core/lib/transport/bdp_estimator.cc + src/core/lib/transport/byte_stream.cc src/core/lib/transport/connectivity_state.cc src/core/lib/transport/error_utils.cc src/core/lib/transport/handshaker.cc @@ -2829,6 +2831,7 @@ add_library(grpc_unsecure src/core/lib/surface/validate_metadata.cc src/core/lib/surface/version.cc src/core/lib/transport/bdp_estimator.cc + src/core/lib/transport/byte_stream.cc src/core/lib/transport/connectivity_state.cc src/core/lib/transport/error_utils.cc src/core/lib/transport/handshaker.cc @@ -6003,6 +6006,33 @@ target_link_libraries(load_file_test ) +endif() +if(gRPC_BUILD_TESTS) + +add_executable(manual_constructor_test + test/core/gprpp/manual_constructor_test.cc +) + +target_include_directories(manual_constructor_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) + +target_link_libraries(manual_constructor_test + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util +) + + endif() if(gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) @@ -8325,6 +8355,41 @@ target_link_libraries(byte_buffer_test ) +endif() +if(gRPC_BUILD_TESTS) + +add_executable(byte_stream_test + test/core/transport/byte_stream_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + +target_include_directories(byte_stream_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} + third_party/googletest/googletest/include + third_party/googletest/googletest + third_party/googletest/googlemock/include + third_party/googletest/googlemock + ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(byte_stream_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util +) + + endif() if(gRPC_BUILD_TESTS) @@ -10666,69 +10731,11 @@ target_link_libraries(flaky_network_test ) -endif() -if(gRPC_BUILD_TESTS) - -add_executable(flow_control_end2end_test - test/core/end2end/cq_verifier.cc - test/core/transport/chttp2/flow_control_end2end_test.cc - third_party/googletest/googletest/src/gtest-all.cc - third_party/googletest/googlemock/src/gmock-all.cc -) - -target_include_directories(flow_control_end2end_test - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} - ${_gRPC_RE2_INCLUDE_DIR} - ${_gRPC_SSL_INCLUDE_DIR} - ${_gRPC_UPB_GENERATED_DIR} - ${_gRPC_UPB_GRPC_GENERATED_DIR} - ${_gRPC_UPB_INCLUDE_DIR} - ${_gRPC_XXHASH_INCLUDE_DIR} - ${_gRPC_ZLIB_INCLUDE_DIR} - third_party/googletest/googletest/include - third_party/googletest/googletest - third_party/googletest/googlemock/include - third_party/googletest/googlemock - ${_gRPC_PROTO_GENS_DIR} -) - -target_link_libraries(flow_control_end2end_test - ${_gRPC_PROTOBUF_LIBRARIES} - ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_test_util -) - - endif() if(gRPC_BUILD_TESTS) add_executable(flow_control_test - src/core/ext/transport/chttp2/transport/flow_control.cc - src/core/ext/upb-generated/google/protobuf/any.upb.c - src/core/ext/upb-generated/google/rpc/status.upb.c - src/core/lib/debug/trace.cc - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc - src/core/lib/iomgr/executor.cc - src/core/lib/iomgr/iomgr_internal.cc - src/core/lib/promise/activity.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/resource_quota.cc - src/core/lib/resource_quota/thread_quota.cc - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/transport/bdp_estimator.cc - src/core/lib/transport/pid_controller.cc + test/core/end2end/cq_verifier.cc test/core/transport/chttp2/flow_control_test.cc third_party/googletest/googletest/src/gtest-all.cc third_party/googletest/googlemock/src/gmock-all.cc @@ -10756,12 +10763,7 @@ target_include_directories(flow_control_test target_link_libraries(flow_control_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - absl::type_traits - absl::statusor - absl::variant - absl::utility - gpr - upb + grpc_test_util ) diff --git a/Makefile b/Makefile index 6e0e43893a0..6f622094c8d 100644 --- a/Makefile +++ b/Makefile @@ -1635,6 +1635,7 @@ LIBGRPC_SRC = \ src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ + src/core/lib/transport/byte_stream.cc \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/error_utils.cc \ src/core/lib/transport/handshaker.cc \ @@ -2062,6 +2063,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ + src/core/lib/transport/byte_stream.cc \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/error_utils.cc \ src/core/lib/transport/handshaker.cc \ diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index a8f91658a9b..6c977af0e2d 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -946,6 +946,7 @@ libs: - src/core/lib/surface/server.h - src/core/lib/surface/validate_metadata.h - src/core/lib/transport/bdp_estimator.h + - src/core/lib/transport/byte_stream.h - src/core/lib/transport/connectivity_state.h - src/core/lib/transport/error_utils.h - src/core/lib/transport/handshaker.h @@ -1613,6 +1614,7 @@ libs: - src/core/lib/surface/validate_metadata.cc - src/core/lib/surface/version.cc - src/core/lib/transport/bdp_estimator.cc + - src/core/lib/transport/byte_stream.cc - src/core/lib/transport/connectivity_state.cc - src/core/lib/transport/error_utils.cc - src/core/lib/transport/handshaker.cc @@ -2121,6 +2123,7 @@ libs: - src/core/lib/surface/server.h - src/core/lib/surface/validate_metadata.h - src/core/lib/transport/bdp_estimator.h + - src/core/lib/transport/byte_stream.h - src/core/lib/transport/connectivity_state.h - src/core/lib/transport/error_utils.h - src/core/lib/transport/handshaker.h @@ -2431,6 +2434,7 @@ libs: - src/core/lib/surface/validate_metadata.cc - src/core/lib/surface/version.cc - src/core/lib/transport/bdp_estimator.cc + - src/core/lib/transport/byte_stream.cc - src/core/lib/transport/connectivity_state.cc - src/core/lib/transport/error_utils.cc - src/core/lib/transport/handshaker.cc @@ -3807,6 +3811,15 @@ targets: deps: - grpc_test_util uses_polling: false +- name: manual_constructor_test + build: test + language: c + headers: [] + src: + - test/core/gprpp/manual_constructor_test.cc + deps: + - grpc_test_util + uses_polling: false - name: memory_quota_stress_test build: test language: c @@ -4881,6 +4894,16 @@ targets: deps: - grpc++_test_util uses_polling: false +- name: byte_stream_test + gtest: true + build: test + language: c++ + headers: [] + src: + - test/core/transport/byte_stream_test.cc + deps: + - grpc_test_util + uses_polling: false - name: call_finalization_test gtest: true build: test @@ -5859,7 +5882,7 @@ targets: - test/cpp/end2end/test_service_impl.cc deps: - grpc++_test_util -- name: flow_control_end2end_test +- name: flow_control_test gtest: true build: test language: c++ @@ -5867,90 +5890,9 @@ targets: - test/core/end2end/cq_verifier.h src: - test/core/end2end/cq_verifier.cc - - test/core/transport/chttp2/flow_control_end2end_test.cc - deps: - - grpc_test_util -- name: flow_control_test - gtest: true - build: test - language: c++ - headers: - - src/core/ext/transport/chttp2/transport/flow_control.h - - src/core/ext/upb-generated/google/protobuf/any.upb.h - - src/core/ext/upb-generated/google/rpc/status.upb.h - - src/core/lib/debug/trace.h - - src/core/lib/gprpp/atomic_utils.h - - src/core/lib/gprpp/bitset.h - - src/core/lib/gprpp/cpp_impl_of.h - - src/core/lib/gprpp/orphanable.h - - src/core/lib/gprpp/ref_counted.h - - src/core/lib/gprpp/ref_counted_ptr.h - - src/core/lib/gprpp/status_helper.h - - src/core/lib/gprpp/time.h - - src/core/lib/iomgr/closure.h - - src/core/lib/iomgr/combiner.h - - src/core/lib/iomgr/error.h - - src/core/lib/iomgr/error_internal.h - - src/core/lib/iomgr/exec_ctx.h - - src/core/lib/iomgr/executor.h - - src/core/lib/iomgr/iomgr_internal.h - - src/core/lib/promise/activity.h - - src/core/lib/promise/context.h - - src/core/lib/promise/detail/basic_seq.h - - src/core/lib/promise/detail/promise_factory.h - - src/core/lib/promise/detail/promise_like.h - - src/core/lib/promise/detail/status.h - - src/core/lib/promise/detail/switch.h - - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - - src/core/lib/promise/loop.h - - src/core/lib/promise/map.h - - src/core/lib/promise/poll.h - - src/core/lib/promise/race.h - - src/core/lib/promise/seq.h - - src/core/lib/resource_quota/memory_quota.h - - src/core/lib/resource_quota/resource_quota.h - - src/core/lib/resource_quota/thread_quota.h - - src/core/lib/resource_quota/trace.h - - src/core/lib/slice/percent_encoding.h - - src/core/lib/slice/slice.h - - src/core/lib/slice/slice_internal.h - - src/core/lib/slice/slice_refcount.h - - src/core/lib/slice/slice_refcount_base.h - - src/core/lib/slice/slice_string_helpers.h - - src/core/lib/transport/bdp_estimator.h - - src/core/lib/transport/pid_controller.h - src: - - src/core/ext/transport/chttp2/transport/flow_control.cc - - src/core/ext/upb-generated/google/protobuf/any.upb.c - - src/core/ext/upb-generated/google/rpc/status.upb.c - - src/core/lib/debug/trace.cc - - src/core/lib/event_engine/memory_allocator.cc - - src/core/lib/gprpp/status_helper.cc - - src/core/lib/gprpp/time.cc - - src/core/lib/iomgr/combiner.cc - - src/core/lib/iomgr/error.cc - - src/core/lib/iomgr/exec_ctx.cc - - src/core/lib/iomgr/executor.cc - - src/core/lib/iomgr/iomgr_internal.cc - - src/core/lib/promise/activity.cc - - src/core/lib/resource_quota/memory_quota.cc - - src/core/lib/resource_quota/resource_quota.cc - - src/core/lib/resource_quota/thread_quota.cc - - src/core/lib/resource_quota/trace.cc - - src/core/lib/slice/percent_encoding.cc - - src/core/lib/slice/slice.cc - - src/core/lib/slice/slice_refcount.cc - - src/core/lib/slice/slice_string_helpers.cc - - src/core/lib/transport/bdp_estimator.cc - - src/core/lib/transport/pid_controller.cc - test/core/transport/chttp2/flow_control_test.cc deps: - - absl/meta:type_traits - - absl/status:statusor - - absl/types:variant - - absl/utility:utility - - gpr - - upb + - grpc_test_util - name: for_each_test gtest: true build: test diff --git a/config.m4 b/config.m4 index e669c0ae2be..5d3ecbc0bdb 100644 --- a/config.m4 +++ b/config.m4 @@ -703,6 +703,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ + src/core/lib/transport/byte_stream.cc \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/error_utils.cc \ src/core/lib/transport/handshaker.cc \ diff --git a/config.w32 b/config.w32 index fbb1edafdc0..a06bb24de93 100644 --- a/config.w32 +++ b/config.w32 @@ -669,6 +669,7 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\surface\\validate_metadata.cc " + "src\\core\\lib\\surface\\version.cc " + "src\\core\\lib\\transport\\bdp_estimator.cc " + + "src\\core\\lib\\transport\\byte_stream.cc " + "src\\core\\lib\\transport\\connectivity_state.cc " + "src\\core\\lib\\transport\\error_utils.cc " + "src\\core\\lib\\transport\\handshaker.cc " + diff --git a/doc/environment_variables.md b/doc/environment_variables.md index 59442f08d36..4f6201acf2b 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -167,6 +167,11 @@ some configuration as environment variables that can be set. channels (mostly due to idleness), so that the next RPC on this channel won't fail. Set to 0 to turn off the backup polls. +* GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL + if set, flow control will be effectively disabled. Max out all values and + assume the remote peer does the same. Thus we can ignore any flow control + bookkeeping, error checking, and decision making + * grpc_cfstream set to 1 to turn on CFStream experiment. With this experiment gRPC uses CFStream API to make TCP connections. The option is only available on iOS platform and when macro GRPC_CFSTREAM is defined. diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index b9517dd4c79..6c0d7cfd4e6 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -904,6 +904,7 @@ Pod::Spec.new do |s| 'src/core/lib/surface/server.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/transport/bdp_estimator.h', + 'src/core/lib/transport/byte_stream.h', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/handshaker.h', @@ -1724,6 +1725,7 @@ Pod::Spec.new do |s| 'src/core/lib/surface/server.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/transport/bdp_estimator.h', + 'src/core/lib/transport/byte_stream.h', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/handshaker.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 8dd7c2b4790..6afb2a748dc 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1507,6 +1507,8 @@ Pod::Spec.new do |s| 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', 'src/core/lib/transport/bdp_estimator.h', + 'src/core/lib/transport/byte_stream.cc', + 'src/core/lib/transport/byte_stream.h', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.cc', @@ -2325,6 +2327,7 @@ Pod::Spec.new do |s| 'src/core/lib/surface/server.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/transport/bdp_estimator.h', + 'src/core/lib/transport/byte_stream.h', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/handshaker.h', diff --git a/grpc.gemspec b/grpc.gemspec index fda6eff3309..5ab7231aede 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1420,6 +1420,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/surface/version.cc ) s.files += %w( src/core/lib/transport/bdp_estimator.cc ) s.files += %w( src/core/lib/transport/bdp_estimator.h ) + s.files += %w( src/core/lib/transport/byte_stream.cc ) + s.files += %w( src/core/lib/transport/byte_stream.h ) s.files += %w( src/core/lib/transport/connectivity_state.cc ) s.files += %w( src/core/lib/transport/connectivity_state.h ) s.files += %w( src/core/lib/transport/error_utils.cc ) diff --git a/grpc.gyp b/grpc.gyp index 2ba108cea9d..dd5c35a9914 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -992,6 +992,7 @@ 'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', + 'src/core/lib/transport/byte_stream.cc', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/handshaker.cc', @@ -1411,6 +1412,7 @@ 'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', + 'src/core/lib/transport/byte_stream.cc', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/handshaker.cc', diff --git a/include/grpc/event_engine/slice_buffer.h b/include/grpc/event_engine/slice_buffer.h index 3861717e7b6..609b6ea0841 100644 --- a/include/grpc/event_engine/slice_buffer.h +++ b/include/grpc/event_engine/slice_buffer.h @@ -54,19 +54,13 @@ class SliceBuffer { SliceBuffer(const SliceBuffer& other) = delete; SliceBuffer(SliceBuffer&& other) noexcept : slice_buffer_(other.slice_buffer_) { - grpc_slice_buffer_init(&slice_buffer_); + grpc_slice_buffer_reset_and_unref(&slice_buffer_); grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_); } /// Upon destruction, the underlying raw slice buffer is cleaned out and all /// slices are unreffed. ~SliceBuffer() { grpc_slice_buffer_destroy(&slice_buffer_); } - SliceBuffer& operator=(const SliceBuffer&) = delete; - SliceBuffer& operator=(SliceBuffer&& other) noexcept { - grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_); - return *this; - } - /// Appends a new slice into the SliceBuffer and makes an attempt to merge /// this slice with the last slice in the SliceBuffer. void Append(Slice slice); @@ -105,7 +99,7 @@ class SliceBuffer { size_t Length() { return slice_buffer_.length; } /// Return a pointer to the back raw grpc_slice_buffer - grpc_slice_buffer* c_slice_buffer() { return &slice_buffer_; } + grpc_slice_buffer* RawSliceBuffer() { return &slice_buffer_; } private: /// The backing raw slice buffer. diff --git a/package.xml b/package.xml index fcfc02942e0..ffc854f0731 100644 --- a/package.xml +++ b/package.xml @@ -1402,6 +1402,8 @@ + + diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index fbf8b4a99d9..6c3c3a2adfc 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -2942,7 +2942,7 @@ void ClientChannel::LoadBalancedCall::RecvMessageReady( gpr_log(GPR_INFO, "chand=%p lb_call=%p: got recv_message_ready: error=%s", self->chand_, self, grpc_error_std_string(error).c_str()); } - if (self->recv_message_->has_value()) { + if (*self->recv_message_ != nullptr) { self->call_attempt_tracer_->RecordReceivedMessage(**self->recv_message_); } Closure::Run(DEBUG_LOCATION, self->original_recv_message_ready_, diff --git a/src/core/ext/filters/client_channel/client_channel.h b/src/core/ext/filters/client_channel/client_channel.h index 1b8da1eb9fc..16efd351547 100644 --- a/src/core/ext/filters/client_channel/client_channel.h +++ b/src/core/ext/filters/client_channel/client_channel.h @@ -68,8 +68,8 @@ #include "src/core/lib/service_config/service_config_call_data.h" #include "src/core/lib/service_config/service_config_parser.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -534,7 +534,7 @@ class ClientChannel::LoadBalancedCall grpc_closure* original_recv_initial_metadata_ready_ = nullptr; // For intercepting recv_message_ready. - absl::optional* recv_message_ = nullptr; + OrphanablePtr* recv_message_ = nullptr; grpc_closure recv_message_ready_; grpc_closure* original_recv_message_ready_ = nullptr; diff --git a/src/core/ext/filters/client_channel/retry_filter.cc b/src/core/ext/filters/client_channel/retry_filter.cc index e3d705a9d1d..5f0633a2cf2 100644 --- a/src/core/ext/filters/client_channel/retry_filter.cc +++ b/src/core/ext/filters/client_channel/retry_filter.cc @@ -33,7 +33,6 @@ #include "absl/strings/string_view.h" #include "absl/strings/strip.h" #include "absl/types/optional.h" -#include "absl/utility/utility.h" #include #include @@ -52,8 +51,8 @@ #include "src/core/lib/channel/status_util.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gpr/useful.h" -#include "src/core/lib/gprpp/construct_destruct.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" @@ -67,8 +66,8 @@ #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/service_config/service_config.h" #include "src/core/lib/service_config/service_config_call_data.h" -#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_refcount.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -457,6 +456,9 @@ class RetryFilter::CallData { grpc_transport_stream_op_batch_payload batch_payload_; // For send_initial_metadata. grpc_metadata_batch send_initial_metadata_{calld_->arena_}; + // For send_message. + // TODO(roth): Restructure this to eliminate use of ManualConstructor. + ManualConstructor send_message_; // For send_trailing_metadata. grpc_metadata_batch send_trailing_metadata_{calld_->arena_}; // For intercepting recv_initial_metadata. @@ -465,8 +467,7 @@ class RetryFilter::CallData { bool trailing_metadata_available_ = false; // For intercepting recv_message. grpc_closure recv_message_ready_; - absl::optional recv_message_; - uint32_t recv_message_flags_; + OrphanablePtr recv_message_; // For intercepting recv_trailing_metadata. grpc_metadata_batch recv_trailing_metadata_{calld_->arena_}; grpc_transport_stream_stats collect_stats_; @@ -627,11 +628,11 @@ class RetryFilter::CallData { // Note: We inline the cache for the first 3 send_message ops and use // dynamic allocation after that. This number was essentially picked // at random; it could be changed in the future to tune performance. - struct CachedSendMessage { - SliceBuffer* slices; - uint32_t flags; - }; - absl::InlinedVector send_messages_; + // TODO(roth): As part of implementing hedging, we may need some + // synchronization here, since ByteStreamCache does not provide any + // synchronization, so it's not safe to have multiple + // CachingByteStreams read from the same ByteStreamCache concurrently. + absl::InlinedVector send_messages_; // send_trailing_metadata bool seen_send_trailing_metadata_ = false; grpc_metadata_batch send_trailing_metadata_{arena_}; @@ -1510,8 +1511,6 @@ void RetryFilter::CallData::CallAttempt::BatchData:: // Return payload. *pending->batch->payload->recv_message.recv_message = std::move(call_attempt_->recv_message_); - *pending->batch->payload->recv_message.flags = - call_attempt_->recv_message_flags_; // Update bookkeeping. // Note: Need to do this before invoking the callback, since invoking // the callback will result in yielding the call combiner. @@ -1556,7 +1555,7 @@ void RetryFilter::CallData::CallAttempt::BatchData::RecvMessageReady( // the recv_trailing_metadata_ready callback, then defer propagating this // callback back to the surface. We can evaluate whether to retry when // recv_trailing_metadata comes back. - if (GPR_UNLIKELY((!call_attempt->recv_message_.has_value() || + if (GPR_UNLIKELY((call_attempt->recv_message_ == nullptr || error != GRPC_ERROR_NONE) && !call_attempt->completed_recv_trailing_metadata_)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_retry_trace)) { @@ -2030,12 +2029,13 @@ void RetryFilter::CallData::CallAttempt::BatchData:: calld->chand_, calld, call_attempt_.get(), call_attempt_->started_send_message_count_); } - CachedSendMessage cache = + ByteStreamCache* cache = calld->send_messages_[call_attempt_->started_send_message_count_]; ++call_attempt_->started_send_message_count_; + call_attempt_->send_message_.Init(cache); batch_.send_message = true; - batch_.payload->send_message.send_message = cache.slices; - batch_.payload->send_message.flags = cache.flags; + batch_.payload->send_message.send_message.reset( + call_attempt_->send_message_.get()); } void RetryFilter::CallData::CallAttempt::BatchData:: @@ -2072,7 +2072,6 @@ void RetryFilter::CallData::CallAttempt::BatchData:: ++call_attempt_->started_recv_message_count_; batch_.recv_message = true; batch_.payload->recv_message.recv_message = &call_attempt_->recv_message_; - batch_.payload->recv_message.flags = &call_attempt_->recv_message_flags_; batch_.payload->recv_message.call_failed_before_recv_message = nullptr; GRPC_CLOSURE_INIT(&call_attempt_->recv_message_ready_, RecvMessageReady, this, grpc_schedule_on_exec_ctx); @@ -2373,9 +2372,9 @@ void RetryFilter::CallData::MaybeCacheSendOpsForBatch(PendingBatch* pending) { } // Set up cache for send_message ops. if (batch->send_message) { - SliceBuffer* cache = arena_->New(std::move( - *absl::exchange(batch->payload->send_message.send_message, nullptr))); - send_messages_.push_back({cache, batch->payload->send_message.flags}); + ByteStreamCache* cache = arena_->New( + std::move(batch->payload->send_message.send_message)); + send_messages_.push_back(cache); } // Save metadata batch for send_trailing_metadata ops. if (batch->send_trailing_metadata) { @@ -2395,13 +2394,14 @@ void RetryFilter::CallData::FreeCachedSendInitialMetadata() { } void RetryFilter::CallData::FreeCachedSendMessage(size_t idx) { - if (send_messages_[idx].slices != nullptr) { + if (send_messages_[idx] != nullptr) { if (GRPC_TRACE_FLAG_ENABLED(grpc_retry_trace)) { gpr_log(GPR_INFO, "chand=%p calld=%p: destroying send_messages[%" PRIuPTR "]", chand_, this, idx); } - Destruct(absl::exchange(send_messages_[idx].slices, nullptr)); + send_messages_[idx]->Destroy(); + send_messages_[idx] = nullptr; } } @@ -2465,7 +2465,7 @@ RetryFilter::CallData::PendingBatch* RetryFilter::CallData::PendingBatchesAdd( if (batch->send_message) { pending_send_message_ = true; bytes_buffered_for_retry_ += - batch->payload->send_message.send_message->Length(); + batch->payload->send_message.send_message->length(); } if (batch->send_trailing_metadata) { pending_send_trailing_metadata_ = true; diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 0bb87d784cb..826954625dd 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "absl/status/statusor.h" diff --git a/src/core/ext/filters/client_channel/subchannel_stream_client.cc b/src/core/ext/filters/client_channel/subchannel_stream_client.cc index 360a64bc32e..e2f35276e48 100644 --- a/src/core/ext/filters/client_channel/subchannel_stream_client.cc +++ b/src/core/ext/filters/client_channel/subchannel_stream_client.cc @@ -20,11 +20,15 @@ #include #include +#include +#include #include #include +#include #include +#include #include #include "src/core/lib/gpr/time_precise.h" @@ -34,6 +38,7 @@ #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/resource_quota/api.h" #include "src/core/lib/resource_quota/resource_quota.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/transport/error_utils.h" #define SUBCHANNEL_STREAM_INITIAL_CONNECT_BACKOFF_SECONDS 1 @@ -247,9 +252,14 @@ void SubchannelStreamClient::CallState::StartCallLocked() { payload_.send_initial_metadata.peer_string = nullptr; batch_.send_initial_metadata = true; // Add send_message op. - send_message_.Append(Slice( - subchannel_stream_client_->event_handler_->EncodeSendMessageLocked())); - payload_.send_message.send_message = &send_message_; + grpc_slice request_slice = + subchannel_stream_client_->event_handler_->EncodeSendMessageLocked(); + grpc_slice_buffer slice_buffer; + grpc_slice_buffer_init(&slice_buffer); + grpc_slice_buffer_add(&slice_buffer, request_slice); + send_message_.emplace(&slice_buffer, 0); + grpc_slice_buffer_destroy_internal(&slice_buffer); + payload_.send_message.send_message.reset(&*send_message_); batch_.send_message = true; // Add send_trailing_metadata op. payload_.send_trailing_metadata.send_trailing_metadata = @@ -364,18 +374,42 @@ void SubchannelStreamClient::CallState::RecvInitialMetadataReady( self->call_->Unref(DEBUG_LOCATION, "recv_initial_metadata_ready"); } -void SubchannelStreamClient::CallState::RecvMessageReady() { - if (!recv_message_.has_value()) { +void SubchannelStreamClient::CallState::DoneReadingRecvMessage( + grpc_error_handle error) { + recv_message_.reset(); + if (error != GRPC_ERROR_NONE) { + GRPC_ERROR_UNREF(error); + Cancel(); + grpc_slice_buffer_destroy_internal(&recv_message_buffer_); call_->Unref(DEBUG_LOCATION, "recv_message_ready"); return; } + // Concatenate the slices to form a single string. + std::unique_ptr recv_message_deleter; + uint8_t* recv_message; + if (recv_message_buffer_.count == 1) { + recv_message = GRPC_SLICE_START_PTR(recv_message_buffer_.slices[0]); + } else { + recv_message = + static_cast(gpr_malloc(recv_message_buffer_.length)); + recv_message_deleter.reset(recv_message); + size_t offset = 0; + for (size_t i = 0; i < recv_message_buffer_.count; ++i) { + memcpy(recv_message + offset, + GRPC_SLICE_START_PTR(recv_message_buffer_.slices[i]), + GRPC_SLICE_LENGTH(recv_message_buffer_.slices[i])); + offset += GRPC_SLICE_LENGTH(recv_message_buffer_.slices[i]); + } + } // Report payload. { MutexLock lock(&subchannel_stream_client_->mu_); if (subchannel_stream_client_->event_handler_ != nullptr) { + absl::string_view serialized_message( + reinterpret_cast(recv_message), recv_message_buffer_.length); absl::Status status = subchannel_stream_client_->event_handler_->RecvMessageReadyLocked( - subchannel_stream_client_.get(), recv_message_->JoinIntoString()); + subchannel_stream_client_.get(), serialized_message); if (!status.ok()) { if (GPR_UNLIKELY(subchannel_stream_client_->tracer_ != nullptr)) { gpr_log(GPR_INFO, @@ -390,7 +424,7 @@ void SubchannelStreamClient::CallState::RecvMessageReady() { } } seen_response_.store(true, std::memory_order_release); - recv_message_.reset(); + grpc_slice_buffer_destroy_internal(&recv_message_buffer_); // Start another recv_message batch. // This re-uses the ref we're holding. // Note: Can't just reuse batch_ here, since we don't know that all @@ -404,11 +438,62 @@ void SubchannelStreamClient::CallState::RecvMessageReady() { StartBatch(&recv_message_batch_); } +grpc_error_handle +SubchannelStreamClient::CallState::PullSliceFromRecvMessage() { + grpc_slice slice; + grpc_error_handle error = recv_message_->Pull(&slice); + if (error == GRPC_ERROR_NONE) { + grpc_slice_buffer_add(&recv_message_buffer_, slice); + } + return error; +} + +void SubchannelStreamClient::CallState::ContinueReadingRecvMessage() { + while (recv_message_->Next(SIZE_MAX, &recv_message_ready_)) { + grpc_error_handle error = PullSliceFromRecvMessage(); + if (error != GRPC_ERROR_NONE) { + DoneReadingRecvMessage(error); + return; + } + if (recv_message_buffer_.length == recv_message_->length()) { + DoneReadingRecvMessage(GRPC_ERROR_NONE); + break; + } + } +} + +void SubchannelStreamClient::CallState::OnByteStreamNext( + void* arg, grpc_error_handle error) { + auto* self = static_cast(arg); + if (error != GRPC_ERROR_NONE) { + self->DoneReadingRecvMessage(GRPC_ERROR_REF(error)); + return; + } + error = self->PullSliceFromRecvMessage(); + if (error != GRPC_ERROR_NONE) { + self->DoneReadingRecvMessage(error); + return; + } + if (self->recv_message_buffer_.length == self->recv_message_->length()) { + self->DoneReadingRecvMessage(GRPC_ERROR_NONE); + } else { + self->ContinueReadingRecvMessage(); + } +} + void SubchannelStreamClient::CallState::RecvMessageReady( void* arg, grpc_error_handle /*error*/) { auto* self = static_cast(arg); GRPC_CALL_COMBINER_STOP(&self->call_combiner_, "recv_message_ready"); - self->RecvMessageReady(); + if (self->recv_message_ == nullptr) { + self->call_->Unref(DEBUG_LOCATION, "recv_message_ready"); + return; + } + grpc_slice_buffer_init(&self->recv_message_buffer_); + GRPC_CLOSURE_INIT(&self->recv_message_ready_, OnByteStreamNext, self, + grpc_schedule_on_exec_ctx); + self->ContinueReadingRecvMessage(); + // Ref will continue to be held until we finish draining the byte stream. } void SubchannelStreamClient::CallState::RecvTrailingMetadataReady( diff --git a/src/core/ext/filters/client_channel/subchannel_stream_client.h b/src/core/ext/filters/client_channel/subchannel_stream_client.h index 32d8f00fd69..08f03f7586d 100644 --- a/src/core/ext/filters/client_channel/subchannel_stream_client.h +++ b/src/core/ext/filters/client_channel/subchannel_stream_client.h @@ -46,7 +46,7 @@ #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/memory_quota.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -132,8 +132,6 @@ class SubchannelStreamClient void CallEndedLocked(bool retry) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&subchannel_stream_client_->mu_); - void RecvMessageReady(); - static void OnComplete(void* arg, grpc_error_handle error); static void RecvInitialMetadataReady(void* arg, grpc_error_handle error); static void RecvMessageReady(void* arg, grpc_error_handle error); @@ -141,6 +139,11 @@ class SubchannelStreamClient static void StartCancel(void* arg, grpc_error_handle error); static void OnCancelComplete(void* arg, grpc_error_handle error); + static void OnByteStreamNext(void* arg, grpc_error_handle error); + void ContinueReadingRecvMessage(); + grpc_error_handle PullSliceFromRecvMessage(); + void DoneReadingRecvMessage(grpc_error_handle error); + static void AfterCallStackDestruction(void* arg, grpc_error_handle error); RefCountedPtr subchannel_stream_client_; @@ -166,7 +169,7 @@ class SubchannelStreamClient grpc_metadata_batch send_initial_metadata_; // send_message - SliceBuffer send_message_; + absl::optional send_message_; // send_trailing_metadata grpc_metadata_batch send_trailing_metadata_; @@ -176,8 +179,9 @@ class SubchannelStreamClient grpc_closure recv_initial_metadata_ready_; // recv_message - absl::optional recv_message_; + OrphanablePtr recv_message_; grpc_closure recv_message_ready_; + grpc_slice_buffer recv_message_buffer_; std::atomic seen_response_{false}; // True if the cancel_stream batch has been started. diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.cc b/src/core/ext/filters/http/message_compress/message_compress_filter.cc index acfafb3f49c..6387050e087 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.cc +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.cc @@ -23,26 +23,32 @@ #include #include +#include #include +#include #include "absl/meta/type_traits.h" #include "absl/types/optional.h" -#include "absl/utility/utility.h" #include #include #include +#include +#include #include #include "src/core/lib/compression/compression_internal.h" #include "src/core/lib/compression/message_compress.h" #include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/profiling/timers.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/call.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -100,40 +106,66 @@ class CallData { channeld->default_compression_algorithm()))) { compression_algorithm_ = channeld->default_compression_algorithm(); } - GRPC_CLOSURE_INIT(&forward_send_message_batch_in_call_combiner_, - ForwardSendMessageBatch, elem, grpc_schedule_on_exec_ctx); + GRPC_CLOSURE_INIT(&start_send_message_batch_in_call_combiner_, + StartSendMessageBatch, elem, grpc_schedule_on_exec_ctx); } - ~CallData() { GRPC_ERROR_UNREF(cancel_error_); } + ~CallData() { + if (state_initialized_) { + grpc_slice_buffer_destroy_internal(&slices_); + } + GRPC_ERROR_UNREF(cancel_error_); + } void CompressStartTransportStreamOpBatch( grpc_call_element* elem, grpc_transport_stream_op_batch* batch); private: bool SkipMessageCompression(); - void FinishSendMessage(grpc_call_element* elem); + void InitializeState(grpc_call_element* elem); void ProcessSendInitialMetadata(grpc_call_element* elem, grpc_metadata_batch* initial_metadata); // Methods for processing a send_message batch + static void StartSendMessageBatch(void* elem_arg, grpc_error_handle unused); + static void OnSendMessageNextDone(void* elem_arg, grpc_error_handle error); + grpc_error_handle PullSliceFromSendMessage(); + void ContinueReadingSendMessage(grpc_call_element* elem); + void FinishSendMessage(grpc_call_element* elem); + void SendMessageBatchContinue(grpc_call_element* elem); static void FailSendMessageBatchInCallCombiner(void* calld_arg, grpc_error_handle error); - static void ForwardSendMessageBatch(void* elem_arg, grpc_error_handle unused); + + static void SendMessageOnComplete(void* calld_arg, grpc_error_handle error); grpc_core::CallCombiner* call_combiner_; grpc_compression_algorithm compression_algorithm_ = GRPC_COMPRESS_NONE; grpc_error_handle cancel_error_ = GRPC_ERROR_NONE; grpc_transport_stream_op_batch* send_message_batch_ = nullptr; bool seen_initial_metadata_ = false; - grpc_closure forward_send_message_batch_in_call_combiner_; + /* Set to true, if the fields below are initialized. */ + bool state_initialized_ = false; + grpc_closure start_send_message_batch_in_call_combiner_; + /* The fields below are only initialized when we compress the payload. + * Keep them at the bottom of the struct, so they don't pollute the + * cache-lines. */ + grpc_slice_buffer slices_; /**< Buffers up input slices to be compressed */ + // Allocate space for the replacement stream + std::aligned_storage::type + replacement_stream_; + grpc_closure* original_send_message_on_complete_ = nullptr; + grpc_closure send_message_on_complete_; + grpc_closure on_send_message_next_done_; }; // Returns true if we should skip message compression for the current message. bool CallData::SkipMessageCompression() { // If the flags of this message indicate that it shouldn't be compressed, we // skip message compression. - uint32_t flags = send_message_batch_->payload->send_message.flags; + uint32_t flags = + send_message_batch_->payload->send_message.send_message->flags(); if (flags & (GRPC_WRITE_NO_COMPRESS | GRPC_WRITE_INTERNAL_COMPRESS)) { return true; } @@ -142,6 +174,16 @@ bool CallData::SkipMessageCompression() { return compression_algorithm_ == GRPC_COMPRESS_NONE; } +void CallData::InitializeState(grpc_call_element* elem) { + GPR_DEBUG_ASSERT(!state_initialized_); + state_initialized_ = true; + grpc_slice_buffer_init(&slices_); + GRPC_CLOSURE_INIT(&send_message_on_complete_, SendMessageOnComplete, this, + grpc_schedule_on_exec_ctx); + GRPC_CLOSURE_INIT(&on_send_message_next_done_, OnSendMessageNextDone, elem, + grpc_schedule_on_exec_ctx); +} + void CallData::ProcessSendInitialMetadata( grpc_call_element* elem, grpc_metadata_batch* initial_metadata) { ChannelData* channeld = static_cast(elem->channel_data); @@ -154,6 +196,7 @@ void CallData::ProcessSendInitialMetadata( break; case GRPC_COMPRESS_DEFLATE: case GRPC_COMPRESS_GZIP: + InitializeState(elem); initial_metadata->Set(grpc_core::GrpcEncodingMetadata(), compression_algorithm_); break; @@ -165,46 +208,68 @@ void CallData::ProcessSendInitialMetadata( channeld->enabled_compression_algorithms()); } +void CallData::SendMessageOnComplete(void* calld_arg, grpc_error_handle error) { + CallData* calld = static_cast(calld_arg); + grpc_slice_buffer_reset_and_unref_internal(&calld->slices_); + grpc_core::Closure::Run(DEBUG_LOCATION, + calld->original_send_message_on_complete_, + GRPC_ERROR_REF(error)); +} + +void CallData::SendMessageBatchContinue(grpc_call_element* elem) { + // Note: The call to grpc_call_next_op() results in yielding the + // call combiner, so we need to clear send_message_batch_ before we do that. + grpc_transport_stream_op_batch* send_message_batch = send_message_batch_; + send_message_batch_ = nullptr; + grpc_call_next_op(elem, send_message_batch); +} + void CallData::FinishSendMessage(grpc_call_element* elem) { + GPR_DEBUG_ASSERT(compression_algorithm_ != GRPC_COMPRESS_NONE); // Compress the data if appropriate. - if (!SkipMessageCompression()) { - grpc_core::SliceBuffer tmp; - uint32_t& send_flags = send_message_batch_->payload->send_message.flags; - grpc_core::SliceBuffer* payload = - send_message_batch_->payload->send_message.send_message; - bool did_compress = - grpc_msg_compress(compression_algorithm_, payload->c_slice_buffer(), - tmp.c_slice_buffer()); - if (did_compress) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { - const char* algo_name; - const size_t before_size = payload->Length(); - const size_t after_size = tmp.Length(); - const float savings_ratio = 1.0f - static_cast(after_size) / - static_cast(before_size); - GPR_ASSERT(grpc_compression_algorithm_name(compression_algorithm_, - &algo_name)); - gpr_log(GPR_INFO, - "Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR - " bytes (%.2f%% savings)", - algo_name, before_size, after_size, 100 * savings_ratio); - } - tmp.Swap(payload); - send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; - } else { - if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { - const char* algo_name; - GPR_ASSERT(grpc_compression_algorithm_name(compression_algorithm_, - &algo_name)); - gpr_log( - GPR_INFO, - "Algorithm '%s' enabled but decided not to compress. Input size: " - "%" PRIuPTR, - algo_name, payload->Length()); - } + grpc_slice_buffer tmp; + grpc_slice_buffer_init(&tmp); + uint32_t send_flags = + send_message_batch_->payload->send_message.send_message->flags(); + bool did_compress = grpc_msg_compress(compression_algorithm_, &slices_, &tmp); + if (did_compress) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { + const char* algo_name; + const size_t before_size = slices_.length; + const size_t after_size = tmp.length; + const float savings_ratio = 1.0f - static_cast(after_size) / + static_cast(before_size); + GPR_ASSERT( + grpc_compression_algorithm_name(compression_algorithm_, &algo_name)); + gpr_log(GPR_INFO, + "Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR + " bytes (%.2f%% savings)", + algo_name, before_size, after_size, 100 * savings_ratio); + } + grpc_slice_buffer_swap(&slices_, &tmp); + send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; + } else { + if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { + const char* algo_name; + GPR_ASSERT( + grpc_compression_algorithm_name(compression_algorithm_, &algo_name)); + gpr_log(GPR_INFO, + "Algorithm '%s' enabled but decided not to compress. Input size: " + "%" PRIuPTR, + algo_name, slices_.length); } } - grpc_call_next_op(elem, absl::exchange(send_message_batch_, nullptr)); + grpc_slice_buffer_destroy_internal(&tmp); + // Swap out the original byte stream with our new one and send the + // batch down. + new (&replacement_stream_) + grpc_core::SliceBufferByteStream(&slices_, send_flags); + send_message_batch_->payload->send_message.send_message.reset( + reinterpret_cast( + &replacement_stream_)); + original_send_message_on_complete_ = send_message_batch_->on_complete; + send_message_batch_->on_complete = &send_message_on_complete_; + SendMessageBatchContinue(elem); } void CallData::FailSendMessageBatchInCallCombiner(void* calld_arg, @@ -218,11 +283,78 @@ void CallData::FailSendMessageBatchInCallCombiner(void* calld_arg, } } -void CallData::ForwardSendMessageBatch(void* elem_arg, - grpc_error_handle /*unused*/) { +// Pulls a slice from the send_message byte stream and adds it to slices_. +grpc_error_handle CallData::PullSliceFromSendMessage() { + grpc_slice incoming_slice; + grpc_error_handle error = + send_message_batch_->payload->send_message.send_message->Pull( + &incoming_slice); + if (error == GRPC_ERROR_NONE) { + grpc_slice_buffer_add(&slices_, incoming_slice); + } + return error; +} + +// Reads as many slices as possible from the send_message byte stream. +// If all data has been read, invokes FinishSendMessage(). Otherwise, +// an async call to ByteStream::Next() has been started, which will +// eventually result in calling OnSendMessageNextDone(). +void CallData::ContinueReadingSendMessage(grpc_call_element* elem) { + if (slices_.length == + send_message_batch_->payload->send_message.send_message->length()) { + FinishSendMessage(elem); + return; + } + while (send_message_batch_->payload->send_message.send_message->Next( + ~static_cast(0), &on_send_message_next_done_)) { + grpc_error_handle error = PullSliceFromSendMessage(); + if (error != GRPC_ERROR_NONE) { + // Closure callback; does not take ownership of error. + FailSendMessageBatchInCallCombiner(this, error); + GRPC_ERROR_UNREF(error); + return; + } + if (slices_.length == + send_message_batch_->payload->send_message.send_message->length()) { + FinishSendMessage(elem); + break; + } + } +} + +// Async callback for ByteStream::Next(). +void CallData::OnSendMessageNextDone(void* elem_arg, grpc_error_handle error) { grpc_call_element* elem = static_cast(elem_arg); CallData* calld = static_cast(elem->call_data); - calld->FinishSendMessage(elem); + if (error != GRPC_ERROR_NONE) { + // Closure callback; does not take ownership of error. + FailSendMessageBatchInCallCombiner(calld, error); + return; + } + error = calld->PullSliceFromSendMessage(); + if (error != GRPC_ERROR_NONE) { + // Closure callback; does not take ownership of error. + FailSendMessageBatchInCallCombiner(calld, error); + GRPC_ERROR_UNREF(error); + return; + } + if (calld->slices_.length == calld->send_message_batch_->payload->send_message + .send_message->length()) { + calld->FinishSendMessage(elem); + } else { + calld->ContinueReadingSendMessage(elem); + } +} + +void CallData::StartSendMessageBatch(void* elem_arg, + grpc_error_handle /*unused*/) { + grpc_call_element* elem = static_cast(elem_arg); + CallData* calld = static_cast(elem->call_data); + if (calld->SkipMessageCompression()) { + calld->SendMessageBatchContinue(elem); + } else { + calld->ContinueReadingSendMessage(elem); + } } void CallData::CompressStartTransportStreamOpBatch( @@ -239,6 +371,9 @@ void CallData::CompressStartTransportStreamOpBatch( GRPC_CLOSURE_CREATE(FailSendMessageBatchInCallCombiner, this, grpc_schedule_on_exec_ctx), GRPC_ERROR_REF(cancel_error_), "failing send_message op"); + } else { + send_message_batch_->payload->send_message.send_message->Shutdown( + GRPC_ERROR_REF(cancel_error_)); } } } else if (cancel_error_ != GRPC_ERROR_NONE) { @@ -259,7 +394,7 @@ void CallData::CompressStartTransportStreamOpBatch( // the call stack) will release the call combiner for each batch it sees. if (send_message_batch_ != nullptr) { GRPC_CALL_COMBINER_START( - call_combiner_, &forward_send_message_batch_in_call_combiner_, + call_combiner_, &start_send_message_batch_in_call_combiner_, GRPC_ERROR_NONE, "starting send_message after send_initial_metadata"); } } @@ -275,7 +410,7 @@ void CallData::CompressStartTransportStreamOpBatch( call_combiner_, "send_message batch pending send_initial_metadata"); return; } - FinishSendMessage(elem); + StartSendMessageBatch(elem, GRPC_ERROR_NONE); } else { // Pass control down the stack. grpc_call_next_op(elem, batch); diff --git a/src/core/ext/filters/http/message_compress/message_decompress_filter.cc b/src/core/ext/filters/http/message_compress/message_decompress_filter.cc index 3b26a75472c..9bb3da48a8a 100644 --- a/src/core/ext/filters/http/message_compress/message_decompress_filter.cc +++ b/src/core/ext/filters/http/message_compress/message_decompress_filter.cc @@ -23,13 +23,17 @@ #include #include +#include #include +#include #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/types/optional.h" #include +#include +#include #include #include @@ -37,11 +41,13 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/compression/message_compress.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/profiling/timers.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -76,6 +82,9 @@ class CallData { OnRecvInitialMetadataReady, this, grpc_schedule_on_exec_ctx); // Initialize state for recv_message_ready callback + grpc_slice_buffer_init(&recv_slices_); + GRPC_CLOSURE_INIT(&on_recv_message_next_done_, OnRecvMessageNextDone, this, + grpc_schedule_on_exec_ctx); GRPC_CLOSURE_INIT(&on_recv_message_ready_, OnRecvMessageReady, this, grpc_schedule_on_exec_ctx); // Initialize state for recv_trailing_metadata_ready callback @@ -92,6 +101,8 @@ class CallData { } } + ~CallData() { grpc_slice_buffer_destroy_internal(&recv_slices_); } + void DecompressStartTransportStreamOpBatch( grpc_call_element* elem, grpc_transport_stream_op_batch* batch); @@ -101,6 +112,10 @@ class CallData { // Methods for processing a receive message event void MaybeResumeOnRecvMessageReady(); static void OnRecvMessageReady(void* arg, grpc_error_handle error); + static void OnRecvMessageNextDone(void* arg, grpc_error_handle error); + grpc_error_handle PullSliceFromRecvMessage(); + void ContinueReadingRecvMessage(); + void FinishRecvMessage(); void ContinueRecvMessageReadyCallback(grpc_error_handle error); // Methods for processing a recv_trailing_metadata event @@ -118,10 +133,17 @@ class CallData { bool seen_recv_message_ready_ = false; int max_recv_message_length_; grpc_compression_algorithm algorithm_ = GRPC_COMPRESS_NONE; - absl::optional* recv_message_ = nullptr; - uint32_t* recv_message_flags_ = nullptr; grpc_closure on_recv_message_ready_; grpc_closure* original_recv_message_ready_ = nullptr; + grpc_closure on_recv_message_next_done_; + OrphanablePtr* recv_message_ = nullptr; + // recv_slices_ holds the slices read from the original recv_message stream. + // It is initialized during construction and reset when a new stream is + // created using it. + grpc_slice_buffer recv_slices_; + std::aligned_storage::type + recv_replacement_stream_; // Fields for handling recv_trailing_metadata_ready callback bool seen_recv_trailing_metadata_ready_ = false; grpc_closure on_recv_trailing_metadata_ready_; @@ -165,46 +187,101 @@ void CallData::OnRecvMessageReady(void* arg, grpc_error_handle error) { if (calld->algorithm_ != GRPC_COMPRESS_NONE) { // recv_message can be NULL if trailing metadata is received instead of // message, or it's possible that the message was not compressed. - if (!calld->recv_message_->has_value() || - (*calld->recv_message_)->Length() == 0 || - ((*calld->recv_message_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) == 0)) { + if (*calld->recv_message_ == nullptr || + (*calld->recv_message_)->length() == 0 || + ((*calld->recv_message_)->flags() & GRPC_WRITE_INTERNAL_COMPRESS) == + 0) { return calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_NONE); } if (calld->max_recv_message_length_ >= 0 && - (*calld->recv_message_)->Length() > + (*calld->recv_message_)->length() > static_cast(calld->max_recv_message_length_)) { GPR_DEBUG_ASSERT(calld->error_ == GRPC_ERROR_NONE); calld->error_ = grpc_error_set_int( GRPC_ERROR_CREATE_FROM_CPP_STRING( absl::StrFormat("Received message larger than max (%u vs. %d)", - (*calld->recv_message_)->Length(), + (*calld->recv_message_)->length(), calld->max_recv_message_length_)), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED); return calld->ContinueRecvMessageReadyCallback( GRPC_ERROR_REF(calld->error_)); } - SliceBuffer decompressed_slices; - if (grpc_msg_decompress(calld->algorithm_, - (*calld->recv_message_)->c_slice_buffer(), - decompressed_slices.c_slice_buffer()) == 0) { - GPR_DEBUG_ASSERT(calld->error_ == GRPC_ERROR_NONE); - calld->error_ = GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat( - "Unexpected error decompressing data for algorithm with " - "enum value ", - calld->algorithm_)); - } else { - *calld->recv_message_flags_ = - (*calld->recv_message_flags_ & (~GRPC_WRITE_INTERNAL_COMPRESS)) | - GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED; - (*calld->recv_message_)->Swap(&decompressed_slices); - } - return calld->ContinueRecvMessageReadyCallback( - GRPC_ERROR_REF(calld->error_)); + grpc_slice_buffer_destroy_internal(&calld->recv_slices_); + grpc_slice_buffer_init(&calld->recv_slices_); + return calld->ContinueReadingRecvMessage(); } } calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_REF(error)); } +void CallData::ContinueReadingRecvMessage() { + while ((*recv_message_) + ->Next((*recv_message_)->length() - recv_slices_.length, + &on_recv_message_next_done_)) { + grpc_error_handle error = PullSliceFromRecvMessage(); + if (error != GRPC_ERROR_NONE) { + return ContinueRecvMessageReadyCallback(error); + } + // We have read the entire message. + if (recv_slices_.length == (*recv_message_)->length()) { + return FinishRecvMessage(); + } + } +} + +grpc_error_handle CallData::PullSliceFromRecvMessage() { + grpc_slice incoming_slice; + grpc_error_handle error = (*recv_message_)->Pull(&incoming_slice); + if (error == GRPC_ERROR_NONE) { + grpc_slice_buffer_add(&recv_slices_, incoming_slice); + } + return error; +} + +void CallData::OnRecvMessageNextDone(void* arg, grpc_error_handle error) { + CallData* calld = static_cast(arg); + if (error != GRPC_ERROR_NONE) { + return calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_REF(error)); + } + error = calld->PullSliceFromRecvMessage(); + if (error != GRPC_ERROR_NONE) { + return calld->ContinueRecvMessageReadyCallback(error); + } + if (calld->recv_slices_.length == (*calld->recv_message_)->length()) { + calld->FinishRecvMessage(); + } else { + calld->ContinueReadingRecvMessage(); + } +} + +void CallData::FinishRecvMessage() { + grpc_slice_buffer decompressed_slices; + grpc_slice_buffer_init(&decompressed_slices); + if (grpc_msg_decompress(algorithm_, &recv_slices_, &decompressed_slices) == + 0) { + GPR_DEBUG_ASSERT(error_ == GRPC_ERROR_NONE); + error_ = GRPC_ERROR_CREATE_FROM_CPP_STRING( + absl::StrCat("Unexpected error decompressing data for algorithm with " + "enum value ", + algorithm_)); + grpc_slice_buffer_destroy_internal(&decompressed_slices); + } else { + uint32_t recv_flags = + ((*recv_message_)->flags() & (~GRPC_WRITE_INTERNAL_COMPRESS)) | + GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED; + // Swap out the original receive byte stream with our new one and send the + // batch down. + // Initializing recv_replacement_stream_ with decompressed_slices removes + // all the slices from decompressed_slices leaving it empty. + new (&recv_replacement_stream_) + SliceBufferByteStream(&decompressed_slices, recv_flags); + recv_message_->reset( + reinterpret_cast(&recv_replacement_stream_)); + recv_message_ = nullptr; + } + ContinueRecvMessageReadyCallback(GRPC_ERROR_REF(error_)); +} + void CallData::ContinueRecvMessageReadyCallback(grpc_error_handle error) { MaybeResumeOnRecvTrailingMetadataReady(); // The surface will clean up the receiving stream if there is an error. @@ -256,7 +333,6 @@ void CallData::DecompressStartTransportStreamOpBatch( // Handle recv_message if (batch->recv_message) { recv_message_ = batch->payload->recv_message.recv_message; - recv_message_flags_ = batch->payload->recv_message.flags; original_recv_message_ready_ = batch->payload->recv_message.recv_message_ready; batch->payload->recv_message.recv_message_ready = &on_recv_message_ready_; diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc index 227396cab9a..b5145a6d4fb 100644 --- a/src/core/ext/filters/message_size/message_size_filter.cc +++ b/src/core/ext/filters/message_size/message_size_filter.cc @@ -39,12 +39,13 @@ #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/service_config/service_config_call_data.h" -#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/channel_init.h" #include "src/core/lib/surface/channel_stack_type.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/transport.h" static void recv_message_ready(void* user_data, grpc_error_handle error); @@ -193,7 +194,7 @@ struct call_data { // The error caused by a message that is too large, or GRPC_ERROR_NONE grpc_error_handle error = GRPC_ERROR_NONE; // Used by recv_message_ready. - absl::optional* recv_message = nullptr; + grpc_core::OrphanablePtr* recv_message = nullptr; // Original recv_message_ready callback, invoked after our own. grpc_closure* next_recv_message_ready = nullptr; // Original recv_trailing_metadata callback, invoked after our own. @@ -209,13 +210,13 @@ struct call_data { static void recv_message_ready(void* user_data, grpc_error_handle error) { grpc_call_element* elem = static_cast(user_data); call_data* calld = static_cast(elem->call_data); - if (calld->recv_message->has_value() && calld->limits.max_recv_size >= 0 && - (*calld->recv_message)->Length() > + if (*calld->recv_message != nullptr && calld->limits.max_recv_size >= 0 && + (*calld->recv_message)->length() > static_cast(calld->limits.max_recv_size)) { grpc_error_handle new_error = grpc_error_set_int( GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( "Received message larger than max (%u vs. %d)", - (*calld->recv_message)->Length(), calld->limits.max_recv_size)), + (*calld->recv_message)->length(), calld->limits.max_recv_size)), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED); error = grpc_error_add_child(GRPC_ERROR_REF(error), new_error); GRPC_ERROR_UNREF(calld->error); @@ -268,13 +269,13 @@ static void message_size_start_transport_stream_op_batch( call_data* calld = static_cast(elem->call_data); // Check max send message size. if (op->send_message && calld->limits.max_send_size >= 0 && - op->payload->send_message.send_message->Length() > + op->payload->send_message.send_message->length() > static_cast(calld->limits.max_send_size)) { grpc_transport_stream_op_batch_finish_with_failure( op, grpc_error_set_int(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( "Sent message larger than max (%u vs. %d)", - op->payload->send_message.send_message->Length(), + op->payload->send_message.send_message->length(), calld->limits.max_send_size)), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED), diff --git a/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc b/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc index bdb76d30dc0..4489a857a02 100644 --- a/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc +++ b/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc @@ -18,6 +18,7 @@ #include #include +#include #include #include "absl/base/thread_annotations.h" diff --git a/src/core/ext/transport/binder/transport/binder_stream.h b/src/core/ext/transport/binder/transport/binder_stream.h index 6c753e8ca09..b04076b1f85 100644 --- a/src/core/ext/transport/binder/transport/binder_stream.h +++ b/src/core/ext/transport/binder/transport/binder_stream.h @@ -80,6 +80,7 @@ struct grpc_binder_stream { grpc_binder_transport* t; grpc_stream_refcount* refcount; grpc_core::Arena* arena; + grpc_core::ManualConstructor sbs; int tx_code; const bool is_client; bool is_closed; @@ -105,7 +106,7 @@ struct grpc_binder_stream { grpc_metadata_batch* recv_initial_metadata; grpc_closure* recv_initial_metadata_ready = nullptr; bool* trailing_metadata_available = nullptr; - absl::optional* recv_message; + grpc_core::OrphanablePtr* recv_message; grpc_closure* recv_message_ready = nullptr; bool* call_failed_before_recv_message = nullptr; grpc_metadata_batch* recv_trailing_metadata; diff --git a/src/core/ext/transport/binder/transport/binder_transport.cc b/src/core/ext/transport/binder/transport/binder_transport.cc index 4cf48b4ca8d..d1374ef5a48 100644 --- a/src/core/ext/transport/binder/transport/binder_transport.cc +++ b/src/core/ext/transport/binder/transport/binder_transport.cc @@ -37,6 +37,7 @@ #include "src/core/ext/transport/binder/wire_format/wire_writer.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -256,9 +257,12 @@ static void recv_message_locked(void* arg, grpc_error_handle /*error*/) { return absl_status_to_grpc_error(args->message.status()); } } - grpc_core::SliceBuffer buf; - buf.Append(grpc_core::Slice(grpc_slice_from_cpp_string(*args->message))); - *gbs->recv_message = std::move(buf); + grpc_slice_buffer buf; + grpc_slice_buffer_init(&buf); + grpc_slice_buffer_add(&buf, grpc_slice_from_cpp_string(*args->message)); + + gbs->sbs.Init(&buf, 0); + gbs->recv_message->reset(gbs->sbs.get()); return GRPC_ERROR_NONE; }(); @@ -407,7 +411,7 @@ static void perform_stream_op_locked(void* stream_op, if (gbs->is_closed) { if (op->send_message) { // Reset the send_message payload to prevent memory leaks. - op->payload->send_message.send_message->Clear(); + op->payload->send_message.send_message.reset(); } if (op->recv_initial_metadata) { grpc_core::ExecCtx::Run( @@ -448,7 +452,27 @@ static void perform_stream_op_locked(void* stream_op, } if (op->send_message) { gpr_log(GPR_INFO, "send_message"); - tx.SetData(op->payload->send_message.send_message->JoinIntoString()); + size_t remaining = op->payload->send_message.send_message->length(); + std::string message_data; + while (remaining > 0) { + grpc_slice message_slice; + // TODO(waynetu): Temporarily assume that the message is ready. + GPR_ASSERT( + op->payload->send_message.send_message->Next(SIZE_MAX, nullptr)); + grpc_error_handle error = + op->payload->send_message.send_message->Pull(&message_slice); + // TODO(waynetu): Cancel the stream if error is not GRPC_ERROR_NONE. + GPR_ASSERT(error == GRPC_ERROR_NONE); + uint8_t* p = GRPC_SLICE_START_PTR(message_slice); + size_t len = GRPC_SLICE_LENGTH(message_slice); + remaining -= len; + message_data += std::string(reinterpret_cast(p), len); + grpc_slice_unref_internal(message_slice); + } + tx.SetData(message_data); + // TODO(b/192369787): Are we supposed to reset here to avoid + // use-after-free issue in call.cc? + op->payload->send_message.send_message.reset(); } if (op->send_trailing_metadata) { diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index b12d1887d91..deec773ed82 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -37,7 +37,6 @@ #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "absl/types/variant.h" #include #include @@ -62,6 +61,9 @@ #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/bitset.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/global_config_env.h" +#include "src/core/lib/gprpp/manual_constructor.h" +#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/time.h" #include "src/core/lib/http/parser.h" @@ -72,17 +74,16 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/profiling/timers.h" -#include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/api.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/memory_quota.h" #include "src/core/lib/resource_quota/resource_quota.h" #include "src/core/lib/resource_quota/trace.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/transport/bdp_estimator.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/http2_errors.h" @@ -91,6 +92,12 @@ #include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport_impl.h" +GPR_GLOBAL_CONFIG_DEFINE_BOOL( + grpc_experimental_disable_flow_control, false, + "If set, flow control will be effectively disabled. Max out all values and " + "assume the remote peer does the same. Thus we can ignore any flow control " + "bookkeeping, error checking, and decision making"); + #define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024) #define MAX_WINDOW 0x7fffffffu #define MAX_WRITE_BUFFER_SIZE (64 * 1024 * 1024) @@ -143,6 +150,8 @@ static void read_action(void* t, grpc_error_handle error); static void read_action_locked(void* t, grpc_error_handle error); static void continue_read_action_locked(grpc_chttp2_transport* t); +static void complete_fetch(void* gs, grpc_error_handle error); +static void complete_fetch_locked(void* gs, grpc_error_handle error); // Set a transport level setting, and push it to our peer static void queue_setting_update(grpc_chttp2_transport* t, grpc_chttp2_setting_id id, uint32_t value); @@ -192,6 +201,8 @@ static void finish_keepalive_ping_locked(void* arg, grpc_error_handle error); static void keepalive_watchdog_fired(void* arg, grpc_error_handle error); static void keepalive_watchdog_fired_locked(void* arg, grpc_error_handle error); +static void reset_byte_stream(void* arg, grpc_error_handle error); + namespace grpc_core { namespace { @@ -268,6 +279,8 @@ grpc_chttp2_transport::~grpc_chttp2_transport() { write_cb_pool = next; } + flow_control.Destroy(); + GRPC_ERROR_UNREF(closed_with_error); gpr_free(ping_acks); if (grpc_core::test_only_destruct_callback != nullptr) { @@ -277,9 +290,11 @@ grpc_chttp2_transport::~grpc_chttp2_transport() { static const grpc_transport_vtable* get_vtable(void); -static void read_channel_args(grpc_chttp2_transport* t, +// Returns whether bdp is enabled +static bool read_channel_args(grpc_chttp2_transport* t, const grpc_channel_args* channel_args, bool is_client) { + bool enable_bdp = true; bool channelz_enabled = GRPC_ENABLE_CHANNELZ_DEFAULT; size_t i; int j; @@ -329,6 +344,9 @@ static void read_channel_args(grpc_chttp2_transport* t, GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE)) { t->write_buffer_size = static_cast(grpc_channel_arg_get_integer( &channel_args->args[i], {0, 0, MAX_WRITE_BUFFER_SIZE})); + } else if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_BDP_PROBE)) { + enable_bdp = grpc_channel_arg_get_bool(&channel_args->args[i], true); } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_KEEPALIVE_TIME_MS)) { const int value = grpc_channel_arg_get_integer( @@ -420,6 +438,7 @@ static void read_channel_args(grpc_chttp2_transport* t, grpc_core::channelz::SocketNode::Security::GetFromChannelArgs( channel_args)); } + return enable_bdp; } static void init_transport_keepalive_settings(grpc_chttp2_transport* t) { @@ -490,10 +509,6 @@ grpc_chttp2_transport::grpc_chttp2_transport( GRPC_CHANNEL_READY), is_client(is_client), next_stream_id(is_client ? 1 : 2), - flow_control(peer_string.c_str(), - grpc_channel_args_find_bool(channel_args, - GRPC_ARG_HTTP2_BDP_PROBE, true), - &memory_owner), deframe_state(is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0) { GPR_ASSERT(strlen(GRPC_CHTTP2_CLIENT_CONNECT_STRING) == GRPC_CHTTP2_CLIENT_CONNECT_STRLEN); @@ -535,8 +550,19 @@ grpc_chttp2_transport::grpc_chttp2_transport( configure_transport_ping_policy(this); init_transport_keepalive_settings(this); - if (channel_args != nullptr) { - read_channel_args(this, channel_args, is_client); + bool enable_bdp = true; + if (channel_args) { + enable_bdp = read_channel_args(this, channel_args, is_client); + } + + static const bool kEnableFlowControl = + !GPR_GLOBAL_CONFIG_GET(grpc_experimental_disable_flow_control); + if (kEnableFlowControl) { + flow_control.Init(this, + enable_bdp); + } else { + flow_control.Init(this); + enable_bdp = false; } // No pings allowed before receiving a header or data frame. @@ -549,9 +575,9 @@ grpc_chttp2_transport::grpc_chttp2_transport( init_keepalive_pings_if_enabled(this); - if (flow_control.bdp_probe()) { + if (enable_bdp) { bdp_ping_blocked = true; - grpc_chttp2_act_on_flowctl_action(flow_control.PeriodicUpdate(), this, + grpc_chttp2_act_on_flowctl_action(flow_control->PeriodicUpdate(), this, nullptr); } @@ -676,17 +702,26 @@ grpc_chttp2_stream::grpc_chttp2_stream(grpc_chttp2_transport* t, refcount(refcount), reffer(this), initial_metadata_buffer(arena), - trailing_metadata_buffer(arena), - flow_control(&t->flow_control) { + trailing_metadata_buffer(arena) { if (server_data) { id = static_cast(reinterpret_cast(server_data)); *t->accepting_stream = this; grpc_chttp2_stream_map_add(&t->stream_map, id, this); post_destructive_reclaimer(t); } + if (t->flow_control->flow_control_enabled()) { + flow_control.Init( + static_cast( + t->flow_control.get()), + this); + } else { + flow_control.Init(); + } grpc_slice_buffer_init(&frame_storage); + grpc_slice_buffer_init(&unprocessed_incoming_frames_buffer); grpc_slice_buffer_init(&flow_controlled_buffer); + GRPC_CLOSURE_INIT(&reset_byte_stream, ::reset_byte_stream, this, nullptr); } grpc_chttp2_stream::~grpc_chttp2_stream() { @@ -703,6 +738,7 @@ grpc_chttp2_stream::~grpc_chttp2_stream() { GPR_ASSERT(grpc_chttp2_stream_map_find(&t->stream_map, id) == nullptr); } + grpc_slice_buffer_destroy_internal(&unprocessed_incoming_frames_buffer); grpc_slice_buffer_destroy_internal(&frame_storage); for (int i = 0; i < STREAM_LIST_COUNT; i++) { @@ -714,6 +750,7 @@ grpc_chttp2_stream::~grpc_chttp2_stream() { } GPR_ASSERT(send_initial_metadata_finished == nullptr); + GPR_ASSERT(fetching_send_message == nullptr); GPR_ASSERT(send_trailing_metadata_finished == nullptr); GPR_ASSERT(recv_initial_metadata_ready == nullptr); GPR_ASSERT(recv_message_ready == nullptr); @@ -721,6 +758,8 @@ grpc_chttp2_stream::~grpc_chttp2_stream() { grpc_slice_buffer_destroy_internal(&flow_controlled_buffer); GRPC_ERROR_UNREF(read_closed_error); GRPC_ERROR_UNREF(write_closed_error); + GRPC_ERROR_UNREF(byte_stream_error); + flow_control.Destroy(); GRPC_CHTTP2_UNREF_TRANSPORT(t, "stream"); grpc_core::ExecCtx::Run(DEBUG_LOCATION, destroy_stream_arg, GRPC_ERROR_NONE); } @@ -1286,6 +1325,94 @@ static bool contains_non_ok_status(grpc_metadata_batch* batch) { GRPC_STATUS_OK; } +static void maybe_become_writable_due_to_send_msg(grpc_chttp2_transport* t, + grpc_chttp2_stream* s) { + if (s->id != 0 && (!s->write_buffering || + s->flow_controlled_buffer.length > t->write_buffer_size)) { + grpc_chttp2_mark_stream_writable(t, s); + grpc_chttp2_initiate_write(t, GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE); + } +} + +static void add_fetched_slice_locked(grpc_chttp2_transport* t, + grpc_chttp2_stream* s) { + s->fetched_send_message_length += + static_cast GRPC_SLICE_LENGTH(s->fetching_slice); + grpc_slice_buffer_add(&s->flow_controlled_buffer, s->fetching_slice); + maybe_become_writable_due_to_send_msg(t, s); +} + +static void continue_fetching_send_locked(grpc_chttp2_transport* t, + grpc_chttp2_stream* s) { + for (;;) { + if (s->fetching_send_message == nullptr) { + // Stream was cancelled before message fetch completed + abort(); /* TODO(ctiller): what cleanup here? */ + } + if (s->fetched_send_message_length == s->fetching_send_message->length()) { + int64_t notify_offset = s->next_message_end_offset; + if (notify_offset <= s->flow_controlled_bytes_written) { + grpc_chttp2_complete_closure_step( + t, s, &s->fetching_send_message_finished, GRPC_ERROR_NONE, + "fetching_send_message_finished"); + } else { + grpc_chttp2_write_cb* cb = t->write_cb_pool; + if (cb == nullptr) { + cb = static_cast(gpr_malloc(sizeof(*cb))); + } else { + t->write_cb_pool = cb->next; + } + cb->call_at_byte = notify_offset; + cb->closure = s->fetching_send_message_finished; + s->fetching_send_message_finished = nullptr; + grpc_chttp2_write_cb** list = + s->fetching_send_message->flags() & GRPC_WRITE_THROUGH + ? &s->on_write_finished_cbs + : &s->on_flow_controlled_cbs; + cb->next = *list; + *list = cb; + } + s->fetching_send_message.reset(); + return; /* early out */ + } else if (s->fetching_send_message->Next( + UINT32_MAX, GRPC_CLOSURE_INIT(&s->complete_fetch_locked, + ::complete_fetch, s, + grpc_schedule_on_exec_ctx))) { + grpc_error_handle error = + s->fetching_send_message->Pull(&s->fetching_slice); + if (error != GRPC_ERROR_NONE) { + s->fetching_send_message.reset(); + grpc_chttp2_cancel_stream(t, s, error); + } else { + add_fetched_slice_locked(t, s); + } + } + } +} + +static void complete_fetch(void* gs, grpc_error_handle error) { + grpc_chttp2_stream* s = static_cast(gs); + s->t->combiner->Run(GRPC_CLOSURE_INIT(&s->complete_fetch_locked, + ::complete_fetch_locked, s, nullptr), + GRPC_ERROR_REF(error)); +} + +static void complete_fetch_locked(void* gs, grpc_error_handle error) { + grpc_chttp2_stream* s = static_cast(gs); + grpc_chttp2_transport* t = s->t; + if (error == GRPC_ERROR_NONE) { + error = s->fetching_send_message->Pull(&s->fetching_slice); + if (error == GRPC_ERROR_NONE) { + add_fetched_slice_locked(t, s); + continue_fetching_send_locked(t, s); + } + } + if (error != GRPC_ERROR_NONE) { + s->fetching_send_message.reset(); + grpc_chttp2_cancel_stream(t, s, error); + } +} + static void log_metadata(const grpc_metadata_batch* md_batch, uint32_t id, bool is_client, bool is_initial) { const std::string prefix = absl::StrCat( @@ -1380,7 +1507,8 @@ static void perform_stream_op_locked(void* stream_op, GPR_ASSERT(s->id != 0); grpc_chttp2_mark_stream_writable(t, s); if (!(op->send_message && - (op->payload->send_message.flags & GRPC_WRITE_BUFFER_HINT))) { + (op->payload->send_message.send_message->flags() & + GRPC_WRITE_BUFFER_HINT))) { grpc_chttp2_initiate_write( t, GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA); } @@ -1404,28 +1532,32 @@ static void perform_stream_op_locked(void* stream_op, GRPC_STATS_INC_HTTP2_OP_SEND_MESSAGE(); t->num_messages_in_next_write++; GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE( - op->payload->send_message.send_message->Length()); + op->payload->send_message.send_message->length()); on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE; - s->send_message_finished = add_closure_barrier(op->on_complete); - const uint32_t flags = op_payload->send_message.flags; + s->fetching_send_message_finished = add_closure_barrier(op->on_complete); if (s->write_closed) { op->payload->send_message.stream_write_closed = true; // We should NOT return an error here, so as to avoid a cancel OP being // started. The surface layer will notice that the stream has been closed // for writes and fail the send message op. - grpc_chttp2_complete_closure_step(t, s, &s->send_message_finished, - GRPC_ERROR_NONE, - "fetching_send_message_finished"); + op->payload->send_message.send_message.reset(); + grpc_chttp2_complete_closure_step( + t, s, &s->fetching_send_message_finished, GRPC_ERROR_NONE, + "fetching_send_message_finished"); } else { + GPR_ASSERT(s->fetching_send_message == nullptr); uint8_t* frame_hdr = grpc_slice_buffer_tiny_add( &s->flow_controlled_buffer, GRPC_HEADER_SIZE_IN_BYTES); + uint32_t flags = op_payload->send_message.send_message->flags(); frame_hdr[0] = (flags & GRPC_WRITE_INTERNAL_COMPRESS) != 0; - size_t len = op_payload->send_message.send_message->Length(); + size_t len = op_payload->send_message.send_message->length(); frame_hdr[1] = static_cast(len >> 24); frame_hdr[2] = static_cast(len >> 16); frame_hdr[3] = static_cast(len >> 8); frame_hdr[4] = static_cast(len); - + s->fetching_send_message = + std::move(op_payload->send_message.send_message); + s->fetched_send_message_length = 0; s->next_message_end_offset = s->flow_controlled_bytes_written + static_cast(s->flow_controlled_buffer.length) + @@ -1436,44 +1568,8 @@ static void perform_stream_op_locked(void* stream_op, } else { s->write_buffering = false; } - - grpc_slice* const slices = - op_payload->send_message.send_message->c_slice_buffer()->slices; - grpc_slice* const end = - slices + op_payload->send_message.send_message->Count(); - for (grpc_slice* slice = slices; slice != end; slice++) { - grpc_slice_buffer_add(&s->flow_controlled_buffer, - grpc_slice_ref_internal(*slice)); - } - - int64_t notify_offset = s->next_message_end_offset; - if (notify_offset <= s->flow_controlled_bytes_written) { - grpc_chttp2_complete_closure_step(t, s, &s->send_message_finished, - GRPC_ERROR_NONE, - "fetching_send_message_finished"); - } else { - grpc_chttp2_write_cb* cb = t->write_cb_pool; - if (cb == nullptr) { - cb = static_cast(gpr_malloc(sizeof(*cb))); - } else { - t->write_cb_pool = cb->next; - } - cb->call_at_byte = notify_offset; - cb->closure = s->send_message_finished; - s->send_message_finished = nullptr; - grpc_chttp2_write_cb** list = flags & GRPC_WRITE_THROUGH - ? &s->on_write_finished_cbs - : &s->on_flow_controlled_cbs; - cb->next = *list; - *list = cb; - } - - if (s->id != 0 && - (!s->write_buffering || - s->flow_controlled_buffer.length > t->write_buffer_size)) { - grpc_chttp2_mark_stream_writable(t, s); - grpc_chttp2_initiate_write(t, GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE); - } + continue_fetching_send_locked(t, s); + maybe_become_writable_due_to_send_msg(t, s); } } @@ -1527,14 +1623,28 @@ static void perform_stream_op_locked(void* stream_op, if (op->recv_message) { GRPC_STATS_INC_HTTP2_OP_RECV_MESSAGE(); + size_t before = 0; GPR_ASSERT(s->recv_message_ready == nullptr); + GPR_ASSERT(!s->pending_byte_stream); s->recv_message_ready = op_payload->recv_message.recv_message_ready; s->recv_message = op_payload->recv_message.recv_message; - s->recv_message->emplace(); - s->recv_message_flags = op_payload->recv_message.flags; s->call_failed_before_recv_message = op_payload->recv_message.call_failed_before_recv_message; - grpc_chttp2_maybe_complete_recv_trailing_metadata(t, s); + if (s->id != 0) { + if (!s->read_closed) { + before = s->frame_storage.length + + s->unprocessed_incoming_frames_buffer.length; + } + } + grpc_chttp2_maybe_complete_recv_message(t, s); + if (s->id != 0) { + if (!s->read_closed && s->frame_storage.length == 0) { + size_t after = s->unprocessed_incoming_frames_buffer_cached_length; + s->flow_control->IncomingByteStreamUpdate(GRPC_HEADER_SIZE_IN_BYTES, + before - after); + grpc_chttp2_act_on_flowctl_action(s->flow_control->MakeAction(), t, s); + } + } } if (op->recv_trailing_metadata) { @@ -1908,6 +2018,10 @@ void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_chttp2_transport* t, s->published_metadata[0] != GRPC_METADATA_NOT_PUBLISHED) { if (s->seen_error) { grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); + if (!s->pending_byte_stream) { + grpc_slice_buffer_reset_and_unref_internal( + &s->unprocessed_incoming_frames_buffer); + } } *s->recv_initial_metadata = std::move(s->initial_metadata_buffer); s->recv_initial_metadata->Set(grpc_core::PeerString(), t->peer_string); @@ -1924,58 +2038,47 @@ void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_chttp2_transport* t, } } -void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t, +void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* /*t*/, grpc_chttp2_stream* s) { grpc_error_handle error = GRPC_ERROR_NONE; if (s->recv_message_ready != nullptr) { + *s->recv_message = nullptr; if (s->final_metadata_requested && s->seen_error) { grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); - s->recv_message->reset(); - } else { - if (s->frame_storage.length != 0) { - while (true) { - GPR_ASSERT(s->frame_storage.length > 0); - uint32_t min_progress_size; - auto r = grpc_deframe_unprocessed_incoming_frames( - s, &min_progress_size, &**s->recv_message, s->recv_message_flags); - if (absl::holds_alternative(r)) { - if (s->read_closed) { - grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); - s->recv_message->reset(); - break; - } else { - s->flow_control.UpdateProgress(min_progress_size); - grpc_chttp2_act_on_flowctl_action(s->flow_control.MakeAction(), t, - s); - return; - } - } else { - error = absl::get(r); - if (error != GRPC_ERROR_NONE) { - s->seen_error = true; - grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); - break; - } else { - if (t->channelz_socket != nullptr) { - t->channelz_socket->RecordMessageReceived(); - } - break; - } - } + if (!s->pending_byte_stream) { + grpc_slice_buffer_reset_and_unref_internal( + &s->unprocessed_incoming_frames_buffer); + } + } + if (!s->pending_byte_stream) { + while (s->unprocessed_incoming_frames_buffer.length > 0 || + s->frame_storage.length > 0) { + if (s->unprocessed_incoming_frames_buffer.length == 0) { + grpc_slice_buffer_swap(&s->unprocessed_incoming_frames_buffer, + &s->frame_storage); + } + error = grpc_deframe_unprocessed_incoming_frames( + &s->data_parser, s, &s->unprocessed_incoming_frames_buffer, nullptr, + s->recv_message); + if (error != GRPC_ERROR_NONE) { + s->seen_error = true; + grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); + grpc_slice_buffer_reset_and_unref_internal( + &s->unprocessed_incoming_frames_buffer); + break; + } else if (*s->recv_message != nullptr) { + break; } - } else if (s->read_closed) { - s->recv_message->reset(); - } else { - s->flow_control.UpdateProgress(GRPC_HEADER_SIZE_IN_BYTES); - grpc_chttp2_act_on_flowctl_action(s->flow_control.MakeAction(), t, s); - return; } } // save the length of the buffer before handing control back to application // threads. Needed to support correct flow control bookkeeping - if (error == GRPC_ERROR_NONE && s->recv_message->has_value()) { + s->unprocessed_incoming_frames_buffer_cached_length = + s->unprocessed_incoming_frames_buffer.length; + if (error == GRPC_ERROR_NONE && *s->recv_message != nullptr) { null_then_sched_closure(&s->recv_message_ready); } else if (s->published_metadata[1] != GRPC_METADATA_NOT_PUBLISHED) { + *s->recv_message = nullptr; if (s->call_failed_before_recv_message != nullptr) { *s->call_failed_before_recv_message = (s->published_metadata[1] != GRPC_METADATA_PUBLISHED_AT_CLOSE); @@ -1993,8 +2096,26 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_chttp2_transport* t, s->write_closed) { if (s->seen_error || !t->is_client) { grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); + if (!s->pending_byte_stream) { + grpc_slice_buffer_reset_and_unref_internal( + &s->unprocessed_incoming_frames_buffer); + } } - if (s->read_closed && s->frame_storage.length == 0 && + bool pending_data = s->pending_byte_stream || + s->unprocessed_incoming_frames_buffer.length > 0; + if (s->read_closed && s->frame_storage.length > 0 && !pending_data && + !s->seen_error && s->recv_trailing_metadata_finished != nullptr) { + // Maybe some SYNC_FLUSH data is left in frame_storage. Consume them and + // maybe decompress the next 5 bytes in the stream. + grpc_slice_buffer_move_first( + &s->frame_storage, + std::min(s->frame_storage.length, size_t(GRPC_HEADER_SIZE_IN_BYTES)), + &s->unprocessed_incoming_frames_buffer); + if (s->unprocessed_incoming_frames_buffer.length > 0) { + pending_data = true; + } + } + if (s->read_closed && s->frame_storage.length == 0 && !pending_data && s->recv_trailing_metadata_finished != nullptr) { grpc_transport_move_stats(&s->stats, s->collecting_stats); s->collecting_stats = nullptr; @@ -2014,6 +2135,20 @@ static void remove_stream(grpc_chttp2_transport* t, uint32_t id, t->incoming_stream = nullptr; grpc_chttp2_parsing_become_skip_parser(t); } + if (s->pending_byte_stream) { + if (s->on_next != nullptr) { + grpc_core::Chttp2IncomingByteStream* bs = s->data_parser.parsing_frame; + if (error == GRPC_ERROR_NONE) { + error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message"); + } + bs->PublishError(error); + bs->Unref(); + s->data_parser.parsing_frame = nullptr; + } else { + GRPC_ERROR_UNREF(s->byte_stream_error); + s->byte_stream_error = GRPC_ERROR_REF(error); + } + } if (grpc_chttp2_stream_map_size(&t->stream_map) == 0) { post_benign_reclaimer(t); @@ -2147,7 +2282,8 @@ void grpc_chttp2_fail_pending_writes(grpc_chttp2_transport* t, GRPC_ERROR_REF(error), "send_trailing_metadata_finished"); - grpc_chttp2_complete_closure_step(t, s, &s->send_message_finished, + s->fetching_send_message.reset(); + grpc_chttp2_complete_closure_step(t, s, &s->fetching_send_message_finished, GRPC_ERROR_REF(error), "fetching_send_message_finished"); flush_write_list(t, s, &s->on_write_finished_cbs, GRPC_ERROR_REF(error)); @@ -2417,11 +2553,8 @@ void grpc_chttp2_act_on_flowctl_action( const grpc_core::chttp2::FlowControlAction& action, grpc_chttp2_transport* t, grpc_chttp2_stream* s) { WithUrgency(t, action.send_stream_update(), - GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL, [t, s]() { - if (s->id != 0) { - grpc_chttp2_mark_stream_writable(t, s); - } - }); + GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL, + [t, s]() { grpc_chttp2_mark_stream_writable(t, s); }); WithUrgency(t, action.send_transport_update(), GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL, []() {}); WithUrgency(t, action.send_initial_window_update(), @@ -2569,13 +2702,13 @@ static void continue_read_action_locked(grpc_chttp2_transport* t) { grpc_schedule_on_exec_ctx); grpc_endpoint_read(t->ep, &t->read_buffer, &t->read_action_locked, urgent, /*min_progress_size=*/1); - grpc_chttp2_act_on_flowctl_action(t->flow_control.MakeAction(), t, nullptr); + grpc_chttp2_act_on_flowctl_action(t->flow_control->MakeAction(), t, nullptr); } // t is reffed prior to calling the first time, and once the callback chain // that kicks off finishes, it's unreffed void schedule_bdp_ping_locked(grpc_chttp2_transport* t) { - t->flow_control.bdp_estimator()->SchedulePing(); + t->flow_control->bdp_estimator()->SchedulePing(); send_ping_locked( t, GRPC_CLOSURE_INIT(&t->start_bdp_ping_locked, start_bdp_ping, t, @@ -2605,7 +2738,7 @@ static void start_bdp_ping_locked(void* tp, grpc_error_handle error) { if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_WAITING) { grpc_timer_cancel(&t->keepalive_ping_timer); } - t->flow_control.bdp_estimator()->StartPing(); + t->flow_control->bdp_estimator()->StartPing(); t->bdp_ping_started = true; } @@ -2636,8 +2769,8 @@ static void finish_bdp_ping_locked(void* tp, grpc_error_handle error) { } t->bdp_ping_started = false; grpc_core::Timestamp next_ping = - t->flow_control.bdp_estimator()->CompletePing(); - grpc_chttp2_act_on_flowctl_action(t->flow_control.PeriodicUpdate(), t, + t->flow_control->bdp_estimator()->CompletePing(); + grpc_chttp2_act_on_flowctl_action(t->flow_control->PeriodicUpdate(), t, nullptr); GPR_ASSERT(!t->have_next_bdp_ping_timer); t->have_next_bdp_ping_timer = true; @@ -2664,7 +2797,7 @@ static void next_bdp_ping_timer_expired_locked(void* tp, GRPC_CHTTP2_UNREF_TRANSPORT(t, "bdp_ping"); return; } - if (t->flow_control.bdp_estimator()->accumulator() == 0) { + if (t->flow_control->bdp_estimator()->accumulator() == 0) { // Block the bdp ping till we receive more data. t->bdp_ping_blocked = true; GRPC_CHTTP2_UNREF_TRANSPORT(t, "bdp_ping"); @@ -2900,6 +3033,187 @@ static void set_pollset_set(grpc_transport* gt, grpc_stream* /*gs*/, grpc_endpoint_add_to_pollset_set(t->ep, pollset_set); } +// +// BYTE STREAM +// + +static void reset_byte_stream(void* arg, grpc_error_handle error) { + grpc_chttp2_stream* s = static_cast(arg); + s->pending_byte_stream = false; + if (error == GRPC_ERROR_NONE) { + grpc_chttp2_maybe_complete_recv_message(s->t, s); + grpc_chttp2_maybe_complete_recv_trailing_metadata(s->t, s); + } else { + GPR_ASSERT(error != GRPC_ERROR_NONE); + grpc_core::ExecCtx::Run(DEBUG_LOCATION, s->on_next, GRPC_ERROR_REF(error)); + s->on_next = nullptr; + GRPC_ERROR_UNREF(s->byte_stream_error); + s->byte_stream_error = GRPC_ERROR_NONE; + grpc_chttp2_cancel_stream(s->t, s, GRPC_ERROR_REF(error)); + s->byte_stream_error = GRPC_ERROR_REF(error); + } +} + +namespace grpc_core { + +Chttp2IncomingByteStream::Chttp2IncomingByteStream( + grpc_chttp2_transport* transport, grpc_chttp2_stream* stream, + uint32_t frame_size, uint32_t flags) + : ByteStream(frame_size, flags), + transport_(transport), + stream_(stream), + refs_(2), + remaining_bytes_(frame_size) { + GRPC_ERROR_UNREF(stream->byte_stream_error); + stream->byte_stream_error = GRPC_ERROR_NONE; +} + +void Chttp2IncomingByteStream::OrphanLocked( + void* arg, grpc_error_handle /*error_ignored*/) { + Chttp2IncomingByteStream* bs = static_cast(arg); + grpc_chttp2_stream* s = bs->stream_; + grpc_chttp2_transport* t = s->t; + bs->Unref(); + s->pending_byte_stream = false; + grpc_chttp2_maybe_complete_recv_message(t, s); + grpc_chttp2_maybe_complete_recv_trailing_metadata(t, s); +} + +void Chttp2IncomingByteStream::Orphan() { + GPR_TIMER_SCOPE("incoming_byte_stream_destroy", 0); + transport_->combiner->Run( + GRPC_CLOSURE_INIT(&destroy_action_, + &Chttp2IncomingByteStream::OrphanLocked, this, nullptr), + GRPC_ERROR_NONE); +} + +void Chttp2IncomingByteStream::NextLocked(void* arg, + grpc_error_handle /*error_ignored*/) { + Chttp2IncomingByteStream* bs = static_cast(arg); + grpc_chttp2_transport* t = bs->transport_; + grpc_chttp2_stream* s = bs->stream_; + size_t cur_length = s->frame_storage.length; + if (!s->read_closed) { + s->flow_control->IncomingByteStreamUpdate(bs->next_action_.max_size_hint, + cur_length); + grpc_chttp2_act_on_flowctl_action(s->flow_control->MakeAction(), t, s); + } + GPR_ASSERT(s->unprocessed_incoming_frames_buffer.length == 0); + if (s->frame_storage.length > 0) { + grpc_slice_buffer_swap(&s->frame_storage, + &s->unprocessed_incoming_frames_buffer); + ExecCtx::Run(DEBUG_LOCATION, bs->next_action_.on_complete, GRPC_ERROR_NONE); + } else if (s->byte_stream_error != GRPC_ERROR_NONE) { + ExecCtx::Run(DEBUG_LOCATION, bs->next_action_.on_complete, + GRPC_ERROR_REF(s->byte_stream_error)); + if (s->data_parser.parsing_frame != nullptr) { + s->data_parser.parsing_frame->Unref(); + s->data_parser.parsing_frame = nullptr; + } + } else if (s->read_closed) { + if (bs->remaining_bytes_ != 0) { + s->byte_stream_error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "Truncated message", &s->read_closed_error, 1); + ExecCtx::Run(DEBUG_LOCATION, bs->next_action_.on_complete, + GRPC_ERROR_REF(s->byte_stream_error)); + if (s->data_parser.parsing_frame != nullptr) { + s->data_parser.parsing_frame->Unref(); + s->data_parser.parsing_frame = nullptr; + } + } else { + // Should never reach here. + GPR_ASSERT(false); + } + } else { + s->on_next = bs->next_action_.on_complete; + } + bs->Unref(); +} + +bool Chttp2IncomingByteStream::Next(size_t max_size_hint, + grpc_closure* on_complete) { + GPR_TIMER_SCOPE("incoming_byte_stream_next", 0); + if (stream_->unprocessed_incoming_frames_buffer.length > 0) { + return true; + } else { + Ref(); + next_action_.max_size_hint = max_size_hint; + next_action_.on_complete = on_complete; + transport_->combiner->Run( + GRPC_CLOSURE_INIT(&next_action_.closure, + &Chttp2IncomingByteStream::NextLocked, this, nullptr), + GRPC_ERROR_NONE); + return false; + } +} + +grpc_error_handle Chttp2IncomingByteStream::Pull(grpc_slice* slice) { + GPR_TIMER_SCOPE("incoming_byte_stream_pull", 0); + grpc_error_handle error; + if (stream_->unprocessed_incoming_frames_buffer.length > 0) { + error = grpc_deframe_unprocessed_incoming_frames( + &stream_->data_parser, stream_, + &stream_->unprocessed_incoming_frames_buffer, slice, nullptr); + if (error != GRPC_ERROR_NONE) { + return error; + } + } else { + error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message"); + stream_->t->combiner->Run(&stream_->reset_byte_stream, + GRPC_ERROR_REF(error)); + return error; + } + return GRPC_ERROR_NONE; +} + +void Chttp2IncomingByteStream::PublishError(grpc_error_handle error) { + GPR_ASSERT(error != GRPC_ERROR_NONE); + ExecCtx::Run(DEBUG_LOCATION, stream_->on_next, GRPC_ERROR_REF(error)); + stream_->on_next = nullptr; + GRPC_ERROR_UNREF(stream_->byte_stream_error); + stream_->byte_stream_error = GRPC_ERROR_REF(error); + grpc_chttp2_cancel_stream(transport_, stream_, GRPC_ERROR_REF(error)); +} + +grpc_error_handle Chttp2IncomingByteStream::Push(const grpc_slice& slice, + grpc_slice* slice_out) { + if (remaining_bytes_ < GRPC_SLICE_LENGTH(slice)) { + grpc_error_handle error = + GRPC_ERROR_CREATE_FROM_STATIC_STRING("Too many bytes in stream"); + transport_->combiner->Run(&stream_->reset_byte_stream, + GRPC_ERROR_REF(error)); + grpc_slice_unref_internal(slice); + return error; + } else { + remaining_bytes_ -= static_cast GRPC_SLICE_LENGTH(slice); + if (slice_out != nullptr) { + *slice_out = slice; + } + return GRPC_ERROR_NONE; + } +} + +grpc_error_handle Chttp2IncomingByteStream::Finished(grpc_error_handle error, + bool reset_on_error) { + if (error == GRPC_ERROR_NONE) { + if (remaining_bytes_ != 0) { + error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message"); + } + } + if (error != GRPC_ERROR_NONE && reset_on_error) { + transport_->combiner->Run(&stream_->reset_byte_stream, + GRPC_ERROR_REF(error)); + } + Unref(); + return error; +} + +void Chttp2IncomingByteStream::Shutdown(grpc_error_handle error) { + GRPC_ERROR_UNREF(Finished(error, true /* reset_on_error */)); +} + +} // namespace grpc_core + // // RESOURCE QUOTAS // diff --git a/src/core/ext/transport/chttp2/transport/flow_control.cc b/src/core/ext/transport/chttp2/transport/flow_control.cc index 769ea438885..54aad55a994 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.cc +++ b/src/core/ext/transport/chttp2/transport/flow_control.cc @@ -23,18 +23,18 @@ #include #include -#include #include -#include #include -#include -#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" -#include "absl/strings/str_join.h" +#include +#include #include +#include "src/core/ext/transport/chttp2/transport/frame.h" +#include "src/core/ext/transport/chttp2/transport/internal.h" +#include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/resource_quota/memory_quota.h" @@ -51,61 +51,140 @@ bool g_test_only_transport_flow_control_window_check; namespace { +constexpr const int kTracePadding = 30; constexpr const int64_t kMaxWindowUpdateSize = (1u << 31) - 1; +char* fmt_int64_diff_str(int64_t old_val, int64_t new_val) { + std::string str; + if (old_val != new_val) { + str = absl::StrFormat("%" PRId64 " -> %" PRId64 "", old_val, new_val); + } else { + str = absl::StrFormat("%" PRId64 "", old_val); + } + return gpr_leftpad(str.c_str(), ' ', kTracePadding); +} + +char* fmt_uint32_diff_str(uint32_t old_val, uint32_t new_val) { + std::string str; + if (old_val != new_val) { + str = absl::StrFormat("%" PRIu32 " -> %" PRIu32 "", old_val, new_val); + } else { + str = absl::StrFormat("%" PRIu32 "", old_val); + } + return gpr_leftpad(str.c_str(), ' ', kTracePadding); +} } // namespace +void FlowControlTrace::Init(const char* reason, TransportFlowControl* tfc, + StreamFlowControl* sfc) { + tfc_ = tfc; + sfc_ = sfc; + reason_ = reason; + remote_window_ = tfc->remote_window(); + target_window_ = tfc->target_window(); + announced_window_ = tfc->announced_window(); + if (sfc != nullptr) { + remote_window_delta_ = sfc->remote_window_delta(); + local_window_delta_ = sfc->local_window_delta(); + announced_window_delta_ = sfc->announced_window_delta(); + } +} + +void FlowControlTrace::Finish() { + uint32_t acked_local_window = + tfc_->transport()->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; + uint32_t remote_window = + tfc_->transport()->settings[GRPC_PEER_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; + char* trw_str = fmt_int64_diff_str(remote_window_, tfc_->remote_window()); + char* tlw_str = fmt_int64_diff_str(target_window_, tfc_->target_window()); + char* taw_str = + fmt_int64_diff_str(announced_window_, tfc_->announced_window()); + char* srw_str; + char* slw_str; + char* saw_str; + if (sfc_ != nullptr) { + srw_str = fmt_int64_diff_str(remote_window_delta_ + remote_window, + sfc_->remote_window_delta() + remote_window); + slw_str = + fmt_int64_diff_str(local_window_delta_ + acked_local_window, + sfc_->local_window_delta() + acked_local_window); + saw_str = + fmt_int64_diff_str(announced_window_delta_ + acked_local_window, + sfc_->announced_window_delta() + acked_local_window); + } else { + srw_str = gpr_leftpad("", ' ', kTracePadding); + slw_str = gpr_leftpad("", ' ', kTracePadding); + saw_str = gpr_leftpad("", ' ', kTracePadding); + } + gpr_log(GPR_DEBUG, + "%p[%u][%s] | %s | trw:%s, tlw:%s, taw:%s, srw:%s, slw:%s, saw:%s", + tfc_, sfc_ != nullptr ? sfc_->stream()->id : 0, + tfc_->transport()->is_client ? "cli" : "svr", reason_, trw_str, + tlw_str, taw_str, srw_str, slw_str, saw_str); + gpr_free(trw_str); + gpr_free(tlw_str); + gpr_free(taw_str); + gpr_free(srw_str); + gpr_free(slw_str); + gpr_free(saw_str); +} + const char* FlowControlAction::UrgencyString(Urgency u) { switch (u) { case Urgency::NO_ACTION_NEEDED: - return "no-action"; + return "no action"; case Urgency::UPDATE_IMMEDIATELY: - return "now"; + return "update immediately"; case Urgency::QUEUE_UPDATE: - return "queue"; + return "queue update"; default: GPR_UNREACHABLE_CODE(return "unknown"); } GPR_UNREACHABLE_CODE(return "unknown"); } -std::ostream& operator<<(std::ostream& out, FlowControlAction::Urgency u) { - return out << FlowControlAction::UrgencyString(u); +void FlowControlAction::Trace(grpc_chttp2_transport* t) const { + char* iw_str = fmt_uint32_diff_str( + t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], + initial_window_size_); + char* mf_str = fmt_uint32_diff_str( + t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], + max_frame_size_); + gpr_log(GPR_DEBUG, "t[%s], s[%s], iw:%s:%s mf:%s:%s", + UrgencyString(send_transport_update_), + UrgencyString(send_stream_update_), + UrgencyString(send_initial_window_update_), iw_str, + UrgencyString(send_max_frame_size_update_), mf_str); + gpr_free(iw_str); + gpr_free(mf_str); } -std::string FlowControlAction::DebugString() const { - std::vector segments; - if (send_transport_update_ != Urgency::NO_ACTION_NEEDED) { - segments.push_back( - absl::StrCat("t:", UrgencyString(send_transport_update_))); - } - if (send_stream_update_ != Urgency::NO_ACTION_NEEDED) { - segments.push_back(absl::StrCat("s:", UrgencyString(send_stream_update_))); - } - if (send_initial_window_update_ != Urgency::NO_ACTION_NEEDED) { - segments.push_back( - absl::StrCat("iw=", initial_window_size_, ":", - UrgencyString(send_initial_window_update_))); - } - if (send_max_frame_size_update_ != Urgency::NO_ACTION_NEEDED) { - segments.push_back( - absl::StrCat("mf=", max_frame_size_, ":", - UrgencyString(send_max_frame_size_update_))); - } - if (segments.empty()) return "no action"; - return absl::StrJoin(segments, ","); +TransportFlowControlDisabled::TransportFlowControlDisabled( + grpc_chttp2_transport* t) { + remote_window_ = kMaxWindow; + target_initial_window_size_ = kMaxWindow; + announced_window_ = kMaxWindow; + t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] = + kFrameSize; + t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] = + kFrameSize; + t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] = + kFrameSize; + t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] = + kMaxWindow; + t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] = + kMaxWindow; + t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] = + kMaxWindow; } -std::ostream& operator<<(std::ostream& out, const FlowControlAction& action) { - return out << action.DebugString(); -} - -TransportFlowControl::TransportFlowControl(const char* name, - bool enable_bdp_probe, - MemoryOwner* memory_owner) - : memory_owner_(memory_owner), +TransportFlowControl::TransportFlowControl(const grpc_chttp2_transport* t, + bool enable_bdp_probe) + : t_(t), enable_bdp_probe_(enable_bdp_probe), - bdp_estimator_(name), + bdp_estimator_(t->peer_string.c_str()), pid_controller_(PidController::Args() .set_gain_p(4) .set_gain_i(8) @@ -117,6 +196,7 @@ TransportFlowControl::TransportFlowControl(const char* name, last_pid_update_(ExecCtx::Get()->Now()) {} uint32_t TransportFlowControl::MaybeSendUpdate(bool writing_anyway) { + FlowControlTrace trace("t updt sent", this, nullptr); const uint32_t target_announced_window = static_cast(target_window()); if ((writing_anyway || announced_window_ <= target_announced_window / 2) && @@ -130,47 +210,80 @@ uint32_t TransportFlowControl::MaybeSendUpdate(bool writing_anyway) { return 0; } -absl::Status TransportFlowControl::ValidateRecvData( +grpc_error_handle TransportFlowControl::ValidateRecvData( int64_t incoming_frame_size) { if (incoming_frame_size > announced_window_) { - return absl::InternalError(absl::StrFormat( + return GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( "frame of size %" PRId64 " overflows local window of %" PRId64, incoming_frame_size, announced_window_)); } - return absl::OkStatus(); + return GRPC_ERROR_NONE; } -void TransportFlowControl::CommitRecvData(int64_t incoming_frame_size) { - announced_window_ -= incoming_frame_size; -} +StreamFlowControl::StreamFlowControl(TransportFlowControl* tfc, + const grpc_chttp2_stream* s) + : tfc_(tfc), s_(s) {} -StreamFlowControl::StreamFlowControl(TransportFlowControl* tfc) : tfc_(tfc) {} +grpc_error_handle StreamFlowControl::RecvData(int64_t incoming_frame_size) { + FlowControlTrace trace(" data recv", tfc_, this); -absl::Status StreamFlowControl::RecvData(int64_t incoming_frame_size) { - absl::Status error = tfc_->ValidateRecvData(incoming_frame_size); - if (!error.ok()) return error; + grpc_error_handle error = GRPC_ERROR_NONE; + error = tfc_->ValidateRecvData(incoming_frame_size); + if (error != GRPC_ERROR_NONE) return error; - int64_t acked_stream_window = - announced_window_delta_ + tfc_->acked_init_window(); + uint32_t sent_init_window = + tfc_->transport()->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; + uint32_t acked_init_window = + tfc_->transport()->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; + + int64_t acked_stream_window = announced_window_delta_ + acked_init_window; + int64_t sent_stream_window = announced_window_delta_ + sent_init_window; if (incoming_frame_size > acked_stream_window) { - return absl::InternalError(absl::StrFormat( - "frame of size %" PRId64 " overflows local window of %" PRId64, - incoming_frame_size, acked_stream_window)); + if (incoming_frame_size <= sent_stream_window) { + gpr_log(GPR_ERROR, + "Incoming frame of size %" PRId64 + " exceeds local window size of %" PRId64 + ".\n" + "The (un-acked, future) window size would be %" PRId64 + " which is not exceeded.\n" + "This would usually cause a disconnection, but allowing it due to" + "broken HTTP2 implementations in the wild.\n" + "See (for example) https://github.com/netty/netty/issues/6520.", + incoming_frame_size, acked_stream_window, sent_stream_window); + } else { + return GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( + "frame of size %" PRId64 " overflows local window of %" PRId64, + incoming_frame_size, acked_stream_window)); + } } UpdateAnnouncedWindowDelta(tfc_, -incoming_frame_size); local_window_delta_ -= incoming_frame_size; - min_progress_size_ -= - std::min(static_cast(min_progress_size_), incoming_frame_size); tfc_->CommitRecvData(incoming_frame_size); - return absl::OkStatus(); + return GRPC_ERROR_NONE; } uint32_t StreamFlowControl::MaybeSendUpdate() { + FlowControlTrace trace("s updt sent", tfc_, this); // If a recently sent settings frame caused the stream's flow control window - // to go in the negative (or < min_progress_size_), update the delta. - // In this case, we want to make sure that bytes are still flowing. - UpdateProgress(min_progress_size_); + // to go in the negative (or < GRPC_HEADER_SIZE_IN_BYTES), update the delta if + // one of the following conditions is satisfied - + // 1) There is a pending byte_stream and higher layers have expressed interest + // in reading additional data through the invokation of `Next()` where the + // bytes are to be available asynchronously. 2) There is a pending + // recv_message op. + // In these cases, we want to make sure that bytes are still flowing. + if (local_window_delta_ < GRPC_HEADER_SIZE_IN_BYTES) { + if (s_->on_next != nullptr) { + GPR_DEBUG_ASSERT(s_->pending_byte_stream); + IncomingByteStreamUpdate(GRPC_HEADER_SIZE_IN_BYTES, 0); + } else if (s_->recv_message != nullptr) { + IncomingByteStreamUpdate(GRPC_HEADER_SIZE_IN_BYTES, + s_->frame_storage.length); + } + } if (local_window_delta_ > announced_window_delta_) { uint32_t announce = static_cast( Clamp(local_window_delta_ - announced_window_delta_, int64_t(0), @@ -181,21 +294,32 @@ uint32_t StreamFlowControl::MaybeSendUpdate() { return 0; } -void StreamFlowControl::UpdateProgress(uint32_t min_progress_size) { +void StreamFlowControl::IncomingByteStreamUpdate(size_t max_size_hint, + size_t have_already) { + FlowControlTrace trace("app st recv", tfc_, this); uint32_t max_recv_bytes; - min_progress_size_ = min_progress_size; - /* clamp max recv hint to an allowable size */ - if (min_progress_size >= kMaxWindowDelta) { + if (max_size_hint >= kMaxWindowDelta) { max_recv_bytes = kMaxWindowDelta; } else { - max_recv_bytes = static_cast(min_progress_size); + max_recv_bytes = static_cast(max_size_hint); + } + + /* account for bytes already received but unknown to higher layers */ + if (max_recv_bytes >= have_already) { + max_recv_bytes -= static_cast(have_already); + } else { + max_recv_bytes = 0; } /* add some small lookahead to keep pipelines flowing */ - GPR_DEBUG_ASSERT(max_recv_bytes <= - kMaxWindowUpdateSize - tfc_->sent_init_window()); + GPR_DEBUG_ASSERT( + max_recv_bytes <= + kMaxWindowUpdateSize - + tfc_->transport() + ->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); if (local_window_delta_ < max_recv_bytes) { uint32_t add_max_recv_bytes = static_cast(max_recv_bytes - local_window_delta_); @@ -203,32 +327,6 @@ void StreamFlowControl::UpdateProgress(uint32_t min_progress_size) { } } -absl::Status TransportFlowControl::RecvData(int64_t incoming_frame_size) { - absl::Status error = ValidateRecvData(incoming_frame_size); - if (error.ok()) return error; - CommitRecvData(incoming_frame_size); - return absl::OkStatus(); -} - -void TransportFlowControl::RecvUpdate(uint32_t size) { remote_window_ += size; } - -int64_t TransportFlowControl::target_window() const { - // See comment above announced_stream_total_over_incoming_window_ for the - // logic behind this decision. - return static_cast( - std::min(static_cast((1u << 31) - 1), - announced_stream_total_over_incoming_window_ + - target_initial_window_size_)); -} - -FlowControlAction TransportFlowControl::UpdateAction(FlowControlAction action) { - if (announced_window_ < target_window() / 2) { - action.set_send_transport_update( - FlowControlAction::Urgency::UPDATE_IMMEDIATELY); - } - return action; -} - // Take in a target and modifies it based on the memory pressure of the system static double AdjustForMemoryPressure(double memory_pressure, double target) { // do not increase window under heavy memory pressure. @@ -247,9 +345,10 @@ static double AdjustForMemoryPressure(double memory_pressure, double target) { } double TransportFlowControl::TargetLogBdp() { - return AdjustForMemoryPressure( - memory_owner_->is_valid() ? memory_owner_->InstantaneousPressure() : 0.0, - 1 + log2(bdp_estimator_.EstimateBdp())); + return AdjustForMemoryPressure(t_->memory_owner.is_valid() + ? t_->memory_owner.InstantaneousPressure() + : 0.0, + 1 + log2(bdp_estimator_.EstimateBdp())); } double TransportFlowControl::SmoothLogBdp(double value) { @@ -262,17 +361,15 @@ double TransportFlowControl::SmoothLogBdp(double value) { return pid_controller_.Update(bdp_error, dt > kMaxDt ? kMaxDt : dt); } -void TransportFlowControl::UpdateSetting( - int64_t* desired_value, int64_t new_desired_value, - FlowControlAction* action, - FlowControlAction& (FlowControlAction::*set)(FlowControlAction::Urgency, - uint32_t)) { - int64_t delta = new_desired_value - *desired_value; +FlowControlAction::Urgency TransportFlowControl::DeltaUrgency( + int64_t value, grpc_chttp2_setting_id setting_id) { + int64_t delta = value - static_cast( + t_->settings[GRPC_LOCAL_SETTINGS][setting_id]); // TODO(ncteisen): tune this - if (delta != 0 && - (delta <= -*desired_value / 5 || delta >= *desired_value / 5)) { - *desired_value = new_desired_value; - (action->*set)(FlowControlAction::Urgency::QUEUE_UPDATE, *desired_value); + if (delta != 0 && (delta <= -value / 5 || delta >= value / 5)) { + return FlowControlAction::Urgency::QUEUE_UPDATE; + } else { + return FlowControlAction::Urgency::NO_ACTION_NEEDED; } } @@ -292,51 +389,46 @@ FlowControlAction TransportFlowControl::PeriodicUpdate() { } // Though initial window 'could' drop to 0, we keep the floor at // kMinInitialWindowSize - UpdateSetting( - &target_initial_window_size_, - static_cast(Clamp(target, double(kMinInitialWindowSize), - double(kMaxInitialWindowSize))), - &action, &FlowControlAction::set_send_initial_window_update); + target_initial_window_size_ = static_cast(Clamp( + target, double(kMinInitialWindowSize), double(kMaxInitialWindowSize))); + action.set_send_initial_window_update( + DeltaUrgency(target_initial_window_size_, + GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE), + static_cast(target_initial_window_size_)); // get bandwidth estimate and update max_frame accordingly. double bw_dbl = bdp_estimator_.EstimateBandwidth(); // we target the max of BDP or bandwidth in microseconds. - UpdateSetting( - &target_frame_size_, - static_cast(Clamp( - std::max(static_cast(Clamp(bw_dbl, 0.0, double(INT_MAX))) / - 1000, - static_cast(target_initial_window_size_)), - 16384, 16777215)), - &action, &FlowControlAction::set_send_max_frame_size_update); + int32_t frame_size = static_cast(Clamp( + std::max( + static_cast(Clamp(bw_dbl, 0.0, double(INT_MAX))) / 1000, + static_cast(target_initial_window_size_)), + 16384, 16777215)); + action.set_send_max_frame_size_update( + DeltaUrgency(static_cast(frame_size), + GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE), + frame_size); } return UpdateAction(action); } FlowControlAction StreamFlowControl::UpdateAction(FlowControlAction action) { - const uint32_t sent_init_window = tfc_->sent_init_window(); - if (local_window_delta_ > announced_window_delta_ && - announced_window_delta_ + sent_init_window <= sent_init_window / 2) { - action.set_send_stream_update( - FlowControlAction::Urgency::UPDATE_IMMEDIATELY); - } else if (local_window_delta_ > announced_window_delta_) { - action.set_send_stream_update(FlowControlAction::Urgency::QUEUE_UPDATE); + // TODO(ncteisen): tune this + if (!s_->read_closed) { + uint32_t sent_init_window = + tfc_->transport()->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; + if (local_window_delta_ > announced_window_delta_ && + announced_window_delta_ + sent_init_window <= sent_init_window / 2) { + action.set_send_stream_update( + FlowControlAction::Urgency::UPDATE_IMMEDIATELY); + } else if (local_window_delta_ > announced_window_delta_) { + action.set_send_stream_update(FlowControlAction::Urgency::QUEUE_UPDATE); + } } return action; } -void StreamFlowControl::UpdateAnnouncedWindowDelta(TransportFlowControl* tfc, - int64_t change) { - tfc->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_); - announced_window_delta_ += change; - tfc->PostUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_); -} - -void StreamFlowControl::SentData(int64_t outgoing_frame_size) { - tfc_->StreamSentData(outgoing_frame_size); - remote_window_delta_ -= outgoing_frame_size; -} - } // namespace chttp2 } // namespace grpc_core diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h index bc6fc7f2be8..185f18b38ca 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.h +++ b/src/core/ext/transport/chttp2/transport/flow_control.h @@ -22,18 +22,20 @@ #include #include +#include -#include -#include - -#include "absl/status/status.h" +#include +#include "src/core/ext/transport/chttp2/transport/http2_settings.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/time.h" -#include "src/core/lib/resource_quota/memory_quota.h" +#include "src/core/lib/iomgr/error.h" #include "src/core/lib/transport/bdp_estimator.h" #include "src/core/lib/transport/pid_controller.h" +struct grpc_chttp2_transport; +struct grpc_chttp2_stream; + extern grpc_core::TraceFlag grpc_flowctl_trace; namespace grpc { @@ -46,7 +48,6 @@ namespace grpc_core { namespace chttp2 { static constexpr uint32_t kDefaultWindow = 65535; -static constexpr uint32_t kDefaultFrameSize = 16384; static constexpr int64_t kMaxWindow = static_cast((1u << 31) - 1); // TODO(ncteisen): Tune this static constexpr uint32_t kFrameSize = 1024 * 1024; @@ -108,16 +109,7 @@ class FlowControlAction { } static const char* UrgencyString(Urgency u); - std::string DebugString() const; - - bool operator==(const FlowControlAction& other) const { - return send_stream_update_ == other.send_stream_update_ && - send_transport_update_ == other.send_transport_update_ && - send_initial_window_update_ == other.send_initial_window_update_ && - send_max_frame_size_update_ == other.send_max_frame_size_update_ && - initial_window_size_ == other.initial_window_size_ && - max_frame_size_ == other.max_frame_size_; - } + void Trace(grpc_chttp2_transport* t) const; private: Urgency send_stream_update_ = Urgency::NO_ACTION_NEEDED; @@ -128,16 +120,120 @@ class FlowControlAction { uint32_t max_frame_size_ = 0; }; -std::ostream& operator<<(std::ostream& out, FlowControlAction::Urgency urgency); -std::ostream& operator<<(std::ostream& out, const FlowControlAction& action); +class FlowControlTrace { + public: + FlowControlTrace(const char* reason, TransportFlowControl* tfc, + StreamFlowControl* sfc) { + if (enabled_) Init(reason, tfc, sfc); + } + + ~FlowControlTrace() { + if (enabled_) Finish(); + } + + private: + void Init(const char* reason, TransportFlowControl* tfc, + StreamFlowControl* sfc); + void Finish(); + + const bool enabled_ = GRPC_TRACE_FLAG_ENABLED(grpc_flowctl_trace); + + TransportFlowControl* tfc_; + StreamFlowControl* sfc_; + const char* reason_; + int64_t remote_window_; + int64_t target_window_; + int64_t announced_window_; + int64_t remote_window_delta_; + int64_t local_window_delta_; + int64_t announced_window_delta_; +}; + +// Fat interface with all methods a flow control implementation needs to +// support. +class TransportFlowControlBase { + public: + TransportFlowControlBase() {} + virtual ~TransportFlowControlBase() {} + + // Is flow control enabled? This is needed in other codepaths like the checks + // in parsing and in writing. + virtual bool flow_control_enabled() const = 0; + + // Called to check if the transport needs to send a WINDOW_UPDATE frame + virtual uint32_t MaybeSendUpdate(bool /* writing_anyway */) = 0; + + // Using the protected members, returns and Action to be taken by the + // tranport. + virtual FlowControlAction MakeAction() = 0; + + // Using the protected members, returns and Action to be taken by the + // tranport. Also checks for updates to our BDP estimate and acts + // accordingly. + virtual FlowControlAction PeriodicUpdate() = 0; + + // Called to do bookkeeping when a stream owned by this transport sends + // data on the wire + virtual void StreamSentData(int64_t /* size */) = 0; + + // Called to do bookkeeping when a stream owned by this transport receives + // data from the wire. Also does error checking for frame size. + virtual grpc_error_handle RecvData(int64_t /* incoming_frame_size */) = 0; + + // Called to do bookkeeping when we receive a WINDOW_UPDATE frame. + virtual void RecvUpdate(uint32_t /* size */) = 0; + + // Returns the BdpEstimator held by this object. Caller is responsible for + // checking for nullptr. TODO(ncteisen): consider fully encapsulating all + // bdp estimator actions inside TransportFlowControl + virtual BdpEstimator* bdp_estimator() { return nullptr; } + + // Getters + int64_t remote_window() const { return remote_window_; } + virtual int64_t target_window() const { return target_initial_window_size_; } + int64_t announced_window() const { return announced_window_; } + + // Used in certain benchmarks in which we don't want FlowControl to be a + // factor + virtual void TestOnlyForceHugeWindow() {} + + protected: + friend class grpc::testing::TrickledCHTTP2; + int64_t remote_window_ = kDefaultWindow; + int64_t target_initial_window_size_ = kDefaultWindow; + int64_t announced_window_ = kDefaultWindow; +}; + +// Implementation of flow control that does NOTHING. Always returns maximum +// values, never initiates writes, and assumes that the remote peer is doing +// the same. To be used to narrow down on flow control as the cause of negative +// performance. +class TransportFlowControlDisabled final : public TransportFlowControlBase { + public: + // Maxes out all values + explicit TransportFlowControlDisabled(grpc_chttp2_transport* t); + + bool flow_control_enabled() const override { return false; } + + // Never do anything. + uint32_t MaybeSendUpdate(bool /* writing_anyway */) override { return 0; } + FlowControlAction MakeAction() override { return FlowControlAction(); } + FlowControlAction PeriodicUpdate() override { return FlowControlAction(); } + void StreamSentData(int64_t /* size */) override {} + grpc_error_handle RecvData(int64_t /* incoming_frame_size */) override { + return GRPC_ERROR_NONE; + } + void RecvUpdate(uint32_t /* size */) override {} +}; // Implementation of flow control that abides to HTTP/2 spec and attempts // to be as performant as possible. -class TransportFlowControl final { +class TransportFlowControl final : public TransportFlowControlBase { public: - explicit TransportFlowControl(const char* name, bool enable_bdp_probe, - MemoryOwner* memory_owner); - ~TransportFlowControl() {} + TransportFlowControl(const grpc_chttp2_transport* t, bool enable_bdp_probe); + ~TransportFlowControl() override {} + + bool flow_control_enabled() const override { return true; } bool bdp_probe() const { return enable_bdp_probe_; } @@ -145,30 +241,50 @@ class TransportFlowControl final { // else returns zero; writing_anyway indicates if a write would happen // regardless of the send - if it is false and this function returns non-zero, // this announce will cause a write to occur - uint32_t MaybeSendUpdate(bool writing_anyway); + uint32_t MaybeSendUpdate(bool writing_anyway) override; // Reads the flow control data and returns and actionable struct that will // tell chttp2 exactly what it needs to do - FlowControlAction MakeAction() { return UpdateAction(FlowControlAction()); } + FlowControlAction MakeAction() override { + return UpdateAction(FlowControlAction()); + } // Call periodically (at a low-ish rate, 100ms - 10s makes sense) // to perform more complex flow control calculations and return an action // to let chttp2 change its parameters - FlowControlAction PeriodicUpdate(); + FlowControlAction PeriodicUpdate() override; - void StreamSentData(int64_t size) { remote_window_ -= size; } + void StreamSentData(int64_t size) override { remote_window_ -= size; } - absl::Status ValidateRecvData(int64_t incoming_frame_size); - void CommitRecvData(int64_t incoming_frame_size); + grpc_error_handle ValidateRecvData(int64_t incoming_frame_size); + void CommitRecvData(int64_t incoming_frame_size) { + announced_window_ -= incoming_frame_size; + } - absl::Status RecvData(int64_t incoming_frame_size); + grpc_error_handle RecvData(int64_t incoming_frame_size) override { + FlowControlTrace trace(" data recv", this, nullptr); + grpc_error_handle error = ValidateRecvData(incoming_frame_size); + if (error != GRPC_ERROR_NONE) return error; + CommitRecvData(incoming_frame_size); + return GRPC_ERROR_NONE; + } // we have received a WINDOW_UPDATE frame for a transport - void RecvUpdate(uint32_t size); + void RecvUpdate(uint32_t size) override { + FlowControlTrace trace("t updt recv", this, nullptr); + remote_window_ += size; + } - int64_t target_window() const; + // See comment above announced_stream_total_over_incoming_window_ for the + // logic behind this decision. + int64_t target_window() const override { + return static_cast( + std::min(static_cast((1u << 31) - 1), + announced_stream_total_over_incoming_window_ + + target_initial_window_size_)); + } - int64_t target_frame_size() const { return target_frame_size_; } + const grpc_chttp2_transport* transport() const { return t_; } void PreUpdateAnnouncedWindowOverIncomingWindow(int64_t delta) { if (delta > 0) { @@ -182,34 +298,28 @@ class TransportFlowControl final { } } - BdpEstimator* bdp_estimator() { return &bdp_estimator_; } + BdpEstimator* bdp_estimator() override { return &bdp_estimator_; } - void TestOnlyForceHugeWindow() { + void TestOnlyForceHugeWindow() override { announced_window_ = 1024 * 1024 * 1024; remote_window_ = 1024 * 1024 * 1024; } - uint32_t acked_init_window() const { return acked_init_window_; } - uint32_t sent_init_window() const { return sent_init_window_; } - - void SetSentInitialWindow(uint32_t value) { sent_init_window_ = value; } - void SetAckedInitialWindow(uint32_t value) { acked_init_window_ = value; } - - // Getters - int64_t remote_window() const { return remote_window_; } - int64_t announced_window() const { return announced_window_; } - private: double TargetLogBdp(); double SmoothLogBdp(double value); - static void UpdateSetting(int64_t* desired_value, int64_t new_desired_value, - FlowControlAction* action, - FlowControlAction& (FlowControlAction::*set)( - FlowControlAction::Urgency, uint32_t)); + FlowControlAction::Urgency DeltaUrgency(int64_t value, + grpc_chttp2_setting_id setting_id); - FlowControlAction UpdateAction(FlowControlAction action); + FlowControlAction UpdateAction(FlowControlAction action) { + if (announced_window_ < target_window() / 2) { + action.set_send_transport_update( + FlowControlAction::Urgency::UPDATE_IMMEDIATELY); + } + return action; + } - MemoryOwner* const memory_owner_; + const grpc_chttp2_transport* const t_; /** calculating what we should give for local window: we track the total amount of flow control over initial window size @@ -230,50 +340,126 @@ class TransportFlowControl final { /* pid controller */ PidController pid_controller_; Timestamp last_pid_update_; +}; - int64_t remote_window_ = kDefaultWindow; - int64_t target_initial_window_size_ = kDefaultWindow; - int64_t target_frame_size_ = kDefaultFrameSize; - int64_t announced_window_ = kDefaultWindow; - uint32_t sent_init_window_ = kDefaultWindow; - uint32_t acked_init_window_ = kDefaultWindow; +// Fat interface with all methods a stream flow control implementation needs +// to support. +class StreamFlowControlBase { + public: + StreamFlowControlBase() {} + virtual ~StreamFlowControlBase() {} + + // Updates an action using the protected members. + virtual FlowControlAction UpdateAction(FlowControlAction /* action */) { + abort(); + } + + // Using the protected members, returns an Action for this stream to be + // taken by the tranport. + virtual FlowControlAction MakeAction() = 0; + + // Bookkeeping for when data is sent on this stream. + virtual void SentData(int64_t /* outgoing_frame_size */) = 0; + + // Bookkeeping and error checking for when data is received by this stream. + virtual grpc_error_handle RecvData(int64_t /* incoming_frame_size */) = 0; + + // Called to check if this stream needs to send a WINDOW_UPDATE frame. + virtual uint32_t MaybeSendUpdate() = 0; + + // Bookkeeping for receiving a WINDOW_UPDATE from for this stream. + virtual void RecvUpdate(uint32_t /* size */) = 0; + + // Bookkeeping for when a call pulls bytes out of the transport. At this + // point we consider the data 'used' and can thus let out peer know we are + // ready for more data. + virtual void IncomingByteStreamUpdate(size_t /* max_size_hint */, + size_t /* have_already */) { + abort(); + } + + // Used in certain benchmarks in which we don't want FlowControl to be a + // factor + virtual void TestOnlyForceHugeWindow() {} + + // Getters + int64_t remote_window_delta() const { return remote_window_delta_; } + int64_t local_window_delta() const { return local_window_delta_; } + int64_t announced_window_delta() const { return announced_window_delta_; } + + protected: + friend class grpc::testing::TrickledCHTTP2; + int64_t remote_window_delta_ = 0; + int64_t local_window_delta_ = 0; + int64_t announced_window_delta_ = 0; +}; + +// Implementation of flow control that does NOTHING. Always returns maximum +// values, never initiates writes, and assumes that the remote peer is doing +// the same. To be used to narrow down on flow control as the cause of negative +// performance. +class StreamFlowControlDisabled : public StreamFlowControlBase { + public: + FlowControlAction UpdateAction(FlowControlAction action) override { + return action; + } + FlowControlAction MakeAction() override { return FlowControlAction(); } + void SentData(int64_t /* outgoing_frame_size */) override {} + grpc_error_handle RecvData(int64_t /* incoming_frame_size */) override { + return GRPC_ERROR_NONE; + } + uint32_t MaybeSendUpdate() override { return 0; } + void RecvUpdate(uint32_t /* size */) override {} + void IncomingByteStreamUpdate(size_t /* max_size_hint */, + size_t /* have_already */) override {} }; // Implementation of flow control that abides to HTTP/2 spec and attempts // to be as performant as possible. -class StreamFlowControl final { +class StreamFlowControl final : public StreamFlowControlBase { public: - explicit StreamFlowControl(TransportFlowControl* tfc); - ~StreamFlowControl() { + StreamFlowControl(TransportFlowControl* tfc, const grpc_chttp2_stream* s); + ~StreamFlowControl() override { tfc_->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_); } - FlowControlAction UpdateAction(FlowControlAction action); - FlowControlAction MakeAction() { return UpdateAction(tfc_->MakeAction()); } + FlowControlAction UpdateAction(FlowControlAction action) override; + FlowControlAction MakeAction() override { + return UpdateAction(tfc_->MakeAction()); + } // we have sent data on the wire, we must track this in our bookkeeping for // the remote peer's flow control. - void SentData(int64_t outgoing_frame_size); + void SentData(int64_t outgoing_frame_size) override { + FlowControlTrace tracer(" data sent", tfc_, this); + tfc_->StreamSentData(outgoing_frame_size); + remote_window_delta_ -= outgoing_frame_size; + } // we have received data from the wire - absl::Status RecvData(int64_t incoming_frame_size); + grpc_error_handle RecvData(int64_t incoming_frame_size) override; // returns an announce if we should send a stream update to our peer, else // returns zero - uint32_t MaybeSendUpdate(); + uint32_t MaybeSendUpdate() override; // we have received a WINDOW_UPDATE frame for a stream - void RecvUpdate(uint32_t size) { remote_window_delta_ += size; } + void RecvUpdate(uint32_t size) override { + FlowControlTrace trace("s updt recv", tfc_, this); + remote_window_delta_ += size; + } // the application is asking for a certain amount of bytes - void UpdateProgress(uint32_t min_progress_size); + void IncomingByteStreamUpdate(size_t max_size_hint, + size_t have_already) override; int64_t remote_window_delta() const { return remote_window_delta_; } int64_t local_window_delta() const { return local_window_delta_; } int64_t announced_window_delta() const { return announced_window_delta_; } - uint32_t min_progress_size() const { return min_progress_size_; } - void TestOnlyForceHugeWindow() { + const grpc_chttp2_stream* stream() const { return s_; } + + void TestOnlyForceHugeWindow() override { announced_window_delta_ = 1024 * 1024 * 1024; local_window_delta_ = 1024 * 1024 * 1024; remote_window_delta_ = 1024 * 1024 * 1024; @@ -281,12 +467,13 @@ class StreamFlowControl final { private: TransportFlowControl* const tfc_; - uint32_t min_progress_size_ = 0; - int64_t remote_window_delta_ = 0; - int64_t local_window_delta_ = 0; - int64_t announced_window_delta_ = 0; + const grpc_chttp2_stream* const s_; - void UpdateAnnouncedWindowDelta(TransportFlowControl* tfc, int64_t change); + void UpdateAnnouncedWindowDelta(TransportFlowControl* tfc, int64_t change) { + tfc->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_); + announced_window_delta_ += change; + tfc->PostUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_); + } }; class TestOnlyTransportTargetWindowEstimatesMocker { diff --git a/src/core/ext/transport/chttp2/transport/frame_data.cc b/src/core/ext/transport/chttp2/transport/frame_data.cc index 49a83ed6d35..b3eb18c1d4b 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.cc +++ b/src/core/ext/transport/chttp2/transport/frame_data.cc @@ -20,24 +20,42 @@ #include "src/core/ext/transport/chttp2/transport/frame_data.h" -#include +#include +#include "absl/base/attributes.h" #include "absl/strings/str_format.h" #include #include #include "src/core/ext/transport/chttp2/transport/internal.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/channel/channelz.h" +#include "src/core/lib/gpr/string.h" +#include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/memory.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_refcount.h" +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/transport/transport.h" -absl::Status grpc_chttp2_data_parser_begin_frame(uint8_t flags, - uint32_t stream_id, - grpc_chttp2_stream* s) { +grpc_chttp2_data_parser::~grpc_chttp2_data_parser() { + if (parsing_frame != nullptr) { + GRPC_ERROR_UNREF(parsing_frame->Finished( + GRPC_ERROR_CREATE_FROM_STATIC_STRING("Parser destroyed"), false)); + } + GRPC_ERROR_UNREF(error); +} + +grpc_error_handle grpc_chttp2_data_parser_begin_frame( + grpc_chttp2_data_parser* /*parser*/, uint8_t flags, uint32_t stream_id, + grpc_chttp2_stream* s) { if (flags & ~GRPC_CHTTP2_DATA_FLAG_END_STREAM) { - return absl::InternalError(absl::StrFormat( - "unsupported data flags: 0x%02x stream: %d", flags, stream_id)); + return grpc_error_set_int(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( + "unsupported data flags: 0x%02x", flags)), + GRPC_ERROR_INT_STREAM_ID, + static_cast(stream_id)); } if (flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM) { @@ -47,7 +65,7 @@ absl::Status grpc_chttp2_data_parser_begin_frame(uint8_t flags, s->received_last_frame = false; } - return absl::OkStatus(); + return GRPC_ERROR_NONE; } void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer* inbuf, @@ -78,59 +96,188 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer* inbuf, stats->data_bytes += write_bytes; } -grpc_core::Poll grpc_deframe_unprocessed_incoming_frames( - grpc_chttp2_stream* s, uint32_t* min_progress_size, - grpc_core::SliceBuffer* stream_out, uint32_t* message_flags) { - grpc_slice_buffer* slices = &s->frame_storage; +grpc_error_handle grpc_deframe_unprocessed_incoming_frames( + grpc_chttp2_data_parser* p, grpc_chttp2_stream* s, + grpc_slice_buffer* slices, grpc_slice* slice_out, + grpc_core::OrphanablePtr* stream_out) { grpc_error_handle error = GRPC_ERROR_NONE; + grpc_chttp2_transport* t = s->t; - if (slices->length < 5) { - if (min_progress_size != nullptr) *min_progress_size = 5 - slices->length; - return grpc_core::Pending{}; - } - - uint8_t header[5]; - grpc_slice_buffer_copy_first_into_buffer(slices, 5, header); - - switch (header[0]) { - case 0: - if (message_flags != nullptr) *message_flags = 0; - break; - case 1: - if (message_flags != nullptr) { - *message_flags = GRPC_WRITE_INTERNAL_COMPRESS; - } - break; - default: - error = GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrFormat("Bad GRPC frame type 0x%02x", header[0])); - error = grpc_error_set_int(error, GRPC_ERROR_INT_STREAM_ID, - static_cast(s->id)); - return error; - } + while (slices->count > 0) { + uint8_t* beg = nullptr; + uint8_t* end = nullptr; + uint8_t* cur = nullptr; - uint32_t length = (static_cast(header[1]) << 24) | - (static_cast(header[2]) << 16) | - (static_cast(header[3]) << 8) | - static_cast(header[4]); + grpc_slice* slice = grpc_slice_buffer_peek_first(slices); + beg = GRPC_SLICE_START_PTR(*slice); + end = GRPC_SLICE_END_PTR(*slice); + cur = beg; + uint32_t message_flags; - if (slices->length < length + 5) { - if (min_progress_size != nullptr) { - *min_progress_size = length + 5 - slices->length; + if (cur == end) { + grpc_slice_buffer_remove_first(slices); + continue; } - return grpc_core::Pending{}; - } - if (min_progress_size != nullptr) *min_progress_size = 0; - - if (stream_out != nullptr) { - s->stats.incoming.framing_bytes += 5; - s->stats.incoming.data_bytes += length; - grpc_slice_buffer_move_first_into_buffer(slices, 5, header); - grpc_slice_buffer_move_first_no_ref(slices, length, - stream_out->c_slice_buffer()); + switch (p->state) { + case GRPC_CHTTP2_DATA_ERROR: + p->state = GRPC_CHTTP2_DATA_ERROR; + grpc_slice_buffer_remove_first(slices); + return GRPC_ERROR_REF(p->error); + case GRPC_CHTTP2_DATA_FH_0: + s->stats.incoming.framing_bytes++; + p->frame_type = *cur; + switch (p->frame_type) { + case 0: + p->is_frame_compressed = false; /* GPR_FALSE */ + break; + case 1: + p->is_frame_compressed = true; /* GPR_TRUE */ + break; + default: + p->error = GRPC_ERROR_CREATE_FROM_CPP_STRING( + absl::StrFormat("Bad GRPC frame type 0x%02x", p->frame_type)); + p->error = grpc_error_set_int(p->error, GRPC_ERROR_INT_STREAM_ID, + static_cast(s->id)); + grpc_core::UniquePtr dmp( + grpc_dump_slice(*slice, GPR_DUMP_HEX | GPR_DUMP_ASCII)); + p->error = grpc_error_set_str(p->error, GRPC_ERROR_STR_RAW_BYTES, + dmp.get()); + p->error = + grpc_error_set_int(p->error, GRPC_ERROR_INT_OFFSET, cur - beg); + p->state = GRPC_CHTTP2_DATA_ERROR; + grpc_slice_buffer_remove_first(slices); + return GRPC_ERROR_REF(p->error); + } + if (++cur == end) { + p->state = GRPC_CHTTP2_DATA_FH_1; + grpc_slice_buffer_remove_first(slices); + continue; + } + ABSL_FALLTHROUGH_INTENDED; + case GRPC_CHTTP2_DATA_FH_1: + s->stats.incoming.framing_bytes++; + p->frame_size = (static_cast(*cur)) << 24; + if (++cur == end) { + p->state = GRPC_CHTTP2_DATA_FH_2; + grpc_slice_buffer_remove_first(slices); + continue; + } + ABSL_FALLTHROUGH_INTENDED; + case GRPC_CHTTP2_DATA_FH_2: + s->stats.incoming.framing_bytes++; + p->frame_size |= (static_cast(*cur)) << 16; + if (++cur == end) { + p->state = GRPC_CHTTP2_DATA_FH_3; + grpc_slice_buffer_remove_first(slices); + continue; + } + ABSL_FALLTHROUGH_INTENDED; + case GRPC_CHTTP2_DATA_FH_3: + s->stats.incoming.framing_bytes++; + p->frame_size |= (static_cast(*cur)) << 8; + if (++cur == end) { + p->state = GRPC_CHTTP2_DATA_FH_4; + grpc_slice_buffer_remove_first(slices); + continue; + } + ABSL_FALLTHROUGH_INTENDED; + case GRPC_CHTTP2_DATA_FH_4: + s->stats.incoming.framing_bytes++; + GPR_ASSERT(stream_out != nullptr); + GPR_ASSERT(p->parsing_frame == nullptr); + p->frame_size |= (static_cast(*cur)); + if (t->channelz_socket != nullptr) { + t->channelz_socket->RecordMessageReceived(); + } + p->state = GRPC_CHTTP2_DATA_FRAME; + ++cur; + message_flags = 0; + if (p->is_frame_compressed) { + message_flags |= GRPC_WRITE_INTERNAL_COMPRESS; + } + p->parsing_frame = new grpc_core::Chttp2IncomingByteStream( + t, s, p->frame_size, message_flags); + stream_out->reset(p->parsing_frame); + if (p->parsing_frame->remaining_bytes() == 0) { + GRPC_ERROR_UNREF(p->parsing_frame->Finished(GRPC_ERROR_NONE, true)); + p->parsing_frame = nullptr; + p->state = GRPC_CHTTP2_DATA_FH_0; + } + s->pending_byte_stream = true; + if (cur != end) { + grpc_slice_buffer_sub_first(slices, static_cast(cur - beg), + static_cast(end - beg)); + } else { + grpc_slice_buffer_remove_first(slices); + } + return GRPC_ERROR_NONE; + case GRPC_CHTTP2_DATA_FRAME: { + GPR_ASSERT(p->parsing_frame != nullptr); + GPR_ASSERT(slice_out != nullptr); + if (cur == end) { + grpc_slice_buffer_remove_first(slices); + continue; + } + uint32_t remaining = static_cast(end - cur); + if (remaining == p->frame_size) { + s->stats.incoming.data_bytes += remaining; + if (GRPC_ERROR_NONE != + (error = p->parsing_frame->Push( + grpc_slice_sub(*slice, static_cast(cur - beg), + static_cast(end - beg)), + slice_out))) { + grpc_slice_buffer_remove_first(slices); + return error; + } + if (GRPC_ERROR_NONE != + (error = p->parsing_frame->Finished(GRPC_ERROR_NONE, true))) { + grpc_slice_buffer_remove_first(slices); + return error; + } + p->parsing_frame = nullptr; + p->state = GRPC_CHTTP2_DATA_FH_0; + grpc_slice_buffer_remove_first(slices); + return GRPC_ERROR_NONE; + } else if (remaining < p->frame_size) { + s->stats.incoming.data_bytes += remaining; + if (GRPC_ERROR_NONE != + (error = p->parsing_frame->Push( + grpc_slice_sub(*slice, static_cast(cur - beg), + static_cast(end - beg)), + slice_out))) { + return error; + } + p->frame_size -= remaining; + grpc_slice_buffer_remove_first(slices); + return GRPC_ERROR_NONE; + } else { + GPR_ASSERT(remaining > p->frame_size); + s->stats.incoming.data_bytes += p->frame_size; + if (GRPC_ERROR_NONE != + p->parsing_frame->Push( + grpc_slice_sub( + *slice, static_cast(cur - beg), + static_cast(cur + p->frame_size - beg)), + slice_out)) { + grpc_slice_buffer_remove_first(slices); + return error; + } + if (GRPC_ERROR_NONE != + (error = p->parsing_frame->Finished(GRPC_ERROR_NONE, true))) { + grpc_slice_buffer_remove_first(slices); + return error; + } + p->parsing_frame = nullptr; + p->state = GRPC_CHTTP2_DATA_FH_0; + cur += p->frame_size; + grpc_slice_buffer_sub_first(slices, static_cast(cur - beg), + static_cast(end - beg)); + return GRPC_ERROR_NONE; + } + } + } } - return GRPC_ERROR_NONE; } @@ -139,9 +286,20 @@ grpc_error_handle grpc_chttp2_data_parser_parse(void* /*parser*/, grpc_chttp2_stream* s, const grpc_slice& slice, int is_last) { - grpc_slice_ref_internal(slice); - grpc_slice_buffer_add(&s->frame_storage, slice); - grpc_chttp2_maybe_complete_recv_message(t, s); + if (!s->pending_byte_stream) { + grpc_slice_ref_internal(slice); + grpc_slice_buffer_add(&s->frame_storage, slice); + grpc_chttp2_maybe_complete_recv_message(t, s); + } else if (s->on_next) { + GPR_ASSERT(s->frame_storage.length == 0); + grpc_slice_ref_internal(slice); + grpc_slice_buffer_add(&s->unprocessed_incoming_frames_buffer, slice); + grpc_core::ExecCtx::Run(DEBUG_LOCATION, s->on_next, GRPC_ERROR_NONE); + s->on_next = nullptr; + } else { + grpc_slice_ref_internal(slice); + grpc_slice_buffer_add(&s->frame_storage, slice); + } if (is_last && s->received_last_frame) { grpc_chttp2_mark_stream_closed( diff --git a/src/core/ext/transport/chttp2/transport/frame_data.h b/src/core/ext/transport/chttp2/transport/frame_data.h index e1805edcb6a..d32bba83069 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.h +++ b/src/core/ext/transport/chttp2/transport/frame_data.h @@ -25,20 +25,45 @@ #include -#include "absl/status/status.h" - #include #include "src/core/ext/transport/chttp2/transport/frame.h" +#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/error.h" -#include "src/core/lib/promise/poll.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/transport.h" +typedef enum { + GRPC_CHTTP2_DATA_FH_0, + GRPC_CHTTP2_DATA_FH_1, + GRPC_CHTTP2_DATA_FH_2, + GRPC_CHTTP2_DATA_FH_3, + GRPC_CHTTP2_DATA_FH_4, + GRPC_CHTTP2_DATA_FRAME, + GRPC_CHTTP2_DATA_ERROR +} grpc_chttp2_stream_state; + +namespace grpc_core { +class Chttp2IncomingByteStream; +} // namespace grpc_core + +struct grpc_chttp2_data_parser { + grpc_chttp2_data_parser() = default; + ~grpc_chttp2_data_parser(); + + grpc_chttp2_stream_state state = GRPC_CHTTP2_DATA_FH_0; + uint8_t frame_type = 0; + uint32_t frame_size = 0; + grpc_error_handle error = GRPC_ERROR_NONE; + + bool is_frame_compressed = false; + grpc_core::Chttp2IncomingByteStream* parsing_frame = nullptr; +}; + /* start processing a new data frame */ -absl::Status grpc_chttp2_data_parser_begin_frame(uint8_t flags, - uint32_t stream_id, - grpc_chttp2_stream* s); +grpc_error_handle grpc_chttp2_data_parser_begin_frame( + grpc_chttp2_data_parser* parser, uint8_t flags, uint32_t stream_id, + grpc_chttp2_stream* s); /* handle a slice of a data frame - is_last indicates the last slice of a frame */ @@ -53,8 +78,9 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer* inbuf, grpc_transport_one_way_stats* stats, grpc_slice_buffer* outbuf); -grpc_core::Poll grpc_deframe_unprocessed_incoming_frames( - grpc_chttp2_stream* s, uint32_t* min_progress_size, - grpc_core::SliceBuffer* stream_out, uint32_t* message_flags); +grpc_error_handle grpc_deframe_unprocessed_incoming_frames( + grpc_chttp2_data_parser* p, grpc_chttp2_stream* s, + grpc_slice_buffer* slices, grpc_slice* slice_out, + grpc_core::OrphanablePtr* stream_out); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_DATA_H */ diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.cc b/src/core/ext/transport/chttp2/transport/frame_settings.cc index c049acedb9e..dfbfde9ee7e 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.cc +++ b/src/core/ext/transport/chttp2/transport/frame_settings.cc @@ -39,6 +39,7 @@ #include "src/core/lib/debug/trace.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/iomgr/exec_ctx.h" static uint8_t* fill_header(uint8_t* out, uint32_t length, uint8_t flags) { @@ -126,7 +127,7 @@ void StreamFlowControlWindowCheck(void* user_data, uint32_t /* key */, grpc_chttp2_stream* s = static_cast(stream); if ((s->t->settings[GRPC_PEER_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] + - s->t->initial_window_update + s->flow_control.remote_window_delta()) > + s->t->initial_window_update + s->flow_control->remote_window_delta()) > ((1u << 31) - 1)) { *error = true; } @@ -218,6 +219,12 @@ grpc_error_handle grpc_chttp2_settings_parser_parse(void* p, if (grpc_wire_id_to_setting_id(parser->id, &id)) { const grpc_chttp2_setting_parameters* sp = &grpc_chttp2_settings_parameters[id]; + // If flow control is disabled we skip these. + if (!t->flow_control->flow_control_enabled() && + (id == GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE || + id == GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE)) { + continue; + } if (parser->value < sp->min_value || parser->value > sp->max_value) { switch (sp->invalid_value_behavior) { case GRPC_CHTTP2_CLAMP_INVALID_VALUE: diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.cc b/src/core/ext/transport/chttp2/transport/frame_window_update.cc index c208d479749..790c654cb59 100644 --- a/src/core/ext/transport/chttp2/transport/frame_window_update.cc +++ b/src/core/ext/transport/chttp2/transport/frame_window_update.cc @@ -29,6 +29,7 @@ #include "src/core/ext/transport/chttp2/transport/flow_control.h" #include "src/core/ext/transport/chttp2/transport/internal.h" +#include "src/core/lib/gprpp/manual_constructor.h" grpc_slice grpc_chttp2_window_update_create( uint32_t id, uint32_t window_delta, grpc_transport_one_way_stats* stats) { @@ -97,10 +98,10 @@ grpc_error_handle grpc_chttp2_window_update_parser_parse( if (t->incoming_stream_id != 0) { if (s != nullptr) { - s->flow_control.RecvUpdate(received_update); + s->flow_control->RecvUpdate(received_update); if (grpc_core::chttp2:: g_test_only_transport_flow_control_window_check && - s->flow_control.remote_window_delta() > + s->flow_control->remote_window_delta() > grpc_core::chttp2::kMaxWindowDelta) { GPR_ASSERT(false); } @@ -111,9 +112,9 @@ grpc_error_handle grpc_chttp2_window_update_parser_parse( } } } else { - bool was_zero = t->flow_control.remote_window() <= 0; - t->flow_control.RecvUpdate(received_update); - bool is_zero = t->flow_control.remote_window() <= 0; + bool was_zero = t->flow_control->remote_window() <= 0; + t->flow_control->RecvUpdate(received_update); + bool is_zero = t->flow_control->remote_window() <= 0; if (was_zero && !is_zero) { grpc_chttp2_initiate_write( t, GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 4af3c4ec9db..d859766eadf 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -27,7 +27,6 @@ #include #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include #include @@ -35,6 +34,7 @@ #include "src/core/ext/transport/chttp2/transport/flow_control.h" #include "src/core/ext/transport/chttp2/transport/frame.h" +#include "src/core/ext/transport/chttp2/transport/frame_data.h" #include "src/core/ext/transport/chttp2/transport/frame_goaway.h" #include "src/core/ext/transport/chttp2/transport/frame_ping.h" #include "src/core/ext/transport/chttp2/transport/frame_rst_stream.h" @@ -48,6 +48,8 @@ #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/bitset.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/manual_constructor.h" +#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/time.h" @@ -58,7 +60,7 @@ #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/memory_quota.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -228,6 +230,76 @@ typedef struct grpc_chttp2_write_cb { struct grpc_chttp2_write_cb* next; } grpc_chttp2_write_cb; +namespace grpc_core { + +class Chttp2IncomingByteStream : public ByteStream { + public: + Chttp2IncomingByteStream(grpc_chttp2_transport* transport, + grpc_chttp2_stream* stream, uint32_t frame_size, + uint32_t flags); + + void Orphan() override; + + bool Next(size_t max_size_hint, grpc_closure* on_complete) override; + grpc_error_handle Pull(grpc_slice* slice) override; + void Shutdown(grpc_error_handle error) override; + + // TODO(roth): When I converted this class to C++, I wanted to make it + // inherit from RefCounted or InternallyRefCounted instead of continuing + // to use its own custom ref-counting code. However, that would require + // using multiple inheritance, which sucks in general. And to make matters + // worse, it causes problems with our New<> and Delete<> wrappers. + // Specifically, unless RefCounted is first in the list of parent classes, + // it will see a different value of the address of the object than the one + // we actually allocated, in which case gpr_free() will be called on a + // different address than the one we got from gpr_malloc(), thus causing a + // crash. Given the fragility of depending on that, as well as a desire to + // avoid multiple inheritance in general, I've decided to leave this + // alone for now. We can revisit this once we're able to link against + // libc++, at which point we can eliminate New<> and Delete<> and + // switch to std::shared_ptr<>. + void Ref() { refs_.Ref(); } + void Unref() { + if (GPR_UNLIKELY(refs_.Unref())) { + delete this; + } + } + + void PublishError(grpc_error_handle error); + + grpc_error_handle Push(const grpc_slice& slice, grpc_slice* slice_out); + + grpc_error_handle Finished(grpc_error_handle error, bool reset_on_error); + + uint32_t remaining_bytes() const { return remaining_bytes_; } + + private: + static void NextLocked(void* arg, grpc_error_handle error_ignored); + static void OrphanLocked(void* arg, grpc_error_handle error_ignored); + + grpc_chttp2_transport* transport_; // Immutable. + grpc_chttp2_stream* stream_; // Immutable. + + RefCount refs_; + + /* Accessed only by transport thread when stream->pending_byte_stream == false + * Accessed only by application thread when stream->pending_byte_stream == + * true */ + uint32_t remaining_bytes_; + + /* Accessed only by transport thread when stream->pending_byte_stream == false + * Accessed only by application thread when stream->pending_byte_stream == + * true */ + struct { + grpc_closure closure; + size_t max_size_hint; + grpc_closure* on_complete; + } next_action_; + grpc_closure destroy_action_; +}; + +} // namespace grpc_core + typedef enum { GRPC_CHTTP2_KEEPALIVE_STATE_WAITING, GRPC_CHTTP2_KEEPALIVE_STATE_PINGING, @@ -356,7 +428,11 @@ struct grpc_chttp2_transport { /** parser for goaway frames */ grpc_chttp2_goaway_parser goaway_parser; - grpc_core::chttp2::TransportFlowControl flow_control; + grpc_core::PolymorphicManualConstructor< + grpc_core::chttp2::TransportFlowControlBase, + grpc_core::chttp2::TransportFlowControl, + grpc_core::chttp2::TransportFlowControlDisabled> + flow_control; /** initial window change. This is tracked as we parse settings frames from * the remote peer. If there is a positive delta, then we will make all * streams readable since they may have become unstalled */ @@ -490,16 +566,19 @@ struct grpc_chttp2_stream { bool* sent_trailing_metadata_op = nullptr; grpc_closure* send_trailing_metadata_finished = nullptr; + grpc_core::OrphanablePtr fetching_send_message; + uint32_t fetched_send_message_length = 0; + grpc_slice fetching_slice = grpc_empty_slice(); int64_t next_message_end_offset; int64_t flow_controlled_bytes_written = 0; int64_t flow_controlled_bytes_flowed = 0; - grpc_closure* send_message_finished = nullptr; + grpc_closure complete_fetch_locked; + grpc_closure* fetching_send_message_finished = nullptr; grpc_metadata_batch* recv_initial_metadata; grpc_closure* recv_initial_metadata_ready = nullptr; bool* trailing_metadata_available = nullptr; - absl::optional* recv_message = nullptr; - uint32_t* recv_message_flags = nullptr; + grpc_core::OrphanablePtr* recv_message = nullptr; bool* call_failed_before_recv_message = nullptr; grpc_closure* recv_message_ready = nullptr; grpc_metadata_batch* recv_trailing_metadata; @@ -536,7 +615,22 @@ struct grpc_chttp2_stream { grpc_metadata_batch initial_metadata_buffer; grpc_metadata_batch trailing_metadata_buffer; - grpc_slice_buffer frame_storage; /* protected by t combiner */ + grpc_slice_buffer frame_storage; /* protected by t combiner */ + + grpc_closure* on_next = nullptr; /* protected by t combiner */ + bool pending_byte_stream = false; /* protected by t combiner */ + // cached length of buffer to be used by the transport thread in cases where + // stream->pending_byte_stream == true. The value is saved before + // application threads are allowed to modify + // unprocessed_incoming_frames_buffer + size_t unprocessed_incoming_frames_buffer_cached_length = 0; + /* Accessed only by transport thread when stream->pending_byte_stream == false + * Accessed only by application thread when stream->pending_byte_stream == + * true */ + grpc_slice_buffer unprocessed_incoming_frames_buffer; + grpc_closure reset_byte_stream; + grpc_error_handle byte_stream_error = + GRPC_ERROR_NONE; /* protected by t combiner */ bool received_last_frame = false; /* protected by t combiner */ grpc_core::Timestamp deadline = grpc_core::Timestamp::InfFuture(); @@ -545,13 +639,22 @@ struct grpc_chttp2_stream { grpc_error_handle forced_close_error = GRPC_ERROR_NONE; /** how many header frames have we received? */ uint8_t header_frames_received = 0; + /** parsing state for data frames */ + /* Accessed only by transport thread when stream->pending_byte_stream == false + * Accessed only by application thread when stream->pending_byte_stream == + * true */ + grpc_chttp2_data_parser data_parser; /** number of bytes received - reset at end of parse thread execution */ int64_t received_bytes = 0; bool sent_initial_metadata = false; bool sent_trailing_metadata = false; - grpc_core::chttp2::StreamFlowControl flow_control; + grpc_core::PolymorphicManualConstructor< + grpc_core::chttp2::StreamFlowControlBase, + grpc_core::chttp2::StreamFlowControl, + grpc_core::chttp2::StreamFlowControlDisabled> + flow_control; grpc_slice_buffer flow_controlled_buffer; diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc index 3ffeee8128c..2c6f06f3197 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.cc +++ b/src/core/ext/transport/chttp2/transport/parsing.cc @@ -24,7 +24,6 @@ #include #include "absl/base/attributes.h" -#include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" @@ -47,11 +46,11 @@ #include "src/core/ext/transport/chttp2/transport/stream_map.h" #include "src/core/lib/channel/channelz.h" #include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/time.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/transport/bdp_estimator.h" -#include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/http2_errors.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -218,9 +217,10 @@ grpc_error_handle grpc_chttp2_perform_read(grpc_chttp2_transport* t, return GRPC_ERROR_NONE; } goto dts_fh_0; /* loop */ - } else if (t->incoming_frame_size > - t->settings[GRPC_ACKED_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) { + } else if (t->flow_control->flow_control_enabled() && + t->incoming_frame_size > + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) { return GRPC_ERROR_CREATE_FROM_CPP_STRING( absl::StrFormat("Frame size %d is larger than max frame size %d", t->incoming_frame_size, @@ -384,7 +384,7 @@ void grpc_chttp2_parsing_become_skip_parser(grpc_chttp2_transport* t) { static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) { // Update BDP accounting since we have received a data frame. - grpc_core::BdpEstimator* bdp_est = t->flow_control.bdp_estimator(); + grpc_core::BdpEstimator* bdp_est = t->flow_control->bdp_estimator(); if (bdp_est) { if (t->bdp_ping_blocked) { t->bdp_ping_blocked = false; @@ -395,17 +395,17 @@ static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) { } grpc_chttp2_stream* s = grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); - absl::Status status; + grpc_error_handle err = GRPC_ERROR_NONE; grpc_core::chttp2::FlowControlAction action; if (s == nullptr) { - status = t->flow_control.RecvData(t->incoming_frame_size); - action = t->flow_control.MakeAction(); + err = t->flow_control->RecvData(t->incoming_frame_size); + action = t->flow_control->MakeAction(); } else { - status = s->flow_control.RecvData(t->incoming_frame_size); - action = s->flow_control.MakeAction(); + err = s->flow_control->RecvData(t->incoming_frame_size); + action = s->flow_control->MakeAction(); } grpc_chttp2_act_on_flowctl_action(action, t, s); - if (!status.ok()) { + if (err != GRPC_ERROR_NONE) { goto error_handler; } if (s == nullptr) { @@ -413,29 +413,33 @@ static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) { } s->received_bytes += t->incoming_frame_size; s->stats.incoming.framing_bytes += 9; - if (s->read_closed) { + if (err == GRPC_ERROR_NONE && s->read_closed) { return init_non_header_skip_frame_parser(t); } - status = - grpc_chttp2_data_parser_begin_frame(t->incoming_frame_flags, s->id, s); + if (err == GRPC_ERROR_NONE) { + err = grpc_chttp2_data_parser_begin_frame( + &s->data_parser, t->incoming_frame_flags, s->id, s); + } error_handler: - if (status.ok()) { + intptr_t unused; + if (err == GRPC_ERROR_NONE) { t->incoming_stream = s; /* t->parser = grpc_chttp2_data_parser_parse;*/ t->parser = grpc_chttp2_data_parser_parse; - t->parser_data = nullptr; + t->parser_data = &s->data_parser; t->ping_state.last_ping_sent_time = grpc_core::Timestamp::InfPast(); return GRPC_ERROR_NONE; - } else if (s != nullptr) { + } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, &unused)) { /* handle stream errors by closing the stream */ - grpc_chttp2_mark_stream_closed(t, s, true, false, - absl_status_to_grpc_error(status)); + if (s != nullptr) { + grpc_chttp2_mark_stream_closed(t, s, true, false, err); + } grpc_chttp2_add_rst_stream_to_next_write(t, t->incoming_stream_id, GRPC_HTTP2_PROTOCOL_ERROR, &s->stats.outgoing); return init_non_header_skip_frame_parser(t); } else { - return absl_status_to_grpc_error(status); + return err; } } @@ -564,10 +568,6 @@ static grpc_error_handle init_header_frame_parser(grpc_chttp2_transport* t, gpr_log(GPR_ERROR, "too many header frames received"); return init_header_skip_frame_parser(t, priority_type); } - if (frame_type == HPackParser::LogInfo::kTrailers && !t->header_eof) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Trailing metadata frame received without an end-o-stream"); - } t->hpack_parser.BeginFrame( incoming_metadata_buffer, t->settings[GRPC_ACKED_SETTINGS] @@ -647,10 +647,6 @@ static grpc_error_handle init_settings_frame_parser(grpc_chttp2_transport* t) { t->hpack_parser.hpack_table()->SetMaxBytes( t->settings[GRPC_ACKED_SETTINGS] [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]); - t->flow_control.SetAckedInitialWindow( - t->settings[GRPC_ACKED_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); - grpc_chttp2_act_on_flowctl_action(t->flow_control.MakeAction(), t, nullptr); t->sent_local_settings = false; } t->parser = grpc_chttp2_settings_parser_parse; diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.cc b/src/core/ext/transport/chttp2/transport/stream_lists.cc index 92c36a60934..aaa73975b5c 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.cc +++ b/src/core/ext/transport/chttp2/transport/stream_lists.cc @@ -20,10 +20,12 @@ #include +#include "src/core/ext/transport/chttp2/transport/flow_control.h" #include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/ext/transport/chttp2/transport/internal.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/bitset.h" +#include "src/core/lib/gprpp/manual_constructor.h" static const char* stream_list_id_string(grpc_chttp2_stream_list_id id) { switch (id) { @@ -187,6 +189,7 @@ void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport* t, void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { + GPR_ASSERT(t->flow_control->flow_control_enabled()); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } @@ -202,6 +205,7 @@ void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport* t, void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { + GPR_ASSERT(t->flow_control->flow_control_enabled()); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc index 82b6c101a88..0c0f59460fb 100644 --- a/src/core/ext/transport/chttp2/transport/writing.cc +++ b/src/core/ext/transport/chttp2/transport/writing.cc @@ -22,6 +22,7 @@ #include #include +#include #include #include "absl/types/optional.h" @@ -49,6 +50,7 @@ #include "src/core/lib/debug/stats.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/time.h" @@ -218,14 +220,14 @@ static void report_stall(grpc_chttp2_transport* t, grpc_chttp2_stream* s, s->flow_controlled_buffer.length, s->flow_controlled_bytes_flowed, t->settings[GRPC_ACKED_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], - t->flow_control.remote_window(), + t->flow_control->remote_window(), static_cast(std::max( int64_t(0), - s->flow_control.remote_window_delta() + + s->flow_control->remote_window_delta() + static_cast( t->settings[GRPC_PEER_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]))), - s->flow_control.remote_window_delta()); + s->flow_control->remote_window_delta()); } } @@ -281,11 +283,6 @@ class WriteContext { void FlushSettings() { if (t_->dirtied_local_settings && !t_->sent_local_settings) { - t_->flow_control.SetSentInitialWindow( - t_->settings[GRPC_LOCAL_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); - grpc_chttp2_act_on_flowctl_action(t_->flow_control.MakeAction(), t_, - nullptr); grpc_slice_buffer_add( &t_->outbuf, grpc_chttp2_settings_create( t_->settings[GRPC_SENT_SETTINGS], @@ -307,7 +304,7 @@ class WriteContext { void FlushWindowUpdates() { uint32_t transport_announce = - t_->flow_control.MaybeSendUpdate(t_->outbuf.count > 0); + t_->flow_control->MaybeSendUpdate(t_->outbuf.count > 0); if (transport_announce) { grpc_transport_one_way_stats throwaway_stats; grpc_slice_buffer_add( @@ -395,7 +392,7 @@ class DataSendContext { uint32_t stream_remote_window() const { return static_cast(std::max( int64_t(0), - s_->flow_control.remote_window_delta() + + s_->flow_control->remote_window_delta() + static_cast( t_->settings[GRPC_PEER_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]))); @@ -405,7 +402,7 @@ class DataSendContext { return static_cast(std::min( t_->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], static_cast(std::min(int64_t(stream_remote_window()), - t_->flow_control.remote_window())))); + t_->flow_control->remote_window())))); } bool AnyOutgoing() const { return max_outgoing() > 0; } @@ -414,11 +411,12 @@ class DataSendContext { uint32_t send_bytes = static_cast( std::min(size_t(max_outgoing()), s_->flow_controlled_buffer.length)); is_last_frame_ = send_bytes == s_->flow_controlled_buffer.length && + s_->fetching_send_message == nullptr && s_->send_trailing_metadata != nullptr && s_->send_trailing_metadata->empty(); grpc_chttp2_encode_data(s_->id, &s_->flow_controlled_buffer, send_bytes, is_last_frame_, &s_->stats.outgoing, &t_->outbuf); - s_->flow_control.SentData(send_bytes); + s_->flow_control->SentData(send_bytes); s_->sending_bytes += send_bytes; } @@ -450,8 +448,8 @@ class StreamWriteContext { gpr_log(GPR_INFO, "W:%p %s[%d] im-(sent,send)=(%d,%d) announce=%d", t_, t_->is_client ? "CLIENT" : "SERVER", s->id, s->sent_initial_metadata, s->send_initial_metadata != nullptr, - (int)(s->flow_control.local_window_delta() - - s->flow_control.announced_window_delta()))); + (int)(s->flow_control->local_window_delta() - + s->flow_control->announced_window_delta()))); } void FlushInitialMetadata() { @@ -464,7 +462,8 @@ class StreamWriteContext { // trailing metadata. This results in a Trailers-Only response, // which is required for retries, as per: // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid - if (!t_->is_client && s_->flow_controlled_buffer.length == 0 && + if (!t_->is_client && s_->fetching_send_message == nullptr && + s_->flow_controlled_buffer.length == 0 && s_->send_trailing_metadata != nullptr && is_default_initial_metadata(s_->send_initial_metadata)) { ConvertInitialMetadataToTrailingMetadata(); @@ -497,7 +496,7 @@ class StreamWriteContext { void FlushWindowUpdates() { /* send any window updates */ - const uint32_t stream_announce = s_->flow_control.MaybeSendUpdate(); + const uint32_t stream_announce = s_->flow_control->MaybeSendUpdate(); if (stream_announce == 0) return; grpc_slice_buffer_add( @@ -517,7 +516,7 @@ class StreamWriteContext { DataSendContext data_send_context(write_context_, t_, s_); if (!data_send_context.AnyOutgoing()) { - if (t_->flow_control.remote_window() <= 0) { + if (t_->flow_control->remote_window() <= 0) { report_stall(t_, s_, "transport"); grpc_chttp2_list_add_stalled_by_transport(t_, s_); } else if (data_send_context.stream_remote_window() <= 0) { @@ -548,6 +547,7 @@ class StreamWriteContext { if (!s_->sent_initial_metadata) return; if (s_->send_trailing_metadata == nullptr) return; + if (s_->fetching_send_message != nullptr) return; if (s_->flow_controlled_buffer.length != 0) return; GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "sending trailing_metadata")); @@ -635,7 +635,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( ctx.FlushQueuedBuffers(); ctx.EnactHpackSettings(); - if (t->flow_control.remote_window() > 0) { + if (t->flow_control->remote_window() > 0) { ctx.UpdateStreamsNoLongerStalled(); } diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.cc b/src/core/ext/transport/cronet/transport/cronet_transport.cc index ccdf9b6b701..c9b7e127232 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.cc +++ b/src/core/ext/transport/cronet/transport/cronet_transport.cc @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -32,10 +33,10 @@ #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" #include "third_party/objective_c/Cronet/bidirectional_stream_c.h" #include +#include #include #include #include @@ -46,6 +47,8 @@ #include "src/core/ext/transport/cronet/transport/cronet_status.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/manual_constructor.h" +#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/error.h" @@ -54,15 +57,14 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/surface/validate_metadata.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport_impl.h" -// IWYU pragma: no_include - #define GRPC_HEADER_SIZE_IN_BYTES 5 #define GRPC_FLUSH_READ_SIZE 4096 @@ -131,7 +133,9 @@ typedef struct grpc_cronet_transport grpc_cronet_transport; http://www.catb.org/esr/structure-packing/#_structure_reordering: */ struct read_state { explicit read_state(grpc_core::Arena* arena) - : trailing_metadata(arena), initial_metadata(arena) {} + : trailing_metadata(arena), initial_metadata(arena) { + grpc_slice_buffer_init(&read_slice_buffer); + } /* vars to store data coming from server */ char* read_buffer = nullptr; @@ -145,7 +149,8 @@ struct read_state { bool read_stream_closed = false; /* vars for holding data destined for the application */ - grpc_core::SliceBuffer read_slice_buffer; + grpc_core::ManualConstructor sbs; + grpc_slice_buffer read_slice_buffer; /* vars for trailing metadata */ grpc_metadata_batch trailing_metadata; @@ -1083,17 +1088,38 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { result = NO_ACTION_POSSIBLE; CRONET_LOG(GPR_DEBUG, "Stream is either cancelled, failed or finished"); } else { + grpc_slice_buffer write_slice_buffer; + grpc_slice slice; + grpc_slice_buffer_init(&write_slice_buffer); + while (write_slice_buffer.length < + stream_op->payload->send_message.send_message->length()) { + /* TODO(roth): When we add support for incremental sending,this code + * will need to be changed to support asynchronous delivery of the + * send_message payload. */ + if (!stream_op->payload->send_message.send_message->Next( + stream_op->payload->send_message.send_message->length(), + nullptr)) { + /* Should never reach here */ + GPR_ASSERT(false); + } + if (GRPC_ERROR_NONE != + stream_op->payload->send_message.send_message->Pull(&slice)) { + /* Should never reach here */ + GPR_ASSERT(false); + } + grpc_slice_buffer_add(&write_slice_buffer, slice); + } size_t write_buffer_size; - create_grpc_frame( - stream_op->payload->send_message.send_message->c_slice_buffer(), - &stream_state->ws.write_buffer, &write_buffer_size, - stream_op->payload->send_message.flags); + create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, + &write_buffer_size, + stream_op->payload->send_message.send_message->flags()); if (write_buffer_size > 0) { CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, %p)", s->cbs, stream_state->ws.write_buffer); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, static_cast(write_buffer_size), false); + grpc_slice_buffer_destroy_internal(&write_slice_buffer); if (t->use_packet_coalescing) { if (!stream_op->send_trailing_metadata) { CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); @@ -1113,6 +1139,7 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { } stream_state->state_op_done[OP_SEND_MESSAGE] = true; oas->state.state_op_done[OP_SEND_MESSAGE] = true; + stream_op->payload->send_message.send_message.reset(); } else if (stream_op->send_trailing_metadata && op_can_be_run(stream_op, s, &oas->state, OP_SEND_TRAILING_METADATA)) { @@ -1224,14 +1251,16 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { stream_state->rs.remaining_bytes = 0; CRONET_LOG(GPR_DEBUG, "read operation complete. Empty response."); /* Clean up read_slice_buffer in case there is unread data. */ - stream_state->rs.read_slice_buffer.Clear(); + grpc_slice_buffer_destroy_internal( + &stream_state->rs.read_slice_buffer); + grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer); uint32_t flags = 0; if (stream_state->rs.compressed) { flags |= GRPC_WRITE_INTERNAL_COMPRESS; } - *stream_op->payload->recv_message.flags = flags; - *stream_op->payload->recv_message.recv_message = - std::move(stream_state->rs.read_slice_buffer); + stream_state->rs.sbs.Init(&stream_state->rs.read_slice_buffer, flags); + stream_op->payload->recv_message.recv_message->reset( + stream_state->rs.sbs.get()); grpc_core::ExecCtx::Run( DEBUG_LOCATION, stream_op->payload->recv_message.recv_message_ready, @@ -1270,16 +1299,17 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { static_cast(stream_state->rs.length_field)); null_and_maybe_free_read_buffer(s); /* Clean up read_slice_buffer in case there is unread data. */ - stream_state->rs.read_slice_buffer.Clear(); - stream_state->rs.read_slice_buffer.Append( - grpc_core::Slice(read_data_slice)); + grpc_slice_buffer_destroy_internal(&stream_state->rs.read_slice_buffer); + grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer); + grpc_slice_buffer_add(&stream_state->rs.read_slice_buffer, + read_data_slice); uint32_t flags = 0; if (stream_state->rs.compressed) { flags = GRPC_WRITE_INTERNAL_COMPRESS; } - *stream_op->payload->recv_message.flags = flags; - *stream_op->payload->recv_message.recv_message = - std::move(stream_state->rs.read_slice_buffer); + stream_state->rs.sbs.Init(&stream_state->rs.read_slice_buffer, flags); + stream_op->payload->recv_message.recv_message->reset( + stream_state->rs.sbs.get()); grpc_core::ExecCtx::Run( DEBUG_LOCATION, stream_op->payload->recv_message.recv_message_ready, GRPC_ERROR_NONE); @@ -1395,6 +1425,8 @@ inline stream_obj::stream_obj(grpc_transport* gt, grpc_stream* gs, inline stream_obj::~stream_obj() { null_and_maybe_free_read_buffer(this); + /* Clean up read_slice_buffer in case there is unread data. */ + grpc_slice_buffer_destroy_internal(&state.rs.read_slice_buffer); GRPC_ERROR_UNREF(state.cancel_error); } diff --git a/src/core/ext/transport/inproc/inproc_transport.cc b/src/core/ext/transport/inproc/inproc_transport.cc index e472b98c2fe..7b1146bab88 100644 --- a/src/core/ext/transport/inproc/inproc_transport.cc +++ b/src/core/ext/transport/inproc/inproc_transport.cc @@ -21,6 +21,7 @@ #include "src/core/ext/transport/inproc/inproc_transport.h" #include +#include #include #include @@ -33,10 +34,11 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "absl/utility/utility.h" #include #include +#include +#include #include #include #include @@ -48,6 +50,8 @@ #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/manual_constructor.h" +#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/time.h" #include "src/core/lib/iomgr/closure.h" @@ -58,11 +62,12 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/surface/server.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -87,10 +92,6 @@ void fill_in_metadata(inproc_stream* s, const grpc_metadata_batch* metadata, uint32_t flags, grpc_metadata_batch* out_md, uint32_t* outflags, bool* markfilled); -void ResetSendMessage(grpc_transport_stream_op_batch* batch) { - absl::exchange(batch->payload->send_message.send_message, nullptr)->Clear(); -} - struct shared_mu { shared_mu() { // Share one lock between both sides since both sides get affected @@ -225,6 +226,10 @@ struct inproc_stream { GRPC_ERROR_UNREF(cancel_self_error); GRPC_ERROR_UNREF(cancel_other_error); + if (recv_inited) { + grpc_slice_buffer_destroy_internal(&recv_message); + } + t->unref(); } @@ -278,6 +283,10 @@ struct inproc_stream { grpc_transport_stream_op_batch* recv_message_op = nullptr; grpc_transport_stream_op_batch* recv_trailing_md_op = nullptr; + grpc_slice_buffer recv_message; + grpc_core::ManualConstructor recv_stream; + bool recv_inited = false; + bool initial_md_sent = false; bool trailing_md_sent = false; bool initial_md_recvd = false; @@ -351,8 +360,8 @@ void fill_in_metadata(inproc_stream* s, const grpc_metadata_batch* metadata, } // TODO(ctiller): copy the metadata batch, don't rely on a bespoke copy - // function. Can only do this once mdelems are out of the way though, too - // many edge cases otherwise. + // function. Can only do this once mdelems are out of the way though, too many + // edge cases otherwise. out_md->Clear(); CopySink sink(out_md); metadata->Encode(&sink); @@ -468,8 +477,8 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) { if (s->recv_initial_md_op) { grpc_error_handle err; if (!s->t->is_client) { - // If this is a server, provide initial metadata with a path and - // authority since it expects that as well as no error yet + // If this is a server, provide initial metadata with a path and authority + // since it expects that as well as no error yet grpc_metadata_batch fake_md(s->arena); fake_md.Set(grpc_core::HttpPathMetadata(), grpc_core::Slice::FromStaticString("/")); @@ -528,7 +537,7 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) { s->recv_message_op = nullptr; } if (s->send_message_op) { - ResetSendMessage(s->send_message_op); + s->send_message_op->payload->send_message.send_message.reset(); complete_if_batch_end_locked( s, error, s->send_message_op, "fail_helper scheduling send-message-on-complete"); @@ -570,11 +579,35 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) { // synchronously. That assumption is true today but may not always be // true in the future. void message_transfer_locked(inproc_stream* sender, inproc_stream* receiver) { - *receiver->recv_message_op->payload->recv_message.recv_message = - std::move(*sender->send_message_op->payload->send_message.send_message); - *receiver->recv_message_op->payload->recv_message.flags = - sender->send_message_op->payload->send_message.flags; - + size_t remaining = + sender->send_message_op->payload->send_message.send_message->length(); + if (receiver->recv_inited) { + grpc_slice_buffer_destroy_internal(&receiver->recv_message); + } + grpc_slice_buffer_init(&receiver->recv_message); + receiver->recv_inited = true; + do { + grpc_slice message_slice; + grpc_closure unused; + GPR_ASSERT( + sender->send_message_op->payload->send_message.send_message->Next( + SIZE_MAX, &unused)); + grpc_error_handle error = + sender->send_message_op->payload->send_message.send_message->Pull( + &message_slice); + if (error != GRPC_ERROR_NONE) { + cancel_stream_locked(sender, GRPC_ERROR_REF(error)); + break; + } + GPR_ASSERT(error == GRPC_ERROR_NONE); + remaining -= GRPC_SLICE_LENGTH(message_slice); + grpc_slice_buffer_add(&receiver->recv_message, message_slice); + } while (remaining > 0); + sender->send_message_op->payload->send_message.send_message.reset(); + + receiver->recv_stream.Init(&receiver->recv_message, 0); + receiver->recv_message_op->payload->recv_message.recv_message->reset( + receiver->recv_stream.get()); INPROC_LOG(GPR_INFO, "message_transfer_locked %p scheduling message-ready", receiver); grpc_core::ExecCtx::Run( @@ -623,7 +656,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { maybe_process_ops_locked(other, GRPC_ERROR_NONE); } else if (!s->t->is_client && s->trailing_md_sent) { // A server send will never be matched if the server already sent status - ResetSendMessage(s->send_message_op); + s->send_message_op->payload->send_message.send_message.reset(); complete_if_batch_end_locked( s, GRPC_ERROR_NONE, s->send_message_op, "op_state_machine scheduling send-message-on-complete case 1"); @@ -764,7 +797,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { if (s->recv_message_op != nullptr) { // This message needs to be wrapped up because it will never be // satisfied - s->recv_message_op->payload->recv_message.recv_message->reset(); + *s->recv_message_op->payload->recv_message.recv_message = nullptr; INPROC_LOG(GPR_INFO, "op_state_machine %p scheduling message-ready", s); grpc_core::ExecCtx::Run( DEBUG_LOCATION, @@ -778,7 +811,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { if ((s->trailing_md_sent || s->t->is_client) && s->send_message_op) { // Nothing further will try to receive from this stream, so finish off // any outstanding send_message op - ResetSendMessage(s->send_message_op); + s->send_message_op->payload->send_message.send_message.reset(); s->send_message_op->payload->send_message.stream_write_closed = true; complete_if_batch_end_locked( s, new_err, s->send_message_op, @@ -798,8 +831,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { // We should schedule the recv_trailing_md_op completion if // 1. this stream is the client-side // 2. this stream is the server-side AND has already sent its trailing md - // (If the server hasn't already sent its trailing md, it doesn't - // have + // (If the server hasn't already sent its trailing md, it doesn't have // a final status, so don't mark this op complete) if (s->t->is_client || s->trailing_md_sent) { grpc_core::ExecCtx::Run( @@ -844,7 +876,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { // No further message will come on this stream, so finish off the // recv_message_op INPROC_LOG(GPR_INFO, "op_state_machine %p scheduling message-ready", s); - s->recv_message_op->payload->recv_message.recv_message->reset(); + *s->recv_message_op->payload->recv_message.recv_message = nullptr; grpc_core::ExecCtx::Run( DEBUG_LOCATION, s->recv_message_op->payload->recv_message.recv_message_ready, @@ -857,7 +889,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { if (s->trailing_md_recvd && s->send_message_op && s->t->is_client) { // Nothing further will try to receive from this stream, so finish off // any outstanding send_message op - ResetSendMessage(s->send_message_op); + s->send_message_op->payload->send_message.send_message.reset(); complete_if_batch_end_locked( s, new_err, s->send_message_op, "op_state_machine scheduling send-message-on-complete case 3"); @@ -1067,10 +1099,10 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs, } } else { if (error != GRPC_ERROR_NONE) { - // Consume any send message that was sent here but that we are not - // pushing to the other side + // Consume any send message that was sent here but that we are not pushing + // to the other side if (op->send_message) { - ResetSendMessage(op); + op->payload->send_message.send_message.reset(); } // Schedule op's closures that we didn't push to op state machine if (op->recv_initial_metadata) { @@ -1106,10 +1138,10 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs, GRPC_ERROR_REF(error)); } if (op->recv_trailing_metadata) { - INPROC_LOG(GPR_INFO, - "perform_stream_op error %p scheduling " - "trailing-metadata-ready %s", - s, grpc_error_std_string(error).c_str()); + INPROC_LOG( + GPR_INFO, + "perform_stream_op error %p scheduling trailing-metadata-ready %s", + s, grpc_error_std_string(error).c_str()); grpc_core::ExecCtx::Run( DEBUG_LOCATION, op->payload->recv_trailing_metadata.recv_trailing_metadata_ready, diff --git a/src/core/lib/channel/call_tracer.h b/src/core/lib/channel/call_tracer.h index c473c354ba1..58b4bd1d386 100644 --- a/src/core/lib/channel/call_tracer.h +++ b/src/core/lib/channel/call_tracer.h @@ -29,7 +29,7 @@ #include #include "src/core/lib/iomgr/error.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -58,13 +58,13 @@ class CallTracer { virtual void RecordOnDoneSendInitialMetadata(gpr_atm* peer_string) = 0; virtual void RecordSendTrailingMetadata( grpc_metadata_batch* send_trailing_metadata) = 0; - virtual void RecordSendMessage(const SliceBuffer& send_message) = 0; + virtual void RecordSendMessage(const ByteStream& send_message) = 0; // The `RecordReceivedInitialMetadata()` and `RecordReceivedMessage()` // methods should only be invoked when the metadata/message was // successfully received, i.e., without any error. virtual void RecordReceivedInitialMetadata( grpc_metadata_batch* recv_initial_metadata, uint32_t flags) = 0; - virtual void RecordReceivedMessage(const SliceBuffer& recv_message) = 0; + virtual void RecordReceivedMessage(const ByteStream& recv_message) = 0; // If the call was cancelled before the recv_trailing_metadata op // was started, recv_trailing_metadata and transport_stream_stats // will be null. diff --git a/src/core/lib/gprpp/manual_constructor.h b/src/core/lib/gprpp/manual_constructor.h index 95e57cd0e4c..81c3031f02f 100644 --- a/src/core/lib/gprpp/manual_constructor.h +++ b/src/core/lib/gprpp/manual_constructor.h @@ -25,9 +25,12 @@ #include +#include #include #include +#include + #include "src/core/lib/gprpp/construct_destruct.h" namespace grpc_core { @@ -99,6 +102,70 @@ class max_align_of { } // namespace manual_ctor_impl +template +class PolymorphicManualConstructor { + public: + // No constructor or destructor because one of the most useful uses of + // this class is as part of a union, and members of a union could not have + // constructors or destructors till C++11. And, anyway, the whole point of + // this class is to bypass constructor and destructor. + + BaseType* get() { return reinterpret_cast(&space_); } + const BaseType* get() const { + return reinterpret_cast(&space_); + } + + BaseType* operator->() { return get(); } + const BaseType* operator->() const { return get(); } + + BaseType& operator*() { return *get(); } + const BaseType& operator*() const { return *get(); } + + template + void Init() { + FinishInit(new (&space_) DerivedType); + } + + // Init() constructs the Type instance using the given arguments + // (which are forwarded to Type's constructor). + // + // Note that Init() with no arguments performs default-initialization, + // not zero-initialization (i.e it behaves the same as "new Type;", not + // "new Type();"), so it will leave non-class types uninitialized. + template + void Init(Ts&&... args) { + FinishInit(new (&space_) DerivedType(std::forward(args)...)); + } + + // Init() that is equivalent to copy and move construction. + // Enables usage like this: + // ManualConstructor> v; + // v.Init({1, 2, 3}); + template + void Init(const DerivedType& x) { + FinishInit(new (&space_) DerivedType(x)); + } + template + void Init(DerivedType&& x) { + FinishInit(new (&space_) DerivedType(std::forward(x))); + } + + void Destroy() { get()->~BaseType(); } + + private: + template + void FinishInit(DerivedType* p) { + static_assert( + manual_ctor_impl::is_one_of::value, + "DerivedType must be one of the predeclared DerivedTypes"); + GPR_ASSERT(static_cast(p) == p); + } + + typename std::aligned_storage< + manual_ctor_impl::max_size_of::value, + manual_ctor_impl::max_align_of::value>::type space_; +}; + template class ManualConstructor { public: diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index a4b8a72e42f..0526fc5370f 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -950,7 +950,6 @@ ssize_t tcp_send(int fd, const struct msghdr* msg, int additional_flags = 0) { ssize_t sent_length; do { /* TODO(klempner): Cork if this is a partial write */ - errno = 0; GRPC_STATS_INC_SYSCALL_WRITE(); sent_length = sendmsg(fd, msg, SENDMSG_FLAGS | additional_flags); } while (sent_length < 0 && errno == EINTR); diff --git a/src/core/lib/slice/slice_buffer.cc b/src/core/lib/slice/slice_buffer.cc index 2d2ee9fc995..b6f33237a8f 100644 --- a/src/core/lib/slice/slice_buffer.cc +++ b/src/core/lib/slice/slice_buffer.cc @@ -51,21 +51,10 @@ void SliceBuffer::Prepend(Slice slice) { grpc_slice_buffer_undo_take_first(&slice_buffer_, slice.TakeCSlice()); } -Slice SliceBuffer::RefSlice(size_t index) const { +Slice SliceBuffer::RefSlice(size_t index) { return Slice(grpc_slice_ref_internal(slice_buffer_.slices[index])); } -std::string SliceBuffer::JoinIntoString() const { - std::string result; - result.reserve(slice_buffer_.length); - for (size_t i = 0; i < slice_buffer_.count; i++) { - result.append(reinterpret_cast( - GRPC_SLICE_START_PTR(slice_buffer_.slices[i])), - GRPC_SLICE_LENGTH(slice_buffer_.slices[i])); - } - return result; -} - } // namespace grpc_core /* grow a buffer; requires GRPC_SLICE_BUFFER_INLINE_ELEMENTS > 1 */ @@ -381,24 +370,6 @@ void grpc_slice_buffer_move_first_into_buffer(grpc_slice_buffer* src, size_t n, } } -void grpc_slice_buffer_copy_first_into_buffer(grpc_slice_buffer* src, size_t n, - void* dst) { - uint8_t* dstp = static_cast(dst); - GPR_ASSERT(src->length >= n); - - for (size_t i = 0; i < src->count; i++) { - grpc_slice slice = src->slices[i]; - size_t slice_len = GRPC_SLICE_LENGTH(slice); - if (slice_len >= n) { - memcpy(dstp, GRPC_SLICE_START_PTR(slice), n); - return; - } - memcpy(dstp, GRPC_SLICE_START_PTR(slice), slice_len); - dstp += slice_len; - n -= slice_len; - } -} - void grpc_slice_buffer_trim_end(grpc_slice_buffer* sb, size_t n, grpc_slice_buffer* garbage) { GPR_ASSERT(n <= sb->length); diff --git a/src/core/lib/slice/slice_buffer.h b/src/core/lib/slice/slice_buffer.h index d08b08cc1f1..9effb828e88 100644 --- a/src/core/lib/slice/slice_buffer.h +++ b/src/core/lib/slice/slice_buffer.h @@ -19,8 +19,6 @@ #include -#include - #include #include @@ -47,15 +45,14 @@ class SliceBuffer { SliceBuffer(const SliceBuffer& other) = delete; SliceBuffer(SliceBuffer&& other) noexcept { grpc_slice_buffer_init(&slice_buffer_); - grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_); + grpc_slice_buffer_move_into(&slice_buffer_, &other.slice_buffer_); } /// Upon destruction, the underlying raw slice buffer is cleaned out and all /// slices are unreffed. ~SliceBuffer() { grpc_slice_buffer_destroy(&slice_buffer_); } - SliceBuffer& operator=(const SliceBuffer&) = delete; SliceBuffer& operator=(SliceBuffer&& other) noexcept { - grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_); + grpc_slice_buffer_move_into(&slice_buffer_, &other.slice_buffer_); return *this; } @@ -68,7 +65,7 @@ class SliceBuffer { size_t AppendIndexed(Slice slice); /// Returns the number of slices held by the SliceBuffer. - size_t Count() const { return slice_buffer_.count; } + size_t Count() { return slice_buffer_.count; } /// Removes/deletes the last n bytes in the SliceBuffer. void RemoveLastNBytes(size_t n) { @@ -91,37 +88,13 @@ class SliceBuffer { /// Increased the ref-count of slice at the specified index and returns the /// associated slice. - Slice RefSlice(size_t index) const; + Slice RefSlice(size_t index); /// The total number of bytes held by the SliceBuffer - size_t Length() const { return slice_buffer_.length; } - - /// Swap with another slice buffer - void Swap(SliceBuffer* other) { - grpc_slice_buffer_swap(c_slice_buffer(), other->c_slice_buffer()); - } - - /// Concatenate all slices and return the resulting string. - std::string JoinIntoString() const; - - // Return a copy of the slice buffer - SliceBuffer Copy() const { - SliceBuffer copy; - for (size_t i = 0; i < Count(); i++) { - copy.Append(RefSlice(i)); - } - return copy; - } + size_t Length() { return slice_buffer_.length; } /// Return a pointer to the back raw grpc_slice_buffer - grpc_slice_buffer* c_slice_buffer() { return &slice_buffer_; } - - /// Return a pointer to the back raw grpc_slice_buffer - const grpc_slice_buffer* c_slice_buffer() const { return &slice_buffer_; } - - const grpc_slice& c_slice_at(size_t index) { - return slice_buffer_.slices[index]; - } + grpc_slice_buffer* RawSliceBuffer() { return &slice_buffer_; } private: /// The backing raw slice buffer. @@ -130,8 +103,4 @@ class SliceBuffer { } // namespace grpc_core -// Copy the first n bytes of src into memory pointed to by dst. -void grpc_slice_buffer_copy_first_into_buffer(grpc_slice_buffer* src, size_t n, - void* dst); - #endif // GRPC_CORE_LIB_SLICE_SLICE_BUFFER_H diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 51085c70da1..56c6c96ed61 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -58,6 +59,8 @@ #include "src/core/lib/gpr/time_precise.h" #include "src/core/lib/gprpp/cpp_impl_of.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/manual_constructor.h" +#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/call_combiner.h" @@ -65,7 +68,6 @@ #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/resource_quota/arena.h" -#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/surface/api_trace.h" @@ -74,6 +76,7 @@ #include "src/core/lib/surface/completion_queue.h" #include "src/core/lib/surface/server.h" #include "src/core/lib/surface/validate_metadata.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -306,6 +309,8 @@ class FilterStackCall final : public Call { void PostCompletion(); void FinishStep(); + void ContinueReceivingSlices(); + void ReceivingSliceReady(grpc_error_handle error); void ProcessDataAfterMetadata(); void ReceivingStreamReady(grpc_error_handle error); void ValidateFilteredMetadata(); @@ -395,13 +400,13 @@ class FilterStackCall final : public Call { /* Contexts for various subsystems (security, tracing, ...). */ grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {}; - SliceBuffer send_slice_buffer_; - absl::optional receiving_slice_buffer_; - uint32_t receiving_stream_flags_; + ManualConstructor sending_stream_; + OrphanablePtr receiving_stream_; bool call_failed_before_recv_message_ = false; grpc_byte_buffer** receiving_buffer_ = nullptr; grpc_slice receiving_slice_ = grpc_empty_slice(); + grpc_closure receiving_slice_ready_; grpc_closure receiving_stream_ready_; grpc_closure receiving_initial_metadata_ready_; grpc_closure receiving_trailing_metadata_ready_; @@ -648,7 +653,7 @@ void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) { auto* c = static_cast(call); c->recv_initial_metadata_.Clear(); c->recv_trailing_metadata_.Clear(); - c->receiving_slice_buffer_.reset(); + c->receiving_stream_.reset(); ParentCall* pc = c->parent_call(); if (pc != nullptr) { pc->~ParentCall(); @@ -1085,7 +1090,6 @@ void FilterStackCall::BatchControl::PostCompletion() { "Attempt to send message after stream was closed.")); } call->sending_message_ = false; - call->send_slice_buffer_.Clear(); } if (op_.send_trailing_metadata) { call->send_trailing_metadata_.Clear(); @@ -1131,27 +1135,95 @@ void FilterStackCall::BatchControl::FinishStep() { } } +void FilterStackCall::BatchControl::ContinueReceivingSlices() { + grpc_error_handle error; + FilterStackCall* call = call_; + for (;;) { + size_t remaining = call->receiving_stream_->length() - + (*call->receiving_buffer_)->data.raw.slice_buffer.length; + if (remaining == 0) { + call->receiving_message_ = false; + call->receiving_stream_.reset(); + FinishStep(); + return; + } + if (call->receiving_stream_->Next(remaining, + &call->receiving_slice_ready_)) { + error = call->receiving_stream_->Pull(&call->receiving_slice_); + if (error == GRPC_ERROR_NONE) { + grpc_slice_buffer_add( + &(*call->receiving_buffer_)->data.raw.slice_buffer, + call->receiving_slice_); + } else { + call->receiving_stream_.reset(); + grpc_byte_buffer_destroy(*call->receiving_buffer_); + *call->receiving_buffer_ = nullptr; + call->receiving_message_ = false; + FinishStep(); + GRPC_ERROR_UNREF(error); + return; + } + } else { + return; + } + } +} + +void FilterStackCall::BatchControl::ReceivingSliceReady( + grpc_error_handle error) { + FilterStackCall* call = call_; + bool release_error = false; + + if (error == GRPC_ERROR_NONE) { + grpc_slice slice; + error = call->receiving_stream_->Pull(&slice); + if (error == GRPC_ERROR_NONE) { + grpc_slice_buffer_add(&(*call->receiving_buffer_)->data.raw.slice_buffer, + slice); + ContinueReceivingSlices(); + } else { + /* Error returned by ByteStream::Pull() needs to be released manually */ + release_error = true; + } + } + + if (error != GRPC_ERROR_NONE) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures)) { + GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error)); + } + call->receiving_stream_.reset(); + grpc_byte_buffer_destroy(*call->receiving_buffer_); + *call->receiving_buffer_ = nullptr; + call->receiving_message_ = false; + FinishStep(); + if (release_error) { + GRPC_ERROR_UNREF(error); + } + } +} + void FilterStackCall::BatchControl::ProcessDataAfterMetadata() { FilterStackCall* call = call_; - if (!call->receiving_slice_buffer_.has_value()) { + if (call->receiving_stream_ == nullptr) { *call->receiving_buffer_ = nullptr; call->receiving_message_ = false; FinishStep(); } else { - call->test_only_last_message_flags_ = call->receiving_stream_flags_; - if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) && + call->test_only_last_message_flags_ = call->receiving_stream_->flags(); + if ((call->receiving_stream_->flags() & GRPC_WRITE_INTERNAL_COMPRESS) && (call->incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) { *call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create( nullptr, 0, call->incoming_compression_algorithm_); } else { *call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0); } - grpc_slice_buffer_move_into( - call->receiving_slice_buffer_->c_slice_buffer(), - &(*call->receiving_buffer_)->data.raw.slice_buffer); - call->receiving_message_ = false; - call->receiving_slice_buffer_.reset(); - FinishStep(); + GRPC_CLOSURE_INIT( + &call->receiving_slice_ready_, + [](void* bctl, grpc_error_handle error) { + static_cast(bctl)->ReceivingSliceReady(error); + }, + this, grpc_schedule_on_exec_ctx); + ContinueReceivingSlices(); } } @@ -1159,7 +1231,7 @@ void FilterStackCall::BatchControl::ReceivingStreamReady( grpc_error_handle error) { FilterStackCall* call = call_; if (error != GRPC_ERROR_NONE) { - call->receiving_slice_buffer_.reset(); + call->receiving_stream_.reset(); if (batch_error_.ok()) { batch_error_.set(error); } @@ -1168,7 +1240,7 @@ void FilterStackCall::BatchControl::ReceivingStreamReady( /* If recv_state is kRecvNone, we will save the batch_control * object with rel_cas, and will not use it after the cas. Its corresponding * acq_load is in receiving_initial_metadata_ready() */ - if (error != GRPC_ERROR_NONE || !call->receiving_slice_buffer_.has_value() || + if (error != GRPC_ERROR_NONE || call->receiving_stream_ == nullptr || !gpr_atm_rel_cas(&call->recv_state_, kRecvNone, reinterpret_cast(this))) { ProcessDataAfterMetadata(); @@ -1449,12 +1521,10 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, } stream_op->send_message = true; sending_message_ = true; - send_slice_buffer_.Clear(); - grpc_slice_buffer_move_into( - &op->data.send_message.send_message->data.raw.slice_buffer, - send_slice_buffer_.c_slice_buffer()); - stream_op_payload->send_message.flags = flags; - stream_op_payload->send_message.send_message = &send_slice_buffer_; + sending_stream_.Init( + &op->data.send_message.send_message->data.raw.slice_buffer, flags); + stream_op_payload->send_message.send_message.reset( + sending_stream_.get()); has_send_ops = true; break; } @@ -1591,11 +1661,8 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, } receiving_message_ = true; stream_op->recv_message = true; - receiving_slice_buffer_.reset(); receiving_buffer_ = op->data.recv_message.recv_message; - stream_op_payload->recv_message.recv_message = &receiving_slice_buffer_; - receiving_stream_flags_ = 0; - stream_op_payload->recv_message.flags = &receiving_stream_flags_; + stream_op_payload->recv_message.recv_message = &receiving_stream_; stream_op_payload->recv_message.call_failed_before_recv_message = &call_failed_before_recv_message_; GRPC_CLOSURE_INIT( @@ -1720,6 +1787,10 @@ done_with_error: } if (stream_op->send_message) { sending_message_ = false; + // No need to invoke call->sending_stream->Orphan() explicitly. + // stream_op_payload->send_message.send_message.reset() calls Deletor + // of call->sending_stream which in-turn invokes the Orphan() method. + stream_op_payload->send_message.send_message.reset(); } if (stream_op->send_trailing_metadata) { sent_final_op_ = false; diff --git a/src/core/lib/transport/byte_stream.cc b/src/core/lib/transport/byte_stream.cc new file mode 100644 index 00000000000..3b909182f69 --- /dev/null +++ b/src/core/lib/transport/byte_stream.cc @@ -0,0 +1,165 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "src/core/lib/transport/byte_stream.h" + +#include +#include + +#include +#include + +#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/slice/slice_refcount.h" + +namespace grpc_core { + +// +// SliceBufferByteStream +// + +SliceBufferByteStream::SliceBufferByteStream(grpc_slice_buffer* slice_buffer, + uint32_t flags) + : ByteStream(static_cast(slice_buffer->length), flags) { + GPR_ASSERT(slice_buffer->length <= UINT32_MAX); + grpc_slice_buffer_init(&backing_buffer_); + grpc_slice_buffer_swap(slice_buffer, &backing_buffer_); + if (backing_buffer_.count == 0) { + grpc_slice_buffer_add_indexed(&backing_buffer_, grpc_empty_slice()); + GPR_ASSERT(backing_buffer_.count > 0); + } +} + +SliceBufferByteStream::~SliceBufferByteStream() {} + +void SliceBufferByteStream::Orphan() { + grpc_slice_buffer_destroy_internal(&backing_buffer_); + GRPC_ERROR_UNREF(shutdown_error_); + shutdown_error_ = GRPC_ERROR_NONE; + // Note: We do not actually delete the object here, since + // SliceBufferByteStream is usually allocated as part of a larger + // object and has an OrphanablePtr of itself passed down through the + // filter stack. +} + +bool SliceBufferByteStream::Next(size_t /*max_size_hint*/, + grpc_closure* /*on_complete*/) { + GPR_DEBUG_ASSERT(backing_buffer_.count > 0); + return true; +} + +grpc_error_handle SliceBufferByteStream::Pull(grpc_slice* slice) { + if (GPR_UNLIKELY(shutdown_error_ != GRPC_ERROR_NONE)) { + return GRPC_ERROR_REF(shutdown_error_); + } + *slice = grpc_slice_buffer_take_first(&backing_buffer_); + return GRPC_ERROR_NONE; +} + +void SliceBufferByteStream::Shutdown(grpc_error_handle error) { + GRPC_ERROR_UNREF(shutdown_error_); + shutdown_error_ = error; +} + +// +// ByteStreamCache +// + +ByteStreamCache::ByteStreamCache(OrphanablePtr underlying_stream) + : underlying_stream_(std::move(underlying_stream)), + length_(underlying_stream_->length()), + flags_(underlying_stream_->flags()) { + grpc_slice_buffer_init(&cache_buffer_); +} + +ByteStreamCache::~ByteStreamCache() { Destroy(); } + +void ByteStreamCache::Destroy() { + underlying_stream_.reset(); + if (cache_buffer_.length > 0) { + grpc_slice_buffer_destroy_internal(&cache_buffer_); + } +} + +// +// ByteStreamCache::CachingByteStream +// + +ByteStreamCache::CachingByteStream::CachingByteStream(ByteStreamCache* cache) + : ByteStream(cache->length_, cache->flags_), cache_(cache) {} + +ByteStreamCache::CachingByteStream::~CachingByteStream() {} + +void ByteStreamCache::CachingByteStream::Orphan() { + GRPC_ERROR_UNREF(shutdown_error_); + shutdown_error_ = GRPC_ERROR_NONE; + // Note: We do not actually delete the object here, since + // CachingByteStream is usually allocated as part of a larger + // object and has an OrphanablePtr of itself passed down through the + // filter stack. +} + +bool ByteStreamCache::CachingByteStream::Next(size_t max_size_hint, + grpc_closure* on_complete) { + if (shutdown_error_ != GRPC_ERROR_NONE) return true; + if (cursor_ < cache_->cache_buffer_.count) return true; + GPR_ASSERT(cache_->underlying_stream_ != nullptr); + return cache_->underlying_stream_->Next(max_size_hint, on_complete); +} + +grpc_error_handle ByteStreamCache::CachingByteStream::Pull(grpc_slice* slice) { + if (shutdown_error_ != GRPC_ERROR_NONE) { + return GRPC_ERROR_REF(shutdown_error_); + } + if (cursor_ < cache_->cache_buffer_.count) { + *slice = grpc_slice_ref_internal(cache_->cache_buffer_.slices[cursor_]); + ++cursor_; + offset_ += GRPC_SLICE_LENGTH(*slice); + return GRPC_ERROR_NONE; + } + GPR_ASSERT(cache_->underlying_stream_ != nullptr); + grpc_error_handle error = cache_->underlying_stream_->Pull(slice); + if (error == GRPC_ERROR_NONE) { + grpc_slice_buffer_add(&cache_->cache_buffer_, + grpc_slice_ref_internal(*slice)); + ++cursor_; + offset_ += GRPC_SLICE_LENGTH(*slice); + // Orphan the underlying stream if it's been drained. + if (offset_ == cache_->underlying_stream_->length()) { + cache_->underlying_stream_.reset(); + } + } + return error; +} + +void ByteStreamCache::CachingByteStream::Shutdown(grpc_error_handle error) { + GRPC_ERROR_UNREF(shutdown_error_); + shutdown_error_ = GRPC_ERROR_REF(error); + if (cache_->underlying_stream_ != nullptr) { + cache_->underlying_stream_->Shutdown(error); + } +} + +void ByteStreamCache::CachingByteStream::Reset() { + cursor_ = 0; + offset_ = 0; +} + +} // namespace grpc_core diff --git a/src/core/lib/transport/byte_stream.h b/src/core/lib/transport/byte_stream.h new file mode 100644 index 00000000000..6c5d6e4d23b --- /dev/null +++ b/src/core/lib/transport/byte_stream.h @@ -0,0 +1,170 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H +#define GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H + +#include + +#include +#include + +#include + +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/iomgr/closure.h" +#include "src/core/lib/iomgr/error.h" + +/** Internal bit flag for grpc_begin_message's \a flags signaling the use of + * compression for the message. (Does not apply for stream compression.) */ +#define GRPC_WRITE_INTERNAL_COMPRESS (0x80000000u) +/** Internal bit flag for determining whether the message was compressed and had + * to be decompressed by the message_decompress filter. (Does not apply for + * stream compression.) */ +#define GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED (0x40000000u) +/** Mask of all valid internal flags. */ +#define GRPC_WRITE_INTERNAL_USED_MASK \ + (GRPC_WRITE_INTERNAL_COMPRESS | GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED) + +namespace grpc_core { + +class ByteStream : public Orphanable { + public: + ~ByteStream() override {} + + // Returns true if the bytes are available immediately (in which case + // on_complete will not be called), or false if the bytes will be available + // asynchronously (in which case on_complete will be called when they + // are available). Should not be called if there is no data left on the + // stream. + // + // max_size_hint can be set as a hint as to the maximum number + // of bytes that would be acceptable to read. + virtual bool Next(size_t max_size_hint, grpc_closure* on_complete) = 0; + + // Returns the next slice in the byte stream when it is available, as + // indicated by Next(). + // + // Once a slice is returned into *slice, it is owned by the caller. + virtual grpc_error_handle Pull(grpc_slice* slice) = 0; + + // Shuts down the byte stream. + // + // If there is a pending call to on_complete from Next(), it will be + // invoked with the error passed to Shutdown(). + // + // The next call to Pull() (if any) will return the error passed to + // Shutdown(). + virtual void Shutdown(grpc_error_handle error) = 0; + + uint32_t length() const { return length_; } + uint32_t flags() const { return flags_; } + + void set_flags(uint32_t flags) { flags_ = flags; } + + protected: + ByteStream(uint32_t length, uint32_t flags) + : length_(length), flags_(flags) {} + + private: + const uint32_t length_; + uint32_t flags_; +}; + +// +// SliceBufferByteStream +// +// A ByteStream that wraps a slice buffer. +// + +class SliceBufferByteStream : public ByteStream { + public: + // Removes all slices in slice_buffer, leaving it empty. + SliceBufferByteStream(grpc_slice_buffer* slice_buffer, uint32_t flags); + + ~SliceBufferByteStream() override; + + void Orphan() override; + + bool Next(size_t max_size_hint, grpc_closure* on_complete) override; + grpc_error_handle Pull(grpc_slice* slice) override; + void Shutdown(grpc_error_handle error) override; + + private: + grpc_error_handle shutdown_error_ = GRPC_ERROR_NONE; + grpc_slice_buffer backing_buffer_; +}; + +// +// CachingByteStream +// +// A ByteStream that that wraps an underlying byte stream but caches +// the resulting slices in a slice buffer. If an initial attempt fails +// without fully draining the underlying stream, a new caching stream +// can be created from the same underlying cache, in which case it will +// return whatever is in the backing buffer before continuing to read the +// underlying stream. +// +// NOTE: No synchronization is done, so it is not safe to have multiple +// CachingByteStreams simultaneously drawing from the same underlying +// ByteStreamCache at the same time. +// + +class ByteStreamCache { + public: + class CachingByteStream : public ByteStream { + public: + explicit CachingByteStream(ByteStreamCache* cache); + + ~CachingByteStream() override; + + void Orphan() override; + + bool Next(size_t max_size_hint, grpc_closure* on_complete) override; + grpc_error_handle Pull(grpc_slice* slice) override; + void Shutdown(grpc_error_handle error) override; + + // Resets the byte stream to the start of the underlying stream. + void Reset(); + + private: + ByteStreamCache* cache_; + size_t cursor_ = 0; + size_t offset_ = 0; + grpc_error_handle shutdown_error_ = GRPC_ERROR_NONE; + }; + + explicit ByteStreamCache(OrphanablePtr underlying_stream); + + ~ByteStreamCache(); + + // Must not be destroyed while still in use by a CachingByteStream. + void Destroy(); + + grpc_slice_buffer* cache_buffer() { return &cache_buffer_; } + + private: + OrphanablePtr underlying_stream_; + uint32_t length_; + uint32_t flags_; + grpc_slice_buffer cache_buffer_; +}; + +} // namespace grpc_core + +#endif /* GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H */ diff --git a/src/core/lib/transport/transport.cc b/src/core/lib/transport/transport.cc index 35e28349aed..b7c4950acb7 100644 --- a/src/core/lib/transport/transport.cc +++ b/src/core/lib/transport/transport.cc @@ -161,6 +161,9 @@ void grpc_transport_stream_op_batch_finish_with_failure( void grpc_transport_stream_op_batch_queue_finish_with_failure( grpc_transport_stream_op_batch* batch, grpc_error_handle error, grpc_core::CallCombinerClosureList* closures) { + if (batch->send_message) { + batch->payload->send_message.send_message.reset(); + } if (batch->cancel_stream) { GRPC_ERROR_UNREF(batch->payload->cancel_stream.cancel_error); } diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index c9b4bc96f89..069c53df87f 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -55,7 +55,7 @@ #include "src/core/lib/promise/latch.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport_fwd.h" @@ -70,17 +70,6 @@ struct grpc_transport_stream_op_batch_payload; #define GRPC_ARG_TRANSPORT "grpc.internal.transport" -/** Internal bit flag for grpc_begin_message's \a flags signaling the use of - * compression for the message. (Does not apply for stream compression.) */ -#define GRPC_WRITE_INTERNAL_COMPRESS (0x80000000u) -/** Internal bit flag for determining whether the message was compressed and had - * to be decompressed by the message_decompress filter. (Does not apply for - * stream compression.) */ -#define GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED (0x40000000u) -/** Mask of all valid internal flags. */ -#define GRPC_WRITE_INTERNAL_USED_MASK \ - (GRPC_WRITE_INTERNAL_COMPRESS | GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED) - namespace grpc_core { // TODO(ctiller): eliminate once MetadataHandle is constructable directly. namespace promise_filter_detail { @@ -342,6 +331,12 @@ struct grpc_transport_stream_op_batch_payload { explicit grpc_transport_stream_op_batch_payload( grpc_call_context_element* context) : context(context) {} + ~grpc_transport_stream_op_batch_payload() { + // We don't really own `send_message`, so release ownership and let the + // owner clean the data. + (void)send_message.send_message.release(); + } + struct { grpc_metadata_batch* send_initial_metadata = nullptr; /** Iff send_initial_metadata != NULL, flags associated with @@ -370,8 +365,7 @@ struct grpc_transport_stream_op_batch_payload { // the op gets down to the transport) takes ownership. // The batch's on_complete will not be called until after the byte // stream is orphaned. - grpc_core::SliceBuffer* send_message; - uint32_t flags = 0; + grpc_core::OrphanablePtr send_message; // Set by the transport if the stream has been closed for writes. If this // is set and send message op is present, we set the operation to be a // failure without sending a cancel OP down the stack. This is so that the @@ -410,11 +404,10 @@ struct grpc_transport_stream_op_batch_payload { } recv_initial_metadata; struct { - // Will be set by the transport to point to the byte stream containing a - // received message. Will be nullopt if trailing metadata is received - // instead of a message. - absl::optional* recv_message = nullptr; - uint32_t* flags = nullptr; + // Will be set by the transport to point to the byte stream + // containing a received message. + // Will be NULL if trailing metadata is received instead of a message. + grpc_core::OrphanablePtr* recv_message = nullptr; // Was this recv_message failed for reasons other than a clean end-of-stream bool* call_failed_before_recv_message = nullptr; /** Should be enqueued when one message is ready to be processed. */ diff --git a/src/core/lib/transport/transport_op_string.cc b/src/core/lib/transport/transport_op_string.cc index ae2459e57f1..685c2d80104 100644 --- a/src/core/lib/transport/transport_op_string.cc +++ b/src/core/lib/transport/transport_op_string.cc @@ -33,7 +33,7 @@ #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/error.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -54,9 +54,10 @@ std::string grpc_transport_stream_op_batch_string( if (op->send_message) { if (op->payload->send_message.send_message != nullptr) { - out.push_back(absl::StrFormat( - " SEND_MESSAGE:flags=0x%08x:len=%d", op->payload->send_message.flags, - op->payload->send_message.send_message->Length())); + out.push_back( + absl::StrFormat(" SEND_MESSAGE:flags=0x%08x:len=%d", + op->payload->send_message.send_message->flags(), + op->payload->send_message.send_message->length())); } else { // This can happen when we check a batch after the transport has // processed and cleared the send_message op. diff --git a/src/cpp/common/channel_filter.h b/src/cpp/common/channel_filter.h index 6667992f5a7..fc10c2cac6f 100644 --- a/src/cpp/common/channel_filter.h +++ b/src/cpp/common/channel_filter.h @@ -27,8 +27,6 @@ #include #include -#include "absl/types/optional.h" - #include #include #include @@ -37,11 +35,12 @@ #include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/context.h" +#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/polling_entity.h" -#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/channel_stack_type.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -153,21 +152,22 @@ class TransportStreamOpBatch { op_->payload->recv_initial_metadata.recv_initial_metadata_ready = closure; } - grpc_core::SliceBuffer* send_message() const { - return op_->send_message ? op_->payload->send_message.send_message + grpc_core::OrphanablePtr* send_message() const { + return op_->send_message ? &op_->payload->send_message.send_message : nullptr; } - - void set_send_message(grpc_core::SliceBuffer* send_message) { + void set_send_message( + grpc_core::OrphanablePtr send_message) { op_->send_message = true; - op_->payload->send_message.send_message = send_message; + op_->payload->send_message.send_message = std::move(send_message); } - absl::optional* recv_message() const { + grpc_core::OrphanablePtr* recv_message() const { return op_->recv_message ? op_->payload->recv_message.recv_message : nullptr; } - void set_recv_message(absl::optional* recv_message) { + void set_recv_message( + grpc_core::OrphanablePtr* recv_message) { op_->recv_message = true; op_->payload->recv_message.recv_message = recv_message; } diff --git a/src/cpp/ext/filters/census/client_filter.cc b/src/cpp/ext/filters/census/client_filter.cc index 410d7399807..9ad447b4237 100644 --- a/src/cpp/ext/filters/census/client_filter.cc +++ b/src/cpp/ext/filters/census/client_filter.cc @@ -50,8 +50,8 @@ #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_refcount.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" #include "src/cpp/ext/filters/census/context.h" @@ -137,12 +137,12 @@ void OpenCensusCallTracer::OpenCensusCallAttemptTracer:: } void OpenCensusCallTracer::OpenCensusCallAttemptTracer::RecordSendMessage( - const grpc_core::SliceBuffer& /*send_message*/) { + const grpc_core::ByteStream& /*send_message*/) { ++sent_message_count_; } void OpenCensusCallTracer::OpenCensusCallAttemptTracer::RecordReceivedMessage( - const grpc_core::SliceBuffer& /*recv_message*/) { + const grpc_core::ByteStream& /*recv_message*/) { ++recv_message_count_; } diff --git a/src/cpp/ext/filters/census/open_census_call_tracer.h b/src/cpp/ext/filters/census/open_census_call_tracer.h index f2f297f8600..990630c17ed 100644 --- a/src/cpp/ext/filters/census/open_census_call_tracer.h +++ b/src/cpp/ext/filters/census/open_census_call_tracer.h @@ -38,7 +38,7 @@ #include "src/core/lib/iomgr/error.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" #include "src/cpp/ext/filters/census/context.h" @@ -58,12 +58,12 @@ class OpenCensusCallTracer : public grpc_core::CallTracer { void RecordSendTrailingMetadata( grpc_metadata_batch* /*send_trailing_metadata*/) override {} void RecordSendMessage( - const grpc_core::SliceBuffer& /*send_message*/) override; + const grpc_core::ByteStream& /*send_message*/) override; void RecordReceivedInitialMetadata( grpc_metadata_batch* /*recv_initial_metadata*/, uint32_t /*flags*/) override {} void RecordReceivedMessage( - const grpc_core::SliceBuffer& /*recv_message*/) override; + const grpc_core::ByteStream& /*recv_message*/) override; void RecordReceivedTrailingMetadata( absl::Status status, grpc_metadata_batch* recv_trailing_metadata, const grpc_transport_stream_stats* transport_stream_stats) override; diff --git a/src/cpp/ext/filters/census/server_filter.cc b/src/cpp/ext/filters/census/server_filter.cc index e3586e14dc9..fe6d013d8aa 100644 --- a/src/cpp/ext/filters/census/server_filter.cc +++ b/src/cpp/ext/filters/census/server_filter.cc @@ -20,6 +20,7 @@ #include "src/cpp/ext/filters/census/server_filter.h" +#include #include #include "absl/strings/str_cat.h" @@ -81,7 +82,7 @@ void CensusServerCallData::OnDoneRecvMessageCb(void* user_data, GPR_ASSERT(calld != nullptr); GPR_ASSERT(channeld != nullptr); // Stream messages are no longer valid after receiving trailing metadata. - if (calld->recv_message_->has_value()) { + if ((*calld->recv_message_) != nullptr) { ++calld->recv_message_count_; } grpc_core::Closure::Run(DEBUG_LOCATION, calld->initial_on_done_recv_message_, diff --git a/src/cpp/ext/filters/census/server_filter.h b/src/cpp/ext/filters/census/server_filter.h index 55e542cb5d7..d5cf021d615 100644 --- a/src/cpp/ext/filters/census/server_filter.h +++ b/src/cpp/ext/filters/census/server_filter.h @@ -28,7 +28,6 @@ #include "absl/strings/string_view.h" #include "absl/time/time.h" -#include "absl/types/optional.h" #include #include @@ -36,10 +35,11 @@ #include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/cpp/common/channel_filter.h" #include "src/cpp/ext/filters/census/context.h" @@ -101,7 +101,7 @@ class CensusServerCallData : public CallData { grpc_closure on_done_recv_message_; absl::Time start_time_; absl::Duration elapsed_time_; - absl::optional* recv_message_; + grpc_core::OrphanablePtr* recv_message_; uint64_t recv_message_count_; uint64_t sent_message_count_; // Buffer needed for grpc_slice to reference it when adding metatdata to diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 3a6b25838db..df1e41a52b8 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -678,6 +678,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', + 'src/core/lib/transport/byte_stream.cc', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/handshaker.cc', diff --git a/src/python/grpcio_tests/tests/unit/_compression_test.py b/src/python/grpcio_tests/tests/unit/_compression_test.py index 30a5d6bf685..d58f4dc06ba 100644 --- a/src/python/grpcio_tests/tests/unit/_compression_test.py +++ b/src/python/grpcio_tests/tests/unit/_compression_test.py @@ -200,15 +200,24 @@ def _get_compression_ratios(client_function, first_channel_kwargs, first_server_handler, second_channel_kwargs, second_multicallable_kwargs, second_server_kwargs, second_server_handler, message): - first_bytes_sent, first_bytes_received = _get_byte_counts( - first_channel_kwargs, first_multicallable_kwargs, client_function, - first_server_kwargs, first_server_handler, message) - second_bytes_sent, second_bytes_received = _get_byte_counts( - second_channel_kwargs, second_multicallable_kwargs, client_function, - second_server_kwargs, second_server_handler, message) - return ((second_bytes_sent - first_bytes_sent) / float(first_bytes_sent), - (second_bytes_received - first_bytes_received) / - float(first_bytes_received)) + try: + # This test requires the byte length of each connection to be deterministic. As + # it turns out, flow control puts bytes on the wire in a nondeterministic + # manner. We disable it here in order to measure compression ratios + # deterministically. + os.environ['GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL'] = 'true' + first_bytes_sent, first_bytes_received = _get_byte_counts( + first_channel_kwargs, first_multicallable_kwargs, client_function, + first_server_kwargs, first_server_handler, message) + second_bytes_sent, second_bytes_received = _get_byte_counts( + second_channel_kwargs, second_multicallable_kwargs, client_function, + second_server_kwargs, second_server_handler, message) + return ((second_bytes_sent - first_bytes_sent) / + float(first_bytes_sent), + (second_bytes_received - first_bytes_received) / + float(first_bytes_received)) + finally: + del os.environ['GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL'] def _unary_unary_client(channel, multicallable_kwargs, message): @@ -293,28 +302,37 @@ class CompressionTest(unittest.TestCase): server_handler = _GenericHandler( functools.partial(set_call_compression, grpc.Compression.Gzip) ) if server_call_compression else _GenericHandler(None) - _get_compression_ratios(client_function, {}, {}, {}, - _GenericHandler(None), channel_kwargs, - multicallable_kwargs, server_kwargs, - server_handler, _REQUEST) + sent_ratio, received_ratio = _get_compression_ratios( + client_function, {}, {}, {}, _GenericHandler(None), channel_kwargs, + multicallable_kwargs, server_kwargs, server_handler, _REQUEST) + + if client_side_compressed: + self.assertCompressed(sent_ratio) + else: + self.assertNotCompressed(sent_ratio) + + if server_side_compressed: + self.assertCompressed(received_ratio) + else: + self.assertNotCompressed(received_ratio) def testDisableNextCompressionStreaming(self): server_kwargs = { 'compression': grpc.Compression.Deflate, } - _get_compression_ratios(_stream_stream_client, {}, {}, {}, - _GenericHandler(None), {}, {}, server_kwargs, - _GenericHandler(disable_next_compression), - _REQUEST) + _, received_ratio = _get_compression_ratios( + _stream_stream_client, {}, {}, {}, _GenericHandler(None), {}, {}, + server_kwargs, _GenericHandler(disable_next_compression), _REQUEST) + self.assertNotCompressed(received_ratio) def testDisableNextCompressionStreamingResets(self): server_kwargs = { 'compression': grpc.Compression.Deflate, } - _get_compression_ratios(_stream_stream_client, {}, {}, {}, - _GenericHandler(None), {}, {}, server_kwargs, - _GenericHandler(disable_first_compression), - _REQUEST) + _, received_ratio = _get_compression_ratios( + _stream_stream_client, {}, {}, {}, _GenericHandler(None), {}, {}, + server_kwargs, _GenericHandler(disable_first_compression), _REQUEST) + self.assertCompressed(received_ratio) def _get_compression_str(name, value): diff --git a/test/core/bad_client/tests/duplicate_header.cc b/test/core/bad_client/tests/duplicate_header.cc index 09c5cfc62d6..28c5f310e4a 100644 --- a/test/core/bad_client/tests/duplicate_header.cc +++ b/test/core/bad_client/tests/duplicate_header.cc @@ -90,7 +90,7 @@ static void verifier(grpc_server* server, grpc_completion_queue* cq, nullptr); GPR_ASSERT(GRPC_CALL_OK == error); - CQ_EXPECT_COMPLETION_ANY_STATUS(cqv, tag(102)); + CQ_EXPECT_COMPLETION(cqv, tag(102), 1); cq_verify(cqv); memset(ops, 0, sizeof(ops)); diff --git a/test/core/end2end/fixtures/h2_full_no_retry.cc b/test/core/end2end/fixtures/h2_full_no_retry.cc deleted file mode 100644 index d69f502f273..00000000000 --- a/test/core/end2end/fixtures/h2_full_no_retry.cc +++ /dev/null @@ -1,118 +0,0 @@ -/* - * - * Copyright 2015 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include - -#include -#include -#include -#include - -#include "src/core/ext/filters/client_channel/client_channel.h" -#include "src/core/ext/filters/http/server/http_server_filter.h" -#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" -#include "src/core/lib/channel/connected_channel.h" -#include "src/core/lib/gprpp/host_port.h" -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/surface/server.h" -#include "test/core/end2end/end2end_tests.h" -#include "test/core/util/port.h" -#include "test/core/util/test_config.h" - -struct fullstack_fixture_data { - std::string localaddr; -}; - -static grpc_end2end_test_fixture chttp2_create_fixture_fullstack( - const grpc_channel_args* /*client_args*/, - const grpc_channel_args* /*server_args*/) { - grpc_end2end_test_fixture f; - int port = grpc_pick_unused_port_or_die(); - fullstack_fixture_data* ffd = new fullstack_fixture_data(); - memset(&f, 0, sizeof(f)); - - ffd->localaddr = grpc_core::JoinHostPort("localhost", port); - - f.fixture_data = ffd; - f.cq = grpc_completion_queue_create_for_next(nullptr); - - return f; -} - -void chttp2_init_client_fullstack(grpc_end2end_test_fixture* f, - const grpc_channel_args* client_args) { - fullstack_fixture_data* ffd = - static_cast(f->fixture_data); - grpc_channel_credentials* creds = grpc_insecure_credentials_create(); - grpc_arg no_retry = grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_ENABLE_RETRIES), 0); - client_args = grpc_channel_args_copy_and_add(client_args, &no_retry, 1); - f->client = grpc_channel_create(ffd->localaddr.c_str(), creds, client_args); - grpc_channel_args_destroy(client_args); - grpc_channel_credentials_release(creds); - GPR_ASSERT(f->client); -} - -void chttp2_init_server_fullstack(grpc_end2end_test_fixture* f, - const grpc_channel_args* server_args) { - fullstack_fixture_data* ffd = - static_cast(f->fixture_data); - if (f->server) { - grpc_server_destroy(f->server); - } - f->server = grpc_server_create(server_args, nullptr); - grpc_server_register_completion_queue(f->server, f->cq, nullptr); - grpc_server_credentials* server_creds = - grpc_insecure_server_credentials_create(); - GPR_ASSERT(grpc_server_add_http2_port(f->server, ffd->localaddr.c_str(), - server_creds)); - grpc_server_credentials_release(server_creds); - grpc_server_start(f->server); -} - -void chttp2_tear_down_fullstack(grpc_end2end_test_fixture* f) { - fullstack_fixture_data* ffd = - static_cast(f->fixture_data); - delete ffd; -} - -/* All test configurations */ -static grpc_end2end_test_config configs[] = { - {"chttp2/fullstack", - FEATURE_MASK_SUPPORTS_DELAYED_CONNECTION | - FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | - FEATURE_MASK_SUPPORTS_AUTHORITY_HEADER, - nullptr, chttp2_create_fixture_fullstack, chttp2_init_client_fullstack, - chttp2_init_server_fullstack, chttp2_tear_down_fullstack}, -}; - -int main(int argc, char** argv) { - size_t i; - - grpc::testing::TestEnvironment env(&argc, argv); - grpc_end2end_tests_pre_init(); - grpc_init(); - - for (i = 0; i < sizeof(configs) / sizeof(*configs); i++) { - grpc_end2end_tests(argc, argv, configs[i]); - } - - grpc_shutdown(); - - return 0; -} diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index a50d22cce88..e585de0aff7 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -36,11 +36,8 @@ def _fixture_options( supports_write_buffering = True, client_channel = True, supports_msvc = True, - supports_retry = None, flaky_tests = [], tags = []): - if supports_retry == None: - supports_retry = client_channel return struct( fullstack = fullstack, includes_proxy = includes_proxy, @@ -57,7 +54,6 @@ def _fixture_options( supports_msvc = supports_msvc, _platforms = _platforms, flaky_tests = flaky_tests, - supports_retry = supports_retry, tags = tags, ) @@ -77,7 +73,6 @@ END2END_FIXTURES = { tags = ["no_test_ios"], ), "h2_full": _fixture_options(), - "h2_full_no_retry": _fixture_options(supports_retry = False), "h2_full+pipe": _fixture_options(_platforms = ["linux"]), "h2_full+trace": _fixture_options(tracing = True), "h2_http_proxy": _fixture_options(supports_proxy_auth = True), @@ -160,7 +155,6 @@ def _test_options( needs_proxy_auth = False, needs_write_buffering = False, needs_client_channel = False, - needs_retry = False, short_name = None, exclude_pollers = []): return struct( @@ -176,7 +170,6 @@ def _test_options( needs_proxy_auth = needs_proxy_auth, needs_write_buffering = needs_write_buffering, needs_client_channel = needs_client_channel, - needs_retry = needs_retry, short_name = short_name, exclude_pollers = exclude_pollers, ) @@ -257,89 +250,81 @@ END2END_TESTS = { "registered_call": _test_options(), "request_with_flags": _test_options(proxyable = False), "request_with_payload": _test_options(), - "retry": _test_options(needs_client_channel = True, needs_retry = True), - "retry_cancellation": _test_options(needs_client_channel = True, needs_retry = True), - "retry_cancel_during_delay": _test_options(needs_client_channel = True, needs_retry = True), + "retry": _test_options(needs_client_channel = True), + "retry_cancellation": _test_options(needs_client_channel = True), + "retry_cancel_during_delay": _test_options(needs_client_channel = True), "retry_cancel_with_multiple_send_batches": _test_options( # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_cancel3", needs_client_channel = True, - needs_retry = True, ), "retry_cancel_after_first_attempt_starts": _test_options( # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_cancel4", needs_client_channel = True, - needs_retry = True, ), - "retry_disabled": _test_options(needs_client_channel = True, needs_retry = True), - "retry_exceeds_buffer_size_in_delay": _test_options(needs_client_channel = True, needs_retry = True), + "retry_disabled": _test_options(needs_client_channel = True), + "retry_exceeds_buffer_size_in_delay": _test_options(needs_client_channel = True), "retry_exceeds_buffer_size_in_initial_batch": _test_options( needs_client_channel = True, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_exceeds_buffer_size_in_init", - needs_retry = True, ), "retry_exceeds_buffer_size_in_subsequent_batch": _test_options( needs_client_channel = True, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_exceeds_buffer_size_in_subseq", - needs_retry = True, ), - "retry_lb_drop": _test_options(needs_client_channel = True, needs_retry = True), - "retry_lb_fail": _test_options(needs_client_channel = True, needs_retry = True), - "retry_non_retriable_status": _test_options(needs_client_channel = True, needs_retry = True), + "retry_lb_drop": _test_options(needs_client_channel = True), + "retry_lb_fail": _test_options(needs_client_channel = True), + "retry_non_retriable_status": _test_options(needs_client_channel = True), "retry_non_retriable_status_before_recv_trailing_metadata_started": _test_options( needs_client_channel = True, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_non_retriable_status2", - needs_retry = True, ), - "retry_per_attempt_recv_timeout": _test_options(needs_client_channel = True, needs_retry = True), + "retry_per_attempt_recv_timeout": _test_options(needs_client_channel = True), "retry_per_attempt_recv_timeout_on_last_attempt": _test_options( needs_client_channel = True, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_per_attempt_recv_timeout2", - needs_retry = True, ), - "retry_recv_initial_metadata": _test_options(needs_client_channel = True, needs_retry = True), - "retry_recv_message": _test_options(needs_client_channel = True, needs_retry = True), - "retry_recv_message_replay": _test_options(needs_client_channel = True, needs_retry = True), - "retry_recv_trailing_metadata_error": _test_options(needs_client_channel = True, needs_retry = True), - "retry_send_initial_metadata_refs": _test_options(needs_client_channel = True, needs_retry = True), - "retry_send_op_fails": _test_options(needs_client_channel = True, needs_retry = True), - "retry_send_recv_batch": _test_options(needs_client_channel = True, needs_retry = True), - "retry_server_pushback_delay": _test_options(needs_client_channel = True, needs_retry = True), - "retry_server_pushback_disabled": _test_options(needs_client_channel = True, needs_retry = True), - "retry_streaming": _test_options(needs_client_channel = True, needs_retry = True), - "retry_streaming_after_commit": _test_options(needs_client_channel = True, needs_retry = True), + "retry_recv_initial_metadata": _test_options(needs_client_channel = True), + "retry_recv_message": _test_options(needs_client_channel = True), + "retry_recv_message_replay": _test_options(needs_client_channel = True), + "retry_recv_trailing_metadata_error": _test_options(needs_client_channel = True), + "retry_send_initial_metadata_refs": _test_options(needs_client_channel = True), + "retry_send_op_fails": _test_options(needs_client_channel = True), + "retry_send_recv_batch": _test_options(needs_client_channel = True), + "retry_server_pushback_delay": _test_options(needs_client_channel = True), + "retry_server_pushback_disabled": _test_options(needs_client_channel = True), + "retry_streaming": _test_options(needs_client_channel = True), + "retry_streaming_after_commit": _test_options(needs_client_channel = True), "retry_streaming_succeeds_before_replay_finished": _test_options( needs_client_channel = True, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_streaming2", - needs_retry = True, ), - "retry_throttled": _test_options(needs_client_channel = True, needs_retry = True), - "retry_too_many_attempts": _test_options(needs_client_channel = True, needs_retry = True), - "retry_transparent_goaway": _test_options(needs_client_channel = True, needs_retry = True), - "retry_transparent_not_sent_on_wire": _test_options(needs_client_channel = True, needs_retry = True), + "retry_throttled": _test_options(needs_client_channel = True), + "retry_too_many_attempts": _test_options(needs_client_channel = True), + "retry_transparent_goaway": _test_options(needs_client_channel = True), + "retry_transparent_not_sent_on_wire": _test_options(needs_client_channel = True), "retry_transparent_max_concurrent_streams": _test_options( needs_client_channel = True, proxyable = False, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_transparent_mcs", - needs_retry = True, ), - "retry_unref_before_finish": _test_options(needs_client_channel = True, needs_retry = True), - "retry_unref_before_recv": _test_options(needs_client_channel = True, needs_retry = True), + "retry_unref_before_finish": _test_options(needs_client_channel = True), + "retry_unref_before_recv": _test_options(needs_client_channel = True), "server_finishes_request": _test_options(), "server_streaming": _test_options(needs_http2 = True), "shutdown_finishes_calls": _test_options(), @@ -390,9 +375,6 @@ def _compatible(fopt, topt): if topt.needs_client_channel: if not fopt.client_channel: return False - if topt.needs_retry: - if not fopt.supports_retry: - return False return True def _platform_support_tags(fopt): diff --git a/test/core/end2end/tests/request_with_flags.cc b/test/core/end2end/tests/request_with_flags.cc index b277036dffb..59e137aeca3 100644 --- a/test/core/end2end/tests/request_with_flags.cc +++ b/test/core/end2end/tests/request_with_flags.cc @@ -25,7 +25,7 @@ #include #include "src/core/lib/gpr/useful.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/byte_stream.h" #include "test/core/end2end/cq_verifier.h" #include "test/core/end2end/end2end_tests.h" diff --git a/test/core/gprpp/BUILD b/test/core/gprpp/BUILD index b5f48b9887d..b61ac9f6e98 100644 --- a/test/core/gprpp/BUILD +++ b/test/core/gprpp/BUILD @@ -84,6 +84,18 @@ grpc_cc_test( ], ) +grpc_cc_test( + name = "manual_constructor_test", + srcs = ["manual_constructor_test.cc"], + language = "C++", + uses_event_engine = False, + uses_polling = False, + deps = [ + "//:gpr", + "//test/core/util:grpc_test_util", + ], +) + grpc_cc_test( name = "bitset_test", srcs = ["bitset_test.cc"], diff --git a/test/core/gprpp/manual_constructor_test.cc b/test/core/gprpp/manual_constructor_test.cc new file mode 100644 index 00000000000..b5e5873bf3f --- /dev/null +++ b/test/core/gprpp/manual_constructor_test.cc @@ -0,0 +1,100 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* Test of gpr synchronization support. */ + +#include "src/core/lib/gprpp/manual_constructor.h" + +#include +#include + +#include + +#include +#include +#include + +#include "test/core/util/test_config.h" + +class A { + public: + A() {} + virtual ~A() {} + virtual const char* foo() { return "A_foo"; } + virtual const char* bar() { return "A_bar"; } +}; + +class B : public A { + public: + B() {} + ~B() override {} + const char* foo() override { return "B_foo"; } + char get_junk() { return junk[0]; } + + private: + char junk[1000]; +}; + +class C : public B { + public: + C() {} + ~C() override {} + const char* bar() override { return "C_bar"; } + char get_more_junk() { return more_junk[0]; } + + private: + char more_junk[1000]; +}; + +class D : public A { + public: + const char* bar() override { return "D_bar"; } +}; + +static void basic_test() { + grpc_core::PolymorphicManualConstructor poly; + poly.Init(); + GPR_ASSERT(!strcmp(poly->foo(), "B_foo")); + GPR_ASSERT(!strcmp(poly->bar(), "A_bar")); +} + +static void complex_test() { + grpc_core::PolymorphicManualConstructor polyB; + polyB.Init(); + GPR_ASSERT(!strcmp(polyB->foo(), "B_foo")); + GPR_ASSERT(!strcmp(polyB->bar(), "A_bar")); + + grpc_core::PolymorphicManualConstructor polyC; + polyC.Init(); + GPR_ASSERT(!strcmp(polyC->foo(), "B_foo")); + GPR_ASSERT(!strcmp(polyC->bar(), "C_bar")); + + grpc_core::PolymorphicManualConstructor polyD; + polyD.Init(); + GPR_ASSERT(!strcmp(polyD->foo(), "A_foo")); + GPR_ASSERT(!strcmp(polyD->bar(), "D_bar")); +} + +/* ------------------------------------------------- */ + +int main(int argc, char* argv[]) { + grpc::testing::TestEnvironment env(&argc, argv); + basic_test(); + complex_test(); + return 0; +} diff --git a/test/core/transport/BUILD b/test/core/transport/BUILD index 94264237979..4909a1ec4e8 100644 --- a/test/core/transport/BUILD +++ b/test/core/transport/BUILD @@ -35,6 +35,22 @@ grpc_cc_test( ], ) +grpc_cc_test( + name = "byte_stream_test", + srcs = ["byte_stream_test.cc"], + external_deps = [ + "gtest", + ], + language = "C++", + uses_event_engine = False, + uses_polling = False, + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:grpc_test_util", + ], +) + grpc_cc_test( name = "connectivity_state_test", srcs = ["connectivity_state_test.cc"], diff --git a/test/core/transport/binder/binder_transport_test.cc b/test/core/transport/binder/binder_transport_test.cc index fabbc155768..b61f55c9d7c 100644 --- a/test/core/transport/binder/binder_transport_test.cc +++ b/test/core/transport/binder/binder_transport_test.cc @@ -242,13 +242,19 @@ struct MakeSendInitialMetadata { struct MakeSendMessage { MakeSendMessage(const std::string& message, grpc_transport_stream_op_batch* op) { - send_stream.Append(grpc_core::Slice::FromCopiedString(message)); + grpc_slice_buffer send_buffer; + grpc_slice_buffer_init(&send_buffer); + grpc_slice send_slice = grpc_slice_from_cpp_string(message); + grpc_slice_buffer_add(&send_buffer, send_slice); + + send_stream.Init(&send_buffer, 0); + grpc_slice_buffer_destroy(&send_buffer); op->send_message = true; - op->payload->send_message.send_message = &send_stream; + op->payload->send_message.send_message.reset(send_stream.get()); } - grpc_core::SliceBuffer send_stream; + grpc_core::ManualConstructor send_stream; }; struct MakeSendTrailingMetadata { @@ -307,7 +313,7 @@ struct MakeRecvMessage { MockGrpcClosure ready; absl::Notification notification; - absl::optional grpc_message; + grpc_core::OrphanablePtr grpc_message; }; struct MakeRecvTrailingMetadata { @@ -559,7 +565,13 @@ TEST_F(BinderTransportTest, PerformRecvMessage) { grpc_core::ExecCtx::Get()->Flush(); recv_message.notification.WaitForNotification(); - EXPECT_EQ(kMessage, recv_message.grpc_message->JoinIntoString()); + EXPECT_TRUE(recv_message.grpc_message->Next(SIZE_MAX, nullptr)); + grpc_slice slice; + EXPECT_EQ(recv_message.grpc_message->Pull(&slice), GRPC_ERROR_NONE); + EXPECT_EQ(kMessage, + std::string(reinterpret_cast(GRPC_SLICE_START_PTR(slice)), + GRPC_SLICE_LENGTH(slice))); + grpc_slice_unref_internal(slice); } TEST_F(BinderTransportTest, PerformRecvTrailingMetadata) { @@ -617,7 +629,13 @@ TEST_F(BinderTransportTest, PerformRecvAll) { trailing_metadata.emplace_back("grpc-status", std::to_string(kStatus)); VerifyMetadataEqual(trailing_metadata, recv_trailing_metadata.grpc_trailing_metadata); - EXPECT_EQ(kMessage, recv_message.grpc_message->JoinIntoString()); + EXPECT_TRUE(recv_message.grpc_message->Next(SIZE_MAX, nullptr)); + grpc_slice slice; + EXPECT_EQ(recv_message.grpc_message->Pull(&slice), GRPC_ERROR_NONE); + EXPECT_EQ(kMessage, + std::string(reinterpret_cast(GRPC_SLICE_START_PTR(slice)), + GRPC_SLICE_LENGTH(slice))); + grpc_slice_unref_internal(slice); } TEST_F(BinderTransportTest, PerformAllOps) { @@ -688,7 +706,13 @@ TEST_F(BinderTransportTest, PerformAllOps) { VerifyMetadataEqual(AppendStatus(kRecvTrailingMetadata, kStatus), recv_trailing_metadata.grpc_trailing_metadata); - EXPECT_EQ(kRecvMessage, recv_message.grpc_message->JoinIntoString()); + EXPECT_TRUE(recv_message.grpc_message->Next(SIZE_MAX, nullptr)); + grpc_slice slice; + EXPECT_EQ(recv_message.grpc_message->Pull(&slice), GRPC_ERROR_NONE); + EXPECT_EQ(kRecvMessage, + std::string(reinterpret_cast(GRPC_SLICE_START_PTR(slice)), + GRPC_SLICE_LENGTH(slice))); + grpc_slice_unref_internal(slice); } TEST_F(BinderTransportTest, WireWriterRpcCallErrorPropagates) { diff --git a/test/core/transport/byte_stream_test.cc b/test/core/transport/byte_stream_test.cc new file mode 100644 index 00000000000..3b3ff743484 --- /dev/null +++ b/test/core/transport/byte_stream_test.cc @@ -0,0 +1,254 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "src/core/lib/transport/byte_stream.h" + +#include + +#include +#include +#include + +#include "src/core/lib/gpr/useful.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/slice/slice_internal.h" +#include "test/core/util/test_config.h" + +namespace grpc_core { +namespace { + +// +// SliceBufferByteStream tests +// + +void NotCalledClosure(void* /*arg*/, grpc_error_handle /*error*/) { + GPR_ASSERT(false); +} + +TEST(SliceBufferByteStream, Basic) { + ExecCtx exec_ctx; + // Create and populate slice buffer. + grpc_slice_buffer buffer; + grpc_slice_buffer_init(&buffer); + grpc_slice input[] = { + grpc_slice_from_static_string("foo"), + grpc_slice_from_static_string("bar"), + }; + for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { + grpc_slice_buffer_add(&buffer, input[i]); + } + // Create byte stream. + SliceBufferByteStream stream(&buffer, 0); + grpc_slice_buffer_destroy_internal(&buffer); + EXPECT_EQ(6U, stream.length()); + grpc_closure closure; + GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr, + grpc_schedule_on_exec_ctx); + // Read each slice. Note that Next() always returns synchronously. + for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { + ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); + grpc_slice output; + grpc_error_handle error = stream.Pull(&output); + EXPECT_TRUE(error == GRPC_ERROR_NONE); + EXPECT_TRUE(grpc_slice_eq(input[i], output)); + grpc_slice_unref_internal(output); + } + // Clean up. + stream.Orphan(); +} + +TEST(SliceBufferByteStream, Shutdown) { + ExecCtx exec_ctx; + // Create and populate slice buffer. + grpc_slice_buffer buffer; + grpc_slice_buffer_init(&buffer); + grpc_slice input[] = { + grpc_slice_from_static_string("foo"), + grpc_slice_from_static_string("bar"), + }; + for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { + grpc_slice_buffer_add(&buffer, input[i]); + } + // Create byte stream. + SliceBufferByteStream stream(&buffer, 0); + grpc_slice_buffer_destroy_internal(&buffer); + EXPECT_EQ(6U, stream.length()); + grpc_closure closure; + GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr, + grpc_schedule_on_exec_ctx); + // Read the first slice. + ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); + grpc_slice output; + grpc_error_handle error = stream.Pull(&output); + EXPECT_TRUE(error == GRPC_ERROR_NONE); + EXPECT_TRUE(grpc_slice_eq(input[0], output)); + grpc_slice_unref_internal(output); + // Now shutdown. + grpc_error_handle shutdown_error = + GRPC_ERROR_CREATE_FROM_STATIC_STRING("shutdown error"); + stream.Shutdown(GRPC_ERROR_REF(shutdown_error)); + // After shutdown, the next pull() should return the error. + ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); + error = stream.Pull(&output); + EXPECT_TRUE(error == shutdown_error); + GRPC_ERROR_UNREF(error); + GRPC_ERROR_UNREF(shutdown_error); + // Clean up. + stream.Orphan(); +} + +// +// CachingByteStream tests +// + +TEST(CachingByteStream, Basic) { + ExecCtx exec_ctx; + // Create and populate slice buffer byte stream. + grpc_slice_buffer buffer; + grpc_slice_buffer_init(&buffer); + grpc_slice input[] = { + grpc_slice_from_static_string("foo"), + grpc_slice_from_static_string("bar"), + }; + for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { + grpc_slice_buffer_add(&buffer, input[i]); + } + SliceBufferByteStream underlying_stream(&buffer, 0); + grpc_slice_buffer_destroy_internal(&buffer); + // Create cache and caching stream. + ByteStreamCache cache((OrphanablePtr(&underlying_stream))); + ByteStreamCache::CachingByteStream stream(&cache); + grpc_closure closure; + GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr, + grpc_schedule_on_exec_ctx); + // Read each slice. Note that next() always returns synchronously, + // because the underlying byte stream always does. + for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { + ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); + grpc_slice output; + grpc_error_handle error = stream.Pull(&output); + EXPECT_TRUE(error == GRPC_ERROR_NONE); + EXPECT_TRUE(grpc_slice_eq(input[i], output)); + grpc_slice_unref_internal(output); + } + // Clean up. + stream.Orphan(); + cache.Destroy(); +} + +TEST(CachingByteStream, Reset) { + ExecCtx exec_ctx; + // Create and populate slice buffer byte stream. + grpc_slice_buffer buffer; + grpc_slice_buffer_init(&buffer); + grpc_slice input[] = { + grpc_slice_from_static_string("foo"), + grpc_slice_from_static_string("bar"), + }; + for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { + grpc_slice_buffer_add(&buffer, input[i]); + } + SliceBufferByteStream underlying_stream(&buffer, 0); + grpc_slice_buffer_destroy_internal(&buffer); + // Create cache and caching stream. + ByteStreamCache cache((OrphanablePtr(&underlying_stream))); + ByteStreamCache::CachingByteStream stream(&cache); + grpc_closure closure; + GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr, + grpc_schedule_on_exec_ctx); + // Read one slice. + ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); + grpc_slice output; + grpc_error_handle error = stream.Pull(&output); + EXPECT_TRUE(error == GRPC_ERROR_NONE); + EXPECT_TRUE(grpc_slice_eq(input[0], output)); + grpc_slice_unref_internal(output); + // Reset the caching stream. The reads should start over from the + // first slice. + stream.Reset(); + for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { + ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); + error = stream.Pull(&output); + EXPECT_TRUE(error == GRPC_ERROR_NONE); + EXPECT_TRUE(grpc_slice_eq(input[i], output)); + grpc_slice_unref_internal(output); + } + // Clean up. + stream.Orphan(); + cache.Destroy(); +} + +TEST(CachingByteStream, SharedCache) { + ExecCtx exec_ctx; + // Create and populate slice buffer byte stream. + grpc_slice_buffer buffer; + grpc_slice_buffer_init(&buffer); + grpc_slice input[] = { + grpc_slice_from_static_string("foo"), + grpc_slice_from_static_string("bar"), + }; + for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { + grpc_slice_buffer_add(&buffer, input[i]); + } + SliceBufferByteStream underlying_stream(&buffer, 0); + grpc_slice_buffer_destroy_internal(&buffer); + // Create cache and two caching streams. + ByteStreamCache cache((OrphanablePtr(&underlying_stream))); + ByteStreamCache::CachingByteStream stream1(&cache); + ByteStreamCache::CachingByteStream stream2(&cache); + grpc_closure closure; + GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr, + grpc_schedule_on_exec_ctx); + // Read one slice from stream1. + EXPECT_TRUE(stream1.Next(~(size_t)0, &closure)); + grpc_slice output; + grpc_error_handle error = stream1.Pull(&output); + EXPECT_TRUE(error == GRPC_ERROR_NONE); + EXPECT_TRUE(grpc_slice_eq(input[0], output)); + grpc_slice_unref_internal(output); + // Read all slices from stream2. + for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { + EXPECT_TRUE(stream2.Next(~(size_t)0, &closure)); + error = stream2.Pull(&output); + EXPECT_TRUE(error == GRPC_ERROR_NONE); + EXPECT_TRUE(grpc_slice_eq(input[i], output)); + grpc_slice_unref_internal(output); + } + // Now read the second slice from stream1. + EXPECT_TRUE(stream1.Next(~(size_t)0, &closure)); + error = stream1.Pull(&output); + EXPECT_TRUE(error == GRPC_ERROR_NONE); + EXPECT_TRUE(grpc_slice_eq(input[1], output)); + grpc_slice_unref_internal(output); + // Clean up. + stream1.Orphan(); + stream2.Orphan(); + cache.Destroy(); +} + +} // namespace +} // namespace grpc_core + +int main(int argc, char** argv) { + grpc::testing::TestEnvironment env(&argc, argv); + ::testing::InitGoogleTest(&argc, argv); + grpc_init(); + int retval = RUN_ALL_TESTS(); + grpc_shutdown(); + return retval; +} diff --git a/test/core/transport/chttp2/BUILD b/test/core/transport/chttp2/BUILD index 1adf32651db..7721baa8233 100644 --- a/test/core/transport/chttp2/BUILD +++ b/test/core/transport/chttp2/BUILD @@ -86,9 +86,9 @@ grpc_cc_test( ) grpc_cc_test( - name = "flow_control_end2end_test", + name = "flow_control_test", size = "large", - srcs = ["flow_control_end2end_test.cc"], + srcs = ["flow_control_test.cc"], external_deps = [ "gtest", ], @@ -101,20 +101,6 @@ grpc_cc_test( ], ) -grpc_cc_test( - name = "flow_control_test", - srcs = ["flow_control_test.cc"], - external_deps = [ - "gtest", - ], - language = "C++", - deps = [ - "//:chttp2_flow_control", - "//:resource_quota", - "//test/core/util:grpc_suppressions", - ], -) - grpc_cc_test( name = "graceful_shutdown_test", srcs = ["graceful_shutdown_test.cc"], diff --git a/test/core/transport/chttp2/flow_control_end2end_test.cc b/test/core/transport/chttp2/flow_control_end2end_test.cc deleted file mode 100644 index 9d0e52ac536..00000000000 --- a/test/core/transport/chttp2/flow_control_end2end_test.cc +++ /dev/null @@ -1,379 +0,0 @@ -/* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include - -#include -#include - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "src/core/ext/filters/client_channel/backup_poller.h" -#include "src/core/ext/transport/chttp2/transport/flow_control.h" -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/gprpp/host_port.h" -#include "src/core/lib/surface/channel.h" -#include "test/core/end2end/cq_verifier.h" -#include "test/core/util/port.h" -#include "test/core/util/test_config.h" - -namespace { - -class TransportTargetWindowSizeMocker - : public grpc_core::chttp2::TestOnlyTransportTargetWindowEstimatesMocker { - public: - static constexpr uint32_t kLargeInitialWindowSize = 1u << 31; - static constexpr uint32_t kSmallInitialWindowSize = 0; - - double ComputeNextTargetInitialWindowSizeFromPeriodicUpdate( - double /* current_target */) override { - // Protecting access to variable window_size_ shared between client and - // server. - grpc_core::MutexLock lock(&mu_); - if (alternating_initial_window_sizes_) { - window_size_ = (window_size_ == kLargeInitialWindowSize) - ? kSmallInitialWindowSize - : kLargeInitialWindowSize; - } - return window_size_; - } - - // Alternates the initial window size targets. Computes a low values if it was - // previously high, or a high value if it was previously low. - void AlternateTargetInitialWindowSizes() { - grpc_core::MutexLock lock(&mu_); - alternating_initial_window_sizes_ = true; - } - - void Reset() { - // Protecting access to variable window_size_ shared between client and - // server. - grpc_core::MutexLock lock(&mu_); - alternating_initial_window_sizes_ = false; - window_size_ = kLargeInitialWindowSize; - } - - private: - grpc_core::Mutex mu_; - bool alternating_initial_window_sizes_ ABSL_GUARDED_BY(mu_) = false; - double window_size_ ABSL_GUARDED_BY(mu_) = kLargeInitialWindowSize; -}; - -TransportTargetWindowSizeMocker* g_target_initial_window_size_mocker; - -void* tag(intptr_t t) { return reinterpret_cast(t); } - -void VerifyChannelReady(grpc_channel* channel, grpc_completion_queue* cq) { - grpc_connectivity_state state = - grpc_channel_check_connectivity_state(channel, 1 /* try_to_connect */); - while (state != GRPC_CHANNEL_READY) { - grpc_channel_watch_connectivity_state( - channel, state, grpc_timeout_seconds_to_deadline(5), cq, nullptr); - grpc_completion_queue_next(cq, grpc_timeout_seconds_to_deadline(5), - nullptr); - state = grpc_channel_check_connectivity_state(channel, 0); - } -} - -void VerifyChannelConnected(grpc_channel* channel, grpc_completion_queue* cq) { - // Verify channel is connected. Use a ping to make sure that clients - // tries sending/receiving bytes if the channel is connected. - grpc_channel_ping(channel, cq, reinterpret_cast(2000), nullptr); - grpc_event ev = grpc_completion_queue_next( - cq, grpc_timeout_seconds_to_deadline(5), nullptr); - GPR_ASSERT(ev.type == GRPC_OP_COMPLETE); - GPR_ASSERT(ev.tag == reinterpret_cast(2000)); - GPR_ASSERT(ev.success == 1); - GPR_ASSERT(grpc_channel_check_connectivity_state(channel, 0) == - GRPC_CHANNEL_READY); -} - -// Shuts down and destroys the server. -void ServerShutdownAndDestroy(grpc_server* server, grpc_completion_queue* cq) { - // Shutdown and destroy server - grpc_server_shutdown_and_notify(server, cq, reinterpret_cast(1000)); - while (grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME), - nullptr) - .tag != reinterpret_cast(1000)) { - } - grpc_server_destroy(server); -} - -grpc_slice LargeSlice(void) { - grpc_slice slice = grpc_slice_malloc(10000000); // ~10MB - memset(GRPC_SLICE_START_PTR(slice), 'x', GRPC_SLICE_LENGTH(slice)); - return slice; -} - -void PerformCallWithLargePayload(grpc_channel* channel, grpc_server* server, - grpc_completion_queue* cq) { - grpc_slice request_payload_slice = LargeSlice(); - grpc_slice response_payload_slice = LargeSlice(); - grpc_call* c; - grpc_call* s; - grpc_byte_buffer* request_payload = - grpc_raw_byte_buffer_create(&request_payload_slice, 1); - grpc_byte_buffer* response_payload = - grpc_raw_byte_buffer_create(&response_payload_slice, 1); - cq_verifier* cqv = cq_verifier_create(cq); - grpc_op ops[6]; - grpc_op* op; - grpc_metadata_array initial_metadata_recv; - grpc_metadata_array trailing_metadata_recv; - grpc_metadata_array request_metadata_recv; - grpc_byte_buffer* request_payload_recv = nullptr; - grpc_byte_buffer* response_payload_recv = nullptr; - grpc_call_details call_details; - grpc_status_code status; - grpc_call_error error; - grpc_slice details; - int was_cancelled = 2; - - gpr_timespec deadline = grpc_timeout_seconds_to_deadline(30); - c = grpc_channel_create_call(channel, nullptr, GRPC_PROPAGATE_DEFAULTS, cq, - grpc_slice_from_static_string("/foo"), nullptr, - deadline, nullptr); - GPR_ASSERT(c); - - grpc_metadata_array_init(&initial_metadata_recv); - grpc_metadata_array_init(&trailing_metadata_recv); - grpc_metadata_array_init(&request_metadata_recv); - grpc_call_details_init(&call_details); - - memset(ops, 0, sizeof(ops)); - op = ops; - op->op = GRPC_OP_SEND_INITIAL_METADATA; - op->data.send_initial_metadata.count = 0; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_SEND_MESSAGE; - op->data.send_message.send_message = request_payload; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_RECV_INITIAL_METADATA; - op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_RECV_MESSAGE; - op->data.recv_message.recv_message = &response_payload_recv; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; - op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; - op->data.recv_status_on_client.status = &status; - op->data.recv_status_on_client.status_details = &details; - op->flags = 0; - op->reserved = nullptr; - op++; - error = grpc_call_start_batch(c, ops, static_cast(op - ops), tag(1), - nullptr); - GPR_ASSERT(GRPC_CALL_OK == error); - - error = grpc_server_request_call(server, &s, &call_details, - &request_metadata_recv, cq, cq, tag(101)); - GPR_ASSERT(GRPC_CALL_OK == error); - CQ_EXPECT_COMPLETION(cqv, tag(101), 1); - cq_verify(cqv); - - memset(ops, 0, sizeof(ops)); - op = ops; - op->op = GRPC_OP_SEND_INITIAL_METADATA; - op->data.send_initial_metadata.count = 0; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_RECV_MESSAGE; - op->data.recv_message.recv_message = &request_payload_recv; - op->flags = 0; - op->reserved = nullptr; - op++; - error = grpc_call_start_batch(s, ops, static_cast(op - ops), tag(102), - nullptr); - GPR_ASSERT(GRPC_CALL_OK == error); - - CQ_EXPECT_COMPLETION(cqv, tag(102), 1); - cq_verify(cqv); - - memset(ops, 0, sizeof(ops)); - op = ops; - op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; - op->data.recv_close_on_server.cancelled = &was_cancelled; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_SEND_MESSAGE; - op->data.send_message.send_message = response_payload; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; - op->data.send_status_from_server.trailing_metadata_count = 0; - op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED; - grpc_slice status_details = grpc_slice_from_static_string("xyz"); - op->data.send_status_from_server.status_details = &status_details; - op->flags = 0; - op->reserved = nullptr; - op++; - error = grpc_call_start_batch(s, ops, static_cast(op - ops), tag(103), - nullptr); - GPR_ASSERT(GRPC_CALL_OK == error); - - CQ_EXPECT_COMPLETION(cqv, tag(103), 1); - CQ_EXPECT_COMPLETION(cqv, tag(1), 1); - cq_verify(cqv); - - GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); - GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); - GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 0); - - grpc_slice_unref(details); - grpc_metadata_array_destroy(&initial_metadata_recv); - grpc_metadata_array_destroy(&trailing_metadata_recv); - grpc_metadata_array_destroy(&request_metadata_recv); - grpc_call_details_destroy(&call_details); - - grpc_call_unref(c); - grpc_call_unref(s); - - cq_verifier_destroy(cqv); - - grpc_byte_buffer_destroy(request_payload); - grpc_byte_buffer_destroy(response_payload); - grpc_byte_buffer_destroy(request_payload_recv); - grpc_byte_buffer_destroy(response_payload_recv); - grpc_slice_unref(request_payload_slice); - grpc_slice_unref(response_payload_slice); -} - -class FlowControlTest : public ::testing::Test { - protected: - void SetUp() override { - cq_ = grpc_completion_queue_create_for_next(nullptr); - // create the server - std::string server_address = - grpc_core::JoinHostPort("localhost", grpc_pick_unused_port_or_die()); - grpc_arg server_args[] = { - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_HTTP2_MAX_PING_STRIKES), 0), - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), -1), - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH), -1)}; - grpc_channel_args server_channel_args = {GPR_ARRAY_SIZE(server_args), - server_args}; - server_ = grpc_server_create(&server_channel_args, nullptr); - grpc_server_register_completion_queue(server_, cq_, nullptr); - grpc_server_credentials* server_creds = - grpc_insecure_server_credentials_create(); - GPR_ASSERT(grpc_server_add_http2_port(server_, server_address.c_str(), - server_creds)); - grpc_server_credentials_release(server_creds); - grpc_server_start(server_); - // create the channel (bdp pings are enabled by default) - grpc_arg client_args[] = { - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA), 0), - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS), 1), - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), -1), - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH), -1)}; - grpc_channel_args client_channel_args = {GPR_ARRAY_SIZE(client_args), - client_args}; - grpc_channel_credentials* creds = grpc_insecure_credentials_create(); - channel_ = grpc_channel_create(server_address.c_str(), creds, - &client_channel_args); - grpc_channel_credentials_release(creds); - VerifyChannelReady(channel_, cq_); - g_target_initial_window_size_mocker->Reset(); - } - - void TearDown() override { - // shutdown and destroy the client and server - grpc_channel_destroy(channel_); - ServerShutdownAndDestroy(server_, cq_); - grpc_completion_queue_shutdown(cq_); - while (grpc_completion_queue_next(cq_, gpr_inf_future(GPR_CLOCK_REALTIME), - nullptr) - .type != GRPC_QUEUE_SHUTDOWN) { - } - grpc_completion_queue_destroy(cq_); - } - - grpc_server* server_ = nullptr; - grpc_channel* channel_ = nullptr; - grpc_completion_queue* cq_ = nullptr; -}; - -TEST_F(FlowControlTest, - TestLargeWindowSizeUpdatesDoNotCauseIllegalFlowControlWindows) { - for (int i = 0; i < 10; ++i) { - PerformCallWithLargePayload(channel_, server_, cq_); - VerifyChannelConnected(channel_, cq_); - } -} - -TEST_F(FlowControlTest, TestWindowSizeUpdatesDoNotCauseStalledStreams) { - g_target_initial_window_size_mocker->AlternateTargetInitialWindowSizes(); - for (int i = 0; i < 100; ++i) { - PerformCallWithLargePayload(channel_, server_, cq_); - VerifyChannelConnected(channel_, cq_); - } -} - -} // namespace - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - // Make sure that we will have an active poller on all client-side fd's that - // are capable of sending and receiving even in the case that we don't have an - // active RPC operation on the fd. - GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 1); - grpc_core::chttp2::g_test_only_transport_flow_control_window_check = true; - g_target_initial_window_size_mocker = new TransportTargetWindowSizeMocker(); - grpc_core::chttp2::g_test_only_transport_target_window_estimates_mocker = - g_target_initial_window_size_mocker; - grpc::testing::TestEnvironment env(&argc, argv); - grpc_init(); - auto result = RUN_ALL_TESTS(); - grpc_shutdown(); - return result; -} diff --git a/test/core/transport/chttp2/flow_control_test.cc b/test/core/transport/chttp2/flow_control_test.cc index 7bada64a46e..a4c22b2c652 100644 --- a/test/core/transport/chttp2/flow_control_test.cc +++ b/test/core/transport/chttp2/flow_control_test.cc @@ -1,115 +1,380 @@ -// Copyright 2022 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include #include "src/core/ext/transport/chttp2/transport/flow_control.h" -#include +#include +#include + +#include +#include +#include + +#include -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/resource_quota/resource_quota.h" +#include +#include +#include +#include +#include +#include +#include +#include -namespace grpc_core { -namespace chttp2 { +#include "src/core/ext/filters/client_channel/backup_poller.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/host_port.h" +#include "src/core/lib/surface/channel.h" +#include "test/core/end2end/cq_verifier.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" namespace { -auto* g_memory_owner = new MemoryOwner( - ResourceQuota::Default()->memory_quota()->CreateMemoryOwner("test")); -} -TEST(FlowControl, NoOp) { - ExecCtx exec_ctx; - TransportFlowControl tfc("test", true, g_memory_owner); - StreamFlowControl sfc(&tfc); - // Check initial values are per http2 spec - EXPECT_EQ(tfc.sent_init_window(), 65535); - EXPECT_EQ(tfc.acked_init_window(), 65535); - EXPECT_EQ(tfc.remote_window(), 65535); - EXPECT_EQ(tfc.target_frame_size(), 16384); - EXPECT_EQ(sfc.remote_window_delta(), 0); - EXPECT_EQ(sfc.min_progress_size(), 0); - EXPECT_EQ(sfc.local_window_delta(), 0); - EXPECT_EQ(sfc.announced_window_delta(), 0); +class TransportTargetWindowSizeMocker + : public grpc_core::chttp2::TestOnlyTransportTargetWindowEstimatesMocker { + public: + static constexpr uint32_t kLargeInitialWindowSize = 1u << 31; + static constexpr uint32_t kSmallInitialWindowSize = 0; + + double ComputeNextTargetInitialWindowSizeFromPeriodicUpdate( + double /* current_target */) override { + // Protecting access to variable window_size_ shared between client and + // server. + grpc_core::MutexLock lock(&mu_); + if (alternating_initial_window_sizes_) { + window_size_ = (window_size_ == kLargeInitialWindowSize) + ? kSmallInitialWindowSize + : kLargeInitialWindowSize; + } + return window_size_; + } + + // Alternates the initial window size targets. Computes a low values if it was + // previously high, or a high value if it was previously low. + void AlternateTargetInitialWindowSizes() { + grpc_core::MutexLock lock(&mu_); + alternating_initial_window_sizes_ = true; + } + + void Reset() { + // Protecting access to variable window_size_ shared between client and + // server. + grpc_core::MutexLock lock(&mu_); + alternating_initial_window_sizes_ = false; + window_size_ = kLargeInitialWindowSize; + } + + private: + grpc_core::Mutex mu_; + bool alternating_initial_window_sizes_ ABSL_GUARDED_BY(mu_) = false; + double window_size_ ABSL_GUARDED_BY(mu_) = kLargeInitialWindowSize; +}; + +TransportTargetWindowSizeMocker* g_target_initial_window_size_mocker; + +void* tag(intptr_t t) { return reinterpret_cast(t); } + +void VerifyChannelReady(grpc_channel* channel, grpc_completion_queue* cq) { + grpc_connectivity_state state = + grpc_channel_check_connectivity_state(channel, 1 /* try_to_connect */); + while (state != GRPC_CHANNEL_READY) { + grpc_channel_watch_connectivity_state( + channel, state, grpc_timeout_seconds_to_deadline(5), cq, nullptr); + grpc_completion_queue_next(cq, grpc_timeout_seconds_to_deadline(5), + nullptr); + state = grpc_channel_check_connectivity_state(channel, 0); + } } -TEST(FlowControl, SendData) { - ExecCtx exec_ctx; - TransportFlowControl tfc("test", true, g_memory_owner); - StreamFlowControl sfc(&tfc); - sfc.SentData(1024); - EXPECT_EQ(sfc.remote_window_delta(), -1024); - EXPECT_EQ(tfc.remote_window(), 65535 - 1024); +void VerifyChannelConnected(grpc_channel* channel, grpc_completion_queue* cq) { + // Verify channel is connected. Use a ping to make sure that clients + // tries sending/receiving bytes if the channel is connected. + grpc_channel_ping(channel, cq, reinterpret_cast(2000), nullptr); + grpc_event ev = grpc_completion_queue_next( + cq, grpc_timeout_seconds_to_deadline(5), nullptr); + GPR_ASSERT(ev.type == GRPC_OP_COMPLETE); + GPR_ASSERT(ev.tag == reinterpret_cast(2000)); + GPR_ASSERT(ev.success == 1); + GPR_ASSERT(grpc_channel_check_connectivity_state(channel, 0) == + GRPC_CHANNEL_READY); } -TEST(FlowControl, InitialTransportUpdate) { - ExecCtx exec_ctx; - TransportFlowControl tfc("test", true, g_memory_owner); - EXPECT_EQ(tfc.MakeAction(), FlowControlAction()); +// Shuts down and destroys the server. +void ServerShutdownAndDestroy(grpc_server* server, grpc_completion_queue* cq) { + // Shutdown and destroy server + grpc_server_shutdown_and_notify(server, cq, reinterpret_cast(1000)); + while (grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME), + nullptr) + .tag != reinterpret_cast(1000)) { + } + grpc_server_destroy(server); } -TEST(FlowControl, InitialStreamUpdate) { - ExecCtx exec_ctx; - TransportFlowControl tfc("test", true, g_memory_owner); - StreamFlowControl sfc(&tfc); - EXPECT_EQ(sfc.MakeAction(), FlowControlAction()); +grpc_slice LargeSlice(void) { + grpc_slice slice = grpc_slice_malloc(10000000); // ~10MB + memset(GRPC_SLICE_START_PTR(slice), 'x', GRPC_SLICE_LENGTH(slice)); + return slice; } -TEST(FlowControl, RecvData) { - ExecCtx exec_ctx; - TransportFlowControl tfc("test", true, g_memory_owner); - StreamFlowControl sfc(&tfc); - EXPECT_EQ(absl::OkStatus(), sfc.RecvData(1024)); - EXPECT_EQ(tfc.announced_window(), 65535 - 1024); - EXPECT_EQ(sfc.local_window_delta(), -1024); +void PerformCallWithLargePayload(grpc_channel* channel, grpc_server* server, + grpc_completion_queue* cq) { + grpc_slice request_payload_slice = LargeSlice(); + grpc_slice response_payload_slice = LargeSlice(); + grpc_call* c; + grpc_call* s; + grpc_byte_buffer* request_payload = + grpc_raw_byte_buffer_create(&request_payload_slice, 1); + grpc_byte_buffer* response_payload = + grpc_raw_byte_buffer_create(&response_payload_slice, 1); + cq_verifier* cqv = cq_verifier_create(cq); + grpc_op ops[6]; + grpc_op* op; + grpc_metadata_array initial_metadata_recv; + grpc_metadata_array trailing_metadata_recv; + grpc_metadata_array request_metadata_recv; + grpc_byte_buffer* request_payload_recv = nullptr; + grpc_byte_buffer* response_payload_recv = nullptr; + grpc_call_details call_details; + grpc_status_code status; + grpc_call_error error; + grpc_slice details; + int was_cancelled = 2; + + gpr_timespec deadline = grpc_timeout_seconds_to_deadline(30); + c = grpc_channel_create_call(channel, nullptr, GRPC_PROPAGATE_DEFAULTS, cq, + grpc_slice_from_static_string("/foo"), nullptr, + deadline, nullptr); + GPR_ASSERT(c); + + grpc_metadata_array_init(&initial_metadata_recv); + grpc_metadata_array_init(&trailing_metadata_recv); + grpc_metadata_array_init(&request_metadata_recv); + grpc_call_details_init(&call_details); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_MESSAGE; + op->data.send_message.send_message = request_payload; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_MESSAGE; + op->data.recv_message.recv_message = &response_payload_recv; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; + op->data.recv_status_on_client.status = &status; + op->data.recv_status_on_client.status_details = &details; + op->flags = 0; + op->reserved = nullptr; + op++; + error = grpc_call_start_batch(c, ops, static_cast(op - ops), tag(1), + nullptr); + GPR_ASSERT(GRPC_CALL_OK == error); + + error = grpc_server_request_call(server, &s, &call_details, + &request_metadata_recv, cq, cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + CQ_EXPECT_COMPLETION(cqv, tag(101), 1); + cq_verify(cqv); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_MESSAGE; + op->data.recv_message.recv_message = &request_payload_recv; + op->flags = 0; + op->reserved = nullptr; + op++; + error = grpc_call_start_batch(s, ops, static_cast(op - ops), tag(102), + nullptr); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(102), 1); + cq_verify(cqv); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; + op->data.recv_close_on_server.cancelled = &was_cancelled; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_MESSAGE; + op->data.send_message.send_message = response_payload; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; + op->data.send_status_from_server.trailing_metadata_count = 0; + op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED; + grpc_slice status_details = grpc_slice_from_static_string("xyz"); + op->data.send_status_from_server.status_details = &status_details; + op->flags = 0; + op->reserved = nullptr; + op++; + error = grpc_call_start_batch(s, ops, static_cast(op - ops), tag(103), + nullptr); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(103), 1); + CQ_EXPECT_COMPLETION(cqv, tag(1), 1); + cq_verify(cqv); + + GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); + GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); + GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); + GPR_ASSERT(was_cancelled == 0); + + grpc_slice_unref(details); + grpc_metadata_array_destroy(&initial_metadata_recv); + grpc_metadata_array_destroy(&trailing_metadata_recv); + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + + grpc_call_unref(c); + grpc_call_unref(s); + + cq_verifier_destroy(cqv); + + grpc_byte_buffer_destroy(request_payload); + grpc_byte_buffer_destroy(response_payload); + grpc_byte_buffer_destroy(request_payload_recv); + grpc_byte_buffer_destroy(response_payload_recv); + grpc_slice_unref(request_payload_slice); + grpc_slice_unref(response_payload_slice); } -TEST(FlowControl, TrackMinProgressSize) { - ExecCtx exec_ctx; - TransportFlowControl tfc("test", true, g_memory_owner); - StreamFlowControl sfc(&tfc); - sfc.UpdateProgress(5); - EXPECT_EQ(sfc.min_progress_size(), 5); - sfc.UpdateProgress(10); - EXPECT_EQ(sfc.min_progress_size(), 10); - EXPECT_EQ(absl::OkStatus(), sfc.RecvData(5)); - EXPECT_EQ(sfc.min_progress_size(), 5); - EXPECT_EQ(absl::OkStatus(), sfc.RecvData(5)); - EXPECT_EQ(sfc.min_progress_size(), 0); - EXPECT_EQ(absl::OkStatus(), sfc.RecvData(5)); - EXPECT_EQ(sfc.min_progress_size(), 0); +class FlowControlTest : public ::testing::Test { + protected: + void SetUp() override { + cq_ = grpc_completion_queue_create_for_next(nullptr); + // create the server + std::string server_address = + grpc_core::JoinHostPort("localhost", grpc_pick_unused_port_or_die()); + grpc_arg server_args[] = { + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_HTTP2_MAX_PING_STRIKES), 0), + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), -1), + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH), -1)}; + grpc_channel_args server_channel_args = {GPR_ARRAY_SIZE(server_args), + server_args}; + server_ = grpc_server_create(&server_channel_args, nullptr); + grpc_server_register_completion_queue(server_, cq_, nullptr); + grpc_server_credentials* server_creds = + grpc_insecure_server_credentials_create(); + GPR_ASSERT(grpc_server_add_http2_port(server_, server_address.c_str(), + server_creds)); + grpc_server_credentials_release(server_creds); + grpc_server_start(server_); + // create the channel (bdp pings are enabled by default) + grpc_arg client_args[] = { + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA), 0), + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS), 1), + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), -1), + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH), -1)}; + grpc_channel_args client_channel_args = {GPR_ARRAY_SIZE(client_args), + client_args}; + grpc_channel_credentials* creds = grpc_insecure_credentials_create(); + channel_ = grpc_channel_create(server_address.c_str(), creds, + &client_channel_args); + grpc_channel_credentials_release(creds); + VerifyChannelReady(channel_, cq_); + g_target_initial_window_size_mocker->Reset(); + } + + void TearDown() override { + // shutdown and destroy the client and server + grpc_channel_destroy(channel_); + ServerShutdownAndDestroy(server_, cq_); + grpc_completion_queue_shutdown(cq_); + while (grpc_completion_queue_next(cq_, gpr_inf_future(GPR_CLOCK_REALTIME), + nullptr) + .type != GRPC_QUEUE_SHUTDOWN) { + } + grpc_completion_queue_destroy(cq_); + } + + grpc_server* server_ = nullptr; + grpc_channel* channel_ = nullptr; + grpc_completion_queue* cq_ = nullptr; +}; + +TEST_F(FlowControlTest, + TestLargeWindowSizeUpdatesDoNotCauseIllegalFlowControlWindows) { + for (int i = 0; i < 10; ++i) { + PerformCallWithLargePayload(channel_, server_, cq_); + VerifyChannelConnected(channel_, cq_); + } } -TEST(FlowControl, NoUpdateWithoutReader) { - ExecCtx exec_ctx; - TransportFlowControl tfc("test", true, g_memory_owner); - StreamFlowControl sfc(&tfc); - for (int i = 0; i < 65535; i++) { - EXPECT_EQ(sfc.RecvData(1), absl::OkStatus()); - EXPECT_EQ(sfc.MakeAction().send_stream_update(), - FlowControlAction::Urgency::NO_ACTION_NEEDED); +TEST_F(FlowControlTest, TestWindowSizeUpdatesDoNotCauseStalledStreams) { + g_target_initial_window_size_mocker->AlternateTargetInitialWindowSizes(); + for (int i = 0; i < 100; ++i) { + PerformCallWithLargePayload(channel_, server_, cq_); + VerifyChannelConnected(channel_, cq_); } - // Empty window needing 1 byte to progress should trigger an immediate read. - sfc.UpdateProgress(1); - EXPECT_EQ(sfc.min_progress_size(), 1); - EXPECT_EQ(sfc.MakeAction().send_stream_update(), - FlowControlAction::Urgency::UPDATE_IMMEDIATELY); - EXPECT_GT(sfc.MaybeSendUpdate(), 0); } -} // namespace chttp2 -} // namespace grpc_core +} // namespace int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + // Make sure that we will have an active poller on all client-side fd's that + // are capable of sending and receiving even in the case that we don't have an + // active RPC operation on the fd. + GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 1); + grpc_core::chttp2::g_test_only_transport_flow_control_window_check = true; + g_target_initial_window_size_mocker = new TransportTargetWindowSizeMocker(); + grpc_core::chttp2::g_test_only_transport_target_window_estimates_mocker = + g_target_initial_window_size_mocker; + grpc::testing::TestEnvironment env(&argc, argv); + grpc_init(); + auto result = RUN_ALL_TESTS(); + grpc_shutdown(); + return result; } diff --git a/test/core/transport/chttp2/remove_stream_from_stalled_lists_test.cc b/test/core/transport/chttp2/remove_stream_from_stalled_lists_test.cc index ea381c3a472..7731c1eb0d2 100644 --- a/test/core/transport/chttp2/remove_stream_from_stalled_lists_test.cc +++ b/test/core/transport/chttp2/remove_stream_from_stalled_lists_test.cc @@ -232,7 +232,10 @@ class TestServer { static void HandleOneRpc(grpc_call* call, grpc_completion_queue* call_cq) { // Send a large enough payload to get us stalled on outgoing flow control - std::string send_payload(4 * 1024 * 1024, 'a'); + std::string send_payload = ""; + for (int i = 0; i < 4 * 1e6; i++) { + send_payload += "a"; + } grpc_slice request_payload_slice = grpc_slice_from_copied_string(send_payload.c_str()); grpc_byte_buffer* request_payload = diff --git a/test/cpp/interop/client_helper.h b/test/cpp/interop/client_helper.h index 3b7ec7a540c..f361ca767a0 100644 --- a/test/cpp/interop/client_helper.h +++ b/test/cpp/interop/client_helper.h @@ -27,7 +27,7 @@ #include #include "src/core/lib/surface/call_test_only.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/byte_stream.h" namespace grpc { namespace testing { diff --git a/test/cpp/interop/http2_client.cc b/test/cpp/interop/http2_client.cc index dc1a5581433..d890d6a3ac1 100644 --- a/test/cpp/interop/http2_client.cc +++ b/test/cpp/interop/http2_client.cc @@ -29,6 +29,7 @@ #include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/useful.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/util/create_test_channel.h" diff --git a/test/cpp/interop/server_helper.cc b/test/cpp/interop/server_helper.cc index b6f9faaff12..5f1dcd5c16c 100644 --- a/test/cpp/interop/server_helper.cc +++ b/test/cpp/interop/server_helper.cc @@ -26,7 +26,7 @@ #include #include "src/core/lib/surface/call_test_only.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/byte_stream.h" #include "test/cpp/util/test_credentials_provider.h" ABSL_DECLARE_FLAG(bool, use_alts); diff --git a/test/cpp/interop/xds_interop_server.cc b/test/cpp/interop/xds_interop_server.cc index 1b3724bd47f..9d278c67cac 100644 --- a/test/cpp/interop/xds_interop_server.cc +++ b/test/cpp/interop/xds_interop_server.cc @@ -35,7 +35,7 @@ #include "src/core/lib/gpr/string.h" #include "src/core/lib/iomgr/gethostname.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/byte_stream.h" #include "src/proto/grpc/testing/empty.pb.h" #include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" diff --git a/test/cpp/microbenchmarks/bm_chttp2_transport.cc b/test/cpp/microbenchmarks/bm_chttp2_transport.cc index 921b289d41d..8af35295058 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_transport.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_transport.cc @@ -430,7 +430,7 @@ static void BM_TransportStreamSend(benchmark::State& state) { // is unreffed after each send_message op. grpc_slice send_slice = grpc_slice_malloc_large(state.range(0)); memset(GRPC_SLICE_START_PTR(send_slice), 0, GRPC_SLICE_LENGTH(send_slice)); - grpc_core::SliceBuffer send_stream; + grpc_core::ManualConstructor send_stream; auto arena = grpc_core::MakeScopedArena(1024, g_memory_allocator); grpc_metadata_batch b(arena.get()); RepresentativeClientInitialMetadata::Prepare(&b); @@ -444,15 +444,18 @@ static void BM_TransportStreamSend(benchmark::State& state) { gpr_event_set(bm_done, reinterpret_cast(1)); return; } - send_stream.Clear(); - send_stream.Append(grpc_core::Slice(grpc_slice_ref(send_slice))); + grpc_slice_buffer send_buffer; + grpc_slice_buffer_init(&send_buffer); + grpc_slice_buffer_add(&send_buffer, grpc_slice_ref(send_slice)); + send_stream.Init(&send_buffer, 0); + grpc_slice_buffer_destroy(&send_buffer); // force outgoing window to be yuge - s->chttp2_stream()->flow_control.TestOnlyForceHugeWindow(); - f.chttp2_transport()->flow_control.TestOnlyForceHugeWindow(); + s->chttp2_stream()->flow_control->TestOnlyForceHugeWindow(); + f.chttp2_transport()->flow_control->TestOnlyForceHugeWindow(); reset_op(); op.on_complete = c.get(); op.send_message = true; - op.payload->send_message.send_message = &send_stream; + op.payload->send_message.send_message.reset(send_stream.get()); s->Op(&op); }); @@ -554,7 +557,7 @@ static void BM_TransportStreamRecv(benchmark::State& state) { s->Init(state); grpc_transport_stream_op_batch_payload op_payload(nullptr); grpc_transport_stream_op_batch op; - absl::optional recv_stream; + grpc_core::OrphanablePtr recv_stream; grpc_slice incoming_data = CreateIncomingDataSlice(state.range(0), 16384); auto reset_op = [&]() { @@ -571,23 +574,57 @@ static void BM_TransportStreamRecv(benchmark::State& state) { uint32_t received; + std::unique_ptr drain_start; + std::unique_ptr drain; + std::unique_ptr drain_continue; + grpc_slice recv_slice; + std::unique_ptr c = MakeTestClosure([&](grpc_error_handle /*error*/) { if (!state.KeepRunning()) return; // force outgoing window to be yuge - s->chttp2_stream()->flow_control.TestOnlyForceHugeWindow(); - f.chttp2_transport()->flow_control.TestOnlyForceHugeWindow(); + s->chttp2_stream()->flow_control->TestOnlyForceHugeWindow(); + f.chttp2_transport()->flow_control->TestOnlyForceHugeWindow(); received = 0; reset_op(); op.on_complete = do_nothing.get(); op.recv_message = true; op.payload->recv_message.recv_message = &recv_stream; op.payload->recv_message.call_failed_before_recv_message = nullptr; - op.payload->recv_message.recv_message_ready = c.get(); + op.payload->recv_message.recv_message_ready = drain_start.get(); s->Op(&op); f.PushInput(grpc_slice_ref(incoming_data)); }); + drain_start = MakeTestClosure([&](grpc_error_handle /*error*/) { + if (recv_stream == nullptr) { + GPR_ASSERT(!state.KeepRunning()); + return; + } + grpc_core::Closure::Run(DEBUG_LOCATION, drain.get(), GRPC_ERROR_NONE); + }); + + drain = MakeTestClosure([&](grpc_error_handle /*error*/) { + do { + if (received == recv_stream->length()) { + recv_stream.reset(); + grpc_core::ExecCtx::Run(DEBUG_LOCATION, c.get(), GRPC_ERROR_NONE); + return; + } + } while (recv_stream->Next(recv_stream->length() - received, + drain_continue.get()) && + GRPC_ERROR_NONE == recv_stream->Pull(&recv_slice) && + (received += GRPC_SLICE_LENGTH(recv_slice), + grpc_slice_unref_internal(recv_slice), true)); + }); + + drain_continue = MakeTestClosure([&](grpc_error_handle /*error*/) { + GPR_ASSERT(GRPC_LOG_IF_ERROR("Pull", recv_stream->Pull(&recv_slice))); + received += GRPC_SLICE_LENGTH(recv_slice); + grpc_slice_unref_internal(recv_slice); + grpc_core::Closure::Run(DEBUG_LOCATION, drain.get(), GRPC_ERROR_NONE); + }); + reset_op(); auto b_recv = absl::make_unique(arena.get()); op.send_initial_metadata = true; diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 4f4f38203b5..a30c23a5a1c 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2403,6 +2403,8 @@ src/core/lib/surface/validate_metadata.h \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ src/core/lib/transport/bdp_estimator.h \ +src/core/lib/transport/byte_stream.cc \ +src/core/lib/transport/byte_stream.h \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/connectivity_state.h \ src/core/lib/transport/error_utils.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 18d1b78c778..6d42d7425ce 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -2199,6 +2199,8 @@ src/core/lib/surface/version.cc \ src/core/lib/transport/README.md \ src/core/lib/transport/bdp_estimator.cc \ src/core/lib/transport/bdp_estimator.h \ +src/core/lib/transport/byte_stream.cc \ +src/core/lib/transport/byte_stream.h \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/connectivity_state.h \ src/core/lib/transport/error_utils.cc \ diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index b89085a9b2e..011754afcec 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -1467,6 +1467,30 @@ ], "uses_polling": false }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "manual_constructor_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": false + }, { "args": [], "benchmark": false, @@ -3041,6 +3065,30 @@ ], "uses_polling": false }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "byte_stream_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": false + }, { "args": [], "benchmark": false, @@ -4189,30 +4237,6 @@ ], "uses_polling": true }, - { - "args": [], - "benchmark": false, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "gtest": true, - "language": "c++", - "name": "flow_control_end2end_test", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "uses_polling": true - }, { "args": [], "benchmark": false,