diff --git a/.gitignore b/.gitignore index 53bd54a1ad1..c2b54bed8d9 100644 --- a/.gitignore +++ b/.gitignore @@ -175,3 +175,6 @@ iwyu_files0.txt iwyu/ iwyu_build/ +# fuzzer logs +fuzz-*.log + diff --git a/BUILD b/BUILD index 0866d6d79ac..7a7861e5f5b 100644 --- a/BUILD +++ b/BUILD @@ -2469,6 +2469,23 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "bdp_estimator", + srcs = [ + "src/core/lib/transport/bdp_estimator.cc", + ], + hdrs = ["src/core/lib/transport/bdp_estimator.h"], + tags = ["grpc-autodeps"], + deps = [ + "exec_ctx", + "gpr_base", + "gpr_codegen", + "gpr_platform", + "grpc_trace", + "time", + ], +) + grpc_cc_library( name = "percent_encoding", srcs = [ @@ -2591,8 +2608,6 @@ grpc_cc_library( "src/core/lib/surface/server.cc", "src/core/lib/surface/validate_metadata.cc", "src/core/lib/surface/version.cc", - "src/core/lib/transport/bdp_estimator.cc", - "src/core/lib/transport/byte_stream.cc", "src/core/lib/transport/connectivity_state.cc", "src/core/lib/transport/error_utils.cc", "src/core/lib/transport/metadata_batch.cc", @@ -2689,8 +2704,6 @@ grpc_cc_library( "src/core/lib/surface/lame_client.h", "src/core/lib/surface/server.h", "src/core/lib/surface/validate_metadata.h", - "src/core/lib/transport/bdp_estimator.h", - "src/core/lib/transport/byte_stream.h", "src/core/lib/transport/connectivity_state.h", "src/core/lib/transport/metadata_batch.h", "src/core/lib/transport/parsed_metadata.h", @@ -3150,6 +3163,7 @@ grpc_cc_library( "absl/types:variant", "absl/status", "absl/status:statusor", + "absl/utility", "upb_lib", ], language = "c++", @@ -3161,6 +3175,7 @@ grpc_cc_library( "channel_stack_type", "chunked_vector", "config", + "construct_destruct", "debug_location", "default_event_engine_factory_hdrs", "dual_ref_counted", @@ -3193,6 +3208,7 @@ grpc_cc_library( "server_address", "service_config_parser", "slice", + "slice_buffer", "slice_refcount", "sockaddr_utils", "time", @@ -3428,8 +3444,8 @@ grpc_cc_library( "grpc_public_hdrs", "grpc_service_config", "json", - "orphanable", "service_config_parser", + "slice_buffer", ], ) @@ -3557,6 +3573,7 @@ grpc_cc_library( "promise", "seq", "slice", + "slice_buffer", "transport_fwd", ], ) @@ -6028,6 +6045,36 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "chttp2_flow_control", + srcs = [ + "src/core/ext/transport/chttp2/transport/flow_control.cc", + ], + hdrs = [ + "src/core/ext/transport/chttp2/transport/flow_control.h", + ], + external_deps = [ + "absl/functional:function_ref", + "absl/status", + "absl/strings", + "absl/strings:str_format", + "absl/types:optional", + "absl/utility", + ], + tags = ["grpc-autodeps"], + deps = [ + "bdp_estimator", + "exec_ctx", + "gpr_base", + "gpr_platform", + "grpc_trace", + "memory_quota", + "pid_controller", + "time", + "useful", + ], +) + grpc_cc_library( name = "grpc_transport_chttp2", srcs = [ @@ -6035,7 +6082,6 @@ grpc_cc_library( "src/core/ext/transport/chttp2/transport/bin_encoder.cc", "src/core/ext/transport/chttp2/transport/chttp2_transport.cc", "src/core/ext/transport/chttp2/transport/context_list.cc", - "src/core/ext/transport/chttp2/transport/flow_control.cc", "src/core/ext/transport/chttp2/transport/frame_data.cc", "src/core/ext/transport/chttp2/transport/frame_goaway.cc", "src/core/ext/transport/chttp2/transport/frame_ping.cc", @@ -6058,7 +6104,6 @@ grpc_cc_library( "src/core/ext/transport/chttp2/transport/bin_encoder.h", "src/core/ext/transport/chttp2/transport/chttp2_transport.h", "src/core/ext/transport/chttp2/transport/context_list.h", - "src/core/ext/transport/chttp2/transport/flow_control.h", "src/core/ext/transport/chttp2/transport/frame.h", "src/core/ext/transport/chttp2/transport/frame_data.h", "src/core/ext/transport/chttp2/transport/frame_goaway.h", @@ -6091,7 +6136,9 @@ grpc_cc_library( visibility = ["@grpc:grpclb"], deps = [ "arena", + "bdp_estimator", "bitset", + "chttp2_flow_control", "chunked_vector", "debug_location", "gpr_base", @@ -6109,11 +6156,13 @@ grpc_cc_library( "memory_quota", "orphanable", "pid_controller", + "poll", "ref_counted", "ref_counted_ptr", "resource_quota", "resource_quota_trace", "slice", + "slice_buffer", "slice_refcount", "status_helper", "time", @@ -6241,8 +6290,10 @@ grpc_cc_library( "absl/status:statusor", "absl/strings", "absl/types:optional", + "absl/utility", ], language = "c++", + tags = ["grpc-autodeps"], deps = [ "arena", "channel_args_preconditioning", @@ -6252,11 +6303,12 @@ grpc_cc_library( "gpr_base", "grpc_base", "grpc_codegen", + "grpc_public_hdrs", "grpc_trace", "iomgr_fwd", - "orphanable", "ref_counted_ptr", "slice", + "slice_buffer", "time", "transport_fwd", "useful", @@ -6850,6 +6902,7 @@ grpc_cc_library( "grpc++_base", "grpc_base", "slice", + "slice_buffer", "slice_refcount", ], ) diff --git a/CMakeLists.txt b/CMakeLists.txt index ede84646b09..4c47485684a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -859,7 +859,6 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_c jwt_verifier_test) add_dependencies(buildtests_c lame_client_test) add_dependencies(buildtests_c load_file_test) - add_dependencies(buildtests_c manual_constructor_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c memory_quota_stress_test) endif() @@ -962,7 +961,6 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_cxx binder_transport_test) add_dependencies(buildtests_cxx bitset_test) add_dependencies(buildtests_cxx byte_buffer_test) - add_dependencies(buildtests_cxx byte_stream_test) add_dependencies(buildtests_cxx call_finalization_test) add_dependencies(buildtests_cxx call_push_pull_test) add_dependencies(buildtests_cxx cancel_ares_query_test) @@ -2267,7 +2265,6 @@ add_library(grpc src/core/lib/surface/validate_metadata.cc src/core/lib/surface/version.cc src/core/lib/transport/bdp_estimator.cc - src/core/lib/transport/byte_stream.cc src/core/lib/transport/connectivity_state.cc src/core/lib/transport/error_utils.cc src/core/lib/transport/handshaker.cc @@ -2355,6 +2352,7 @@ target_link_libraries(grpc absl::flat_hash_set absl::inlined_vector absl::bind_front + absl::function_ref absl::hash absl::type_traits absl::statusor @@ -2859,7 +2857,6 @@ add_library(grpc_unsecure src/core/lib/surface/validate_metadata.cc src/core/lib/surface/version.cc src/core/lib/transport/bdp_estimator.cc - src/core/lib/transport/byte_stream.cc src/core/lib/transport/connectivity_state.cc src/core/lib/transport/error_utils.cc src/core/lib/transport/handshaker.cc @@ -2923,6 +2920,7 @@ target_link_libraries(grpc_unsecure absl::flat_hash_set absl::inlined_vector absl::bind_front + absl::function_ref absl::hash absl::type_traits absl::statusor @@ -6034,33 +6032,6 @@ target_link_libraries(load_file_test ) -endif() -if(gRPC_BUILD_TESTS) - -add_executable(manual_constructor_test - test/core/gprpp/manual_constructor_test.cc -) - -target_include_directories(manual_constructor_test - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} - ${_gRPC_RE2_INCLUDE_DIR} - ${_gRPC_SSL_INCLUDE_DIR} - ${_gRPC_UPB_GENERATED_DIR} - ${_gRPC_UPB_GRPC_GENERATED_DIR} - ${_gRPC_UPB_INCLUDE_DIR} - ${_gRPC_XXHASH_INCLUDE_DIR} - ${_gRPC_ZLIB_INCLUDE_DIR} -) - -target_link_libraries(manual_constructor_test - ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_test_util -) - - endif() if(gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) @@ -8383,41 +8354,6 @@ target_link_libraries(byte_buffer_test ) -endif() -if(gRPC_BUILD_TESTS) - -add_executable(byte_stream_test - test/core/transport/byte_stream_test.cc - third_party/googletest/googletest/src/gtest-all.cc - third_party/googletest/googlemock/src/gmock-all.cc -) - -target_include_directories(byte_stream_test - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} - ${_gRPC_RE2_INCLUDE_DIR} - ${_gRPC_SSL_INCLUDE_DIR} - ${_gRPC_UPB_GENERATED_DIR} - ${_gRPC_UPB_GRPC_GENERATED_DIR} - ${_gRPC_UPB_INCLUDE_DIR} - ${_gRPC_XXHASH_INCLUDE_DIR} - ${_gRPC_ZLIB_INCLUDE_DIR} - third_party/googletest/googletest/include - third_party/googletest/googletest - third_party/googletest/googlemock/include - third_party/googletest/googlemock - ${_gRPC_PROTO_GENS_DIR} -) - -target_link_libraries(byte_stream_test - ${_gRPC_PROTOBUF_LIBRARIES} - ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_test_util -) - - endif() if(gRPC_BUILD_TESTS) @@ -10766,7 +10702,29 @@ endif() if(gRPC_BUILD_TESTS) add_executable(flow_control_test - test/core/end2end/cq_verifier.cc + src/core/ext/transport/chttp2/transport/flow_control.cc + src/core/ext/upb-generated/google/protobuf/any.upb.c + src/core/ext/upb-generated/google/rpc/status.upb.c + src/core/lib/debug/trace.cc + src/core/lib/event_engine/memory_allocator.cc + src/core/lib/gprpp/status_helper.cc + src/core/lib/gprpp/time.cc + src/core/lib/iomgr/combiner.cc + src/core/lib/iomgr/error.cc + src/core/lib/iomgr/exec_ctx.cc + src/core/lib/iomgr/executor.cc + src/core/lib/iomgr/iomgr_internal.cc + src/core/lib/promise/activity.cc + src/core/lib/resource_quota/memory_quota.cc + src/core/lib/resource_quota/resource_quota.cc + src/core/lib/resource_quota/thread_quota.cc + src/core/lib/resource_quota/trace.cc + src/core/lib/slice/percent_encoding.cc + src/core/lib/slice/slice.cc + src/core/lib/slice/slice_refcount.cc + src/core/lib/slice/slice_string_helpers.cc + src/core/lib/transport/bdp_estimator.cc + src/core/lib/transport/pid_controller.cc test/core/transport/chttp2/flow_control_test.cc third_party/googletest/googletest/src/gtest-all.cc third_party/googletest/googlemock/src/gmock-all.cc @@ -10794,7 +10752,13 @@ target_include_directories(flow_control_test target_link_libraries(flow_control_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_test_util + absl::function_ref + absl::type_traits + absl::statusor + absl::variant + absl::utility + gpr + upb ) @@ -20462,7 +20426,7 @@ generate_pkgconfig( "gRPC" "high performance general RPC framework" "${gRPC_CORE_VERSION}" - "gpr openssl absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" + "gpr openssl absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" "-lgrpc -laddress_sorting -lre2 -lupb -lcares -lz" "" "grpc.pc") @@ -20472,7 +20436,7 @@ generate_pkgconfig( "gRPC unsecure" "high performance general RPC framework without SSL" "${gRPC_CORE_VERSION}" - "gpr absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" + "gpr absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" "-lgrpc_unsecure" "" "grpc_unsecure.pc") @@ -20482,7 +20446,7 @@ generate_pkgconfig( "gRPC++" "C++ wrapper for gRPC" "${gRPC_CPP_VERSION}" - "grpc absl_base absl_bind_front absl_cleanup absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" + "grpc absl_base absl_bind_front absl_cleanup absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" "-lgrpc++" "" "grpc++.pc") @@ -20492,7 +20456,7 @@ generate_pkgconfig( "gRPC++ unsecure" "C++ wrapper for gRPC without SSL" "${gRPC_CPP_VERSION}" - "grpc_unsecure absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" + "grpc_unsecure absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" "-lgrpc++_unsecure" "" "grpc++_unsecure.pc") diff --git a/Makefile b/Makefile index bcd125b3b81..bb0899d6840 100644 --- a/Makefile +++ b/Makefile @@ -1640,7 +1640,6 @@ LIBGRPC_SRC = \ src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ - src/core/lib/transport/byte_stream.cc \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/error_utils.cc \ src/core/lib/transport/handshaker.cc \ @@ -2073,7 +2072,6 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ - src/core/lib/transport/byte_stream.cc \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/error_utils.cc \ src/core/lib/transport/handshaker.cc \ diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index c1adeed03b0..5d702fb59a6 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -952,7 +952,6 @@ libs: - src/core/lib/surface/server.h - src/core/lib/surface/validate_metadata.h - src/core/lib/transport/bdp_estimator.h - - src/core/lib/transport/byte_stream.h - src/core/lib/transport/connectivity_state.h - src/core/lib/transport/error_utils.h - src/core/lib/transport/handshaker.h @@ -1625,7 +1624,6 @@ libs: - src/core/lib/surface/validate_metadata.cc - src/core/lib/surface/version.cc - src/core/lib/transport/bdp_estimator.cc - - src/core/lib/transport/byte_stream.cc - src/core/lib/transport/connectivity_state.cc - src/core/lib/transport/error_utils.cc - src/core/lib/transport/handshaker.cc @@ -1675,6 +1673,7 @@ libs: - absl/container:flat_hash_set - absl/container:inlined_vector - absl/functional:bind_front + - absl/functional:function_ref - absl/hash:hash - absl/meta:type_traits - absl/status:statusor @@ -2139,7 +2138,6 @@ libs: - src/core/lib/surface/server.h - src/core/lib/surface/validate_metadata.h - src/core/lib/transport/bdp_estimator.h - - src/core/lib/transport/byte_stream.h - src/core/lib/transport/connectivity_state.h - src/core/lib/transport/error_utils.h - src/core/lib/transport/handshaker.h @@ -2455,7 +2453,6 @@ libs: - src/core/lib/surface/validate_metadata.cc - src/core/lib/surface/version.cc - src/core/lib/transport/bdp_estimator.cc - - src/core/lib/transport/byte_stream.cc - src/core/lib/transport/connectivity_state.cc - src/core/lib/transport/error_utils.cc - src/core/lib/transport/handshaker.cc @@ -2481,6 +2478,7 @@ libs: - absl/container:flat_hash_set - absl/container:inlined_vector - absl/functional:bind_front + - absl/functional:function_ref - absl/hash:hash - absl/meta:type_traits - absl/status:statusor @@ -3832,15 +3830,6 @@ targets: deps: - grpc_test_util uses_polling: false -- name: manual_constructor_test - build: test - language: c - headers: [] - src: - - test/core/gprpp/manual_constructor_test.cc - deps: - - grpc_test_util - uses_polling: false - name: memory_quota_stress_test build: test language: c @@ -4915,16 +4904,6 @@ targets: deps: - grpc++_test_util uses_polling: false -- name: byte_stream_test - gtest: true - build: test - language: c++ - headers: [] - src: - - test/core/transport/byte_stream_test.cc - deps: - - grpc_test_util - uses_polling: false - name: call_finalization_test gtest: true build: test @@ -5912,12 +5891,84 @@ targets: build: test language: c++ headers: - - test/core/end2end/cq_verifier.h + - src/core/ext/transport/chttp2/transport/flow_control.h + - src/core/ext/upb-generated/google/protobuf/any.upb.h + - src/core/ext/upb-generated/google/rpc/status.upb.h + - src/core/lib/debug/trace.h + - src/core/lib/gprpp/atomic_utils.h + - src/core/lib/gprpp/bitset.h + - src/core/lib/gprpp/cpp_impl_of.h + - src/core/lib/gprpp/orphanable.h + - src/core/lib/gprpp/ref_counted.h + - src/core/lib/gprpp/ref_counted_ptr.h + - src/core/lib/gprpp/status_helper.h + - src/core/lib/gprpp/time.h + - src/core/lib/iomgr/closure.h + - src/core/lib/iomgr/combiner.h + - src/core/lib/iomgr/error.h + - src/core/lib/iomgr/error_internal.h + - src/core/lib/iomgr/exec_ctx.h + - src/core/lib/iomgr/executor.h + - src/core/lib/iomgr/iomgr_internal.h + - src/core/lib/promise/activity.h + - src/core/lib/promise/context.h + - src/core/lib/promise/detail/basic_seq.h + - src/core/lib/promise/detail/promise_factory.h + - src/core/lib/promise/detail/promise_like.h + - src/core/lib/promise/detail/status.h + - src/core/lib/promise/detail/switch.h + - src/core/lib/promise/exec_ctx_wakeup_scheduler.h + - src/core/lib/promise/loop.h + - src/core/lib/promise/map.h + - src/core/lib/promise/poll.h + - src/core/lib/promise/race.h + - src/core/lib/promise/seq.h + - src/core/lib/resource_quota/memory_quota.h + - src/core/lib/resource_quota/resource_quota.h + - src/core/lib/resource_quota/thread_quota.h + - src/core/lib/resource_quota/trace.h + - src/core/lib/slice/percent_encoding.h + - src/core/lib/slice/slice.h + - src/core/lib/slice/slice_internal.h + - src/core/lib/slice/slice_refcount.h + - src/core/lib/slice/slice_refcount_base.h + - src/core/lib/slice/slice_string_helpers.h + - src/core/lib/transport/bdp_estimator.h + - src/core/lib/transport/pid_controller.h src: - - test/core/end2end/cq_verifier.cc + - src/core/ext/transport/chttp2/transport/flow_control.cc + - src/core/ext/upb-generated/google/protobuf/any.upb.c + - src/core/ext/upb-generated/google/rpc/status.upb.c + - src/core/lib/debug/trace.cc + - src/core/lib/event_engine/memory_allocator.cc + - src/core/lib/gprpp/status_helper.cc + - src/core/lib/gprpp/time.cc + - src/core/lib/iomgr/combiner.cc + - src/core/lib/iomgr/error.cc + - src/core/lib/iomgr/exec_ctx.cc + - src/core/lib/iomgr/executor.cc + - src/core/lib/iomgr/iomgr_internal.cc + - src/core/lib/promise/activity.cc + - src/core/lib/resource_quota/memory_quota.cc + - src/core/lib/resource_quota/resource_quota.cc + - src/core/lib/resource_quota/thread_quota.cc + - src/core/lib/resource_quota/trace.cc + - src/core/lib/slice/percent_encoding.cc + - src/core/lib/slice/slice.cc + - src/core/lib/slice/slice_refcount.cc + - src/core/lib/slice/slice_string_helpers.cc + - src/core/lib/transport/bdp_estimator.cc + - src/core/lib/transport/pid_controller.cc - test/core/transport/chttp2/flow_control_test.cc deps: - - grpc_test_util + - absl/functional:function_ref + - absl/meta:type_traits + - absl/status:statusor + - absl/types:variant + - absl/utility:utility + - gpr + - upb + uses_polling: false - name: for_each_test gtest: true build: test diff --git a/config.m4 b/config.m4 index 9ed9418f2e0..5bd3fd9f50a 100644 --- a/config.m4 +++ b/config.m4 @@ -708,7 +708,6 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ - src/core/lib/transport/byte_stream.cc \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/error_utils.cc \ src/core/lib/transport/handshaker.cc \ diff --git a/config.w32 b/config.w32 index 7a6341f07b2..4918f2edf8e 100644 --- a/config.w32 +++ b/config.w32 @@ -674,7 +674,6 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\surface\\validate_metadata.cc " + "src\\core\\lib\\surface\\version.cc " + "src\\core\\lib\\transport\\bdp_estimator.cc " + - "src\\core\\lib\\transport\\byte_stream.cc " + "src\\core\\lib\\transport\\connectivity_state.cc " + "src\\core\\lib\\transport\\error_utils.cc " + "src\\core\\lib\\transport\\handshaker.cc " + diff --git a/doc/environment_variables.md b/doc/environment_variables.md index 4f6201acf2b..59442f08d36 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -167,11 +167,6 @@ some configuration as environment variables that can be set. channels (mostly due to idleness), so that the next RPC on this channel won't fail. Set to 0 to turn off the backup polls. -* GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL - if set, flow control will be effectively disabled. Max out all values and - assume the remote peer does the same. Thus we can ignore any flow control - bookkeeping, error checking, and decision making - * grpc_cfstream set to 1 to turn on CFStream experiment. With this experiment gRPC uses CFStream API to make TCP connections. The option is only available on iOS platform and when macro GRPC_CFSTREAM is defined. diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 71a5b64da85..f732989eda8 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -203,6 +203,7 @@ Pod::Spec.new do |s| ss.dependency 'abseil/container/flat_hash_set', abseil_version ss.dependency 'abseil/container/inlined_vector', abseil_version ss.dependency 'abseil/functional/bind_front', abseil_version + ss.dependency 'abseil/functional/function_ref', abseil_version ss.dependency 'abseil/hash/hash', abseil_version ss.dependency 'abseil/memory/memory', abseil_version ss.dependency 'abseil/meta/type_traits', abseil_version @@ -910,7 +911,6 @@ Pod::Spec.new do |s| 'src/core/lib/surface/server.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/transport/bdp_estimator.h', - 'src/core/lib/transport/byte_stream.h', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/handshaker.h', @@ -1737,7 +1737,6 @@ Pod::Spec.new do |s| 'src/core/lib/surface/server.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/transport/bdp_estimator.h', - 'src/core/lib/transport/byte_stream.h', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/handshaker.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 84908871feb..78549d3c363 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -178,6 +178,7 @@ Pod::Spec.new do |s| ss.dependency 'abseil/container/flat_hash_set', abseil_version ss.dependency 'abseil/container/inlined_vector', abseil_version ss.dependency 'abseil/functional/bind_front', abseil_version + ss.dependency 'abseil/functional/function_ref', abseil_version ss.dependency 'abseil/hash/hash', abseil_version ss.dependency 'abseil/memory/memory', abseil_version ss.dependency 'abseil/meta/type_traits', abseil_version @@ -1516,8 +1517,6 @@ Pod::Spec.new do |s| 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', 'src/core/lib/transport/bdp_estimator.h', - 'src/core/lib/transport/byte_stream.cc', - 'src/core/lib/transport/byte_stream.h', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.cc', @@ -2342,7 +2341,6 @@ Pod::Spec.new do |s| 'src/core/lib/surface/server.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/transport/bdp_estimator.h', - 'src/core/lib/transport/byte_stream.h', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/handshaker.h', diff --git a/grpc.gemspec b/grpc.gemspec index 5f79dd3d68a..59754bffcdd 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1431,8 +1431,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/surface/version.cc ) s.files += %w( src/core/lib/transport/bdp_estimator.cc ) s.files += %w( src/core/lib/transport/bdp_estimator.h ) - s.files += %w( src/core/lib/transport/byte_stream.cc ) - s.files += %w( src/core/lib/transport/byte_stream.h ) s.files += %w( src/core/lib/transport/connectivity_state.cc ) s.files += %w( src/core/lib/transport/connectivity_state.h ) s.files += %w( src/core/lib/transport/error_utils.cc ) diff --git a/grpc.gyp b/grpc.gyp index 2179683415f..abff82779ea 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -360,6 +360,7 @@ 'absl/container:flat_hash_set', 'absl/container:inlined_vector', 'absl/functional:bind_front', + 'absl/functional:function_ref', 'absl/hash:hash', 'absl/meta:type_traits', 'absl/status:statusor', @@ -996,7 +997,6 @@ 'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', - 'src/core/lib/transport/byte_stream.cc', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/handshaker.cc', @@ -1118,6 +1118,7 @@ 'absl/container:flat_hash_set', 'absl/container:inlined_vector', 'absl/functional:bind_front', + 'absl/functional:function_ref', 'absl/hash:hash', 'absl/meta:type_traits', 'absl/status:statusor', @@ -1420,7 +1421,6 @@ 'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', - 'src/core/lib/transport/byte_stream.cc', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/handshaker.cc', diff --git a/include/grpc/event_engine/slice_buffer.h b/include/grpc/event_engine/slice_buffer.h index 609b6ea0841..3861717e7b6 100644 --- a/include/grpc/event_engine/slice_buffer.h +++ b/include/grpc/event_engine/slice_buffer.h @@ -54,13 +54,19 @@ class SliceBuffer { SliceBuffer(const SliceBuffer& other) = delete; SliceBuffer(SliceBuffer&& other) noexcept : slice_buffer_(other.slice_buffer_) { - grpc_slice_buffer_reset_and_unref(&slice_buffer_); + grpc_slice_buffer_init(&slice_buffer_); grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_); } /// Upon destruction, the underlying raw slice buffer is cleaned out and all /// slices are unreffed. ~SliceBuffer() { grpc_slice_buffer_destroy(&slice_buffer_); } + SliceBuffer& operator=(const SliceBuffer&) = delete; + SliceBuffer& operator=(SliceBuffer&& other) noexcept { + grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_); + return *this; + } + /// Appends a new slice into the SliceBuffer and makes an attempt to merge /// this slice with the last slice in the SliceBuffer. void Append(Slice slice); @@ -99,7 +105,7 @@ class SliceBuffer { size_t Length() { return slice_buffer_.length; } /// Return a pointer to the back raw grpc_slice_buffer - grpc_slice_buffer* RawSliceBuffer() { return &slice_buffer_; } + grpc_slice_buffer* c_slice_buffer() { return &slice_buffer_; } private: /// The backing raw slice buffer. diff --git a/package.xml b/package.xml index c5d32110ba0..59652fde28e 100644 --- a/package.xml +++ b/package.xml @@ -1413,8 +1413,6 @@ - - diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index db4db4428ab..be14d2667eb 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -2953,7 +2953,7 @@ void ClientChannel::LoadBalancedCall::RecvMessageReady( gpr_log(GPR_INFO, "chand=%p lb_call=%p: got recv_message_ready: error=%s", self->chand_, self, grpc_error_std_string(error).c_str()); } - if (*self->recv_message_ != nullptr) { + if (self->recv_message_->has_value()) { self->call_attempt_tracer_->RecordReceivedMessage(**self->recv_message_); } Closure::Run(DEBUG_LOCATION, self->original_recv_message_ready_, diff --git a/src/core/ext/filters/client_channel/client_channel.h b/src/core/ext/filters/client_channel/client_channel.h index 16efd351547..1b8da1eb9fc 100644 --- a/src/core/ext/filters/client_channel/client_channel.h +++ b/src/core/ext/filters/client_channel/client_channel.h @@ -68,8 +68,8 @@ #include "src/core/lib/service_config/service_config_call_data.h" #include "src/core/lib/service_config/service_config_parser.h" #include "src/core/lib/slice/slice.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/channel.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -534,7 +534,7 @@ class ClientChannel::LoadBalancedCall grpc_closure* original_recv_initial_metadata_ready_ = nullptr; // For intercepting recv_message_ready. - OrphanablePtr* recv_message_ = nullptr; + absl::optional* recv_message_ = nullptr; grpc_closure recv_message_ready_; grpc_closure* original_recv_message_ready_ = nullptr; diff --git a/src/core/ext/filters/client_channel/retry_filter.cc b/src/core/ext/filters/client_channel/retry_filter.cc index b4d776e2b3b..0eafe502090 100644 --- a/src/core/ext/filters/client_channel/retry_filter.cc +++ b/src/core/ext/filters/client_channel/retry_filter.cc @@ -33,6 +33,7 @@ #include "absl/strings/string_view.h" #include "absl/strings/strip.h" #include "absl/types/optional.h" +#include "absl/utility/utility.h" #include #include @@ -51,8 +52,8 @@ #include "src/core/lib/channel/status_util.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gpr/useful.h" +#include "src/core/lib/gprpp/construct_destruct.h" #include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" @@ -66,8 +67,8 @@ #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/service_config/service_config.h" #include "src/core/lib/service_config/service_config_call_data.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_refcount.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -456,9 +457,6 @@ class RetryFilter::CallData { grpc_transport_stream_op_batch_payload batch_payload_; // For send_initial_metadata. grpc_metadata_batch send_initial_metadata_{calld_->arena_}; - // For send_message. - // TODO(roth): Restructure this to eliminate use of ManualConstructor. - ManualConstructor send_message_; // For send_trailing_metadata. grpc_metadata_batch send_trailing_metadata_{calld_->arena_}; // For intercepting recv_initial_metadata. @@ -467,7 +465,8 @@ class RetryFilter::CallData { bool trailing_metadata_available_ = false; // For intercepting recv_message. grpc_closure recv_message_ready_; - OrphanablePtr recv_message_; + absl::optional recv_message_; + uint32_t recv_message_flags_; // For intercepting recv_trailing_metadata. grpc_metadata_batch recv_trailing_metadata_{calld_->arena_}; grpc_transport_stream_stats collect_stats_; @@ -628,11 +627,11 @@ class RetryFilter::CallData { // Note: We inline the cache for the first 3 send_message ops and use // dynamic allocation after that. This number was essentially picked // at random; it could be changed in the future to tune performance. - // TODO(roth): As part of implementing hedging, we may need some - // synchronization here, since ByteStreamCache does not provide any - // synchronization, so it's not safe to have multiple - // CachingByteStreams read from the same ByteStreamCache concurrently. - absl::InlinedVector send_messages_; + struct CachedSendMessage { + SliceBuffer* slices; + uint32_t flags; + }; + absl::InlinedVector send_messages_; // send_trailing_metadata bool seen_send_trailing_metadata_ = false; grpc_metadata_batch send_trailing_metadata_{arena_}; @@ -1511,6 +1510,8 @@ void RetryFilter::CallData::CallAttempt::BatchData:: // Return payload. *pending->batch->payload->recv_message.recv_message = std::move(call_attempt_->recv_message_); + *pending->batch->payload->recv_message.flags = + call_attempt_->recv_message_flags_; // Update bookkeeping. // Note: Need to do this before invoking the callback, since invoking // the callback will result in yielding the call combiner. @@ -1555,7 +1556,7 @@ void RetryFilter::CallData::CallAttempt::BatchData::RecvMessageReady( // the recv_trailing_metadata_ready callback, then defer propagating this // callback back to the surface. We can evaluate whether to retry when // recv_trailing_metadata comes back. - if (GPR_UNLIKELY((call_attempt->recv_message_ == nullptr || + if (GPR_UNLIKELY((!call_attempt->recv_message_.has_value() || !GRPC_ERROR_IS_NONE(error)) && !call_attempt->completed_recv_trailing_metadata_)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_retry_trace)) { @@ -2029,13 +2030,12 @@ void RetryFilter::CallData::CallAttempt::BatchData:: calld->chand_, calld, call_attempt_.get(), call_attempt_->started_send_message_count_); } - ByteStreamCache* cache = + CachedSendMessage cache = calld->send_messages_[call_attempt_->started_send_message_count_]; ++call_attempt_->started_send_message_count_; - call_attempt_->send_message_.Init(cache); batch_.send_message = true; - batch_.payload->send_message.send_message.reset( - call_attempt_->send_message_.get()); + batch_.payload->send_message.send_message = cache.slices; + batch_.payload->send_message.flags = cache.flags; } void RetryFilter::CallData::CallAttempt::BatchData:: @@ -2072,6 +2072,7 @@ void RetryFilter::CallData::CallAttempt::BatchData:: ++call_attempt_->started_recv_message_count_; batch_.recv_message = true; batch_.payload->recv_message.recv_message = &call_attempt_->recv_message_; + batch_.payload->recv_message.flags = &call_attempt_->recv_message_flags_; batch_.payload->recv_message.call_failed_before_recv_message = nullptr; GRPC_CLOSURE_INIT(&call_attempt_->recv_message_ready_, RecvMessageReady, this, grpc_schedule_on_exec_ctx); @@ -2372,9 +2373,9 @@ void RetryFilter::CallData::MaybeCacheSendOpsForBatch(PendingBatch* pending) { } // Set up cache for send_message ops. if (batch->send_message) { - ByteStreamCache* cache = arena_->New( - std::move(batch->payload->send_message.send_message)); - send_messages_.push_back(cache); + SliceBuffer* cache = arena_->New(std::move( + *absl::exchange(batch->payload->send_message.send_message, nullptr))); + send_messages_.push_back({cache, batch->payload->send_message.flags}); } // Save metadata batch for send_trailing_metadata ops. if (batch->send_trailing_metadata) { @@ -2394,14 +2395,13 @@ void RetryFilter::CallData::FreeCachedSendInitialMetadata() { } void RetryFilter::CallData::FreeCachedSendMessage(size_t idx) { - if (send_messages_[idx] != nullptr) { + if (send_messages_[idx].slices != nullptr) { if (GRPC_TRACE_FLAG_ENABLED(grpc_retry_trace)) { gpr_log(GPR_INFO, "chand=%p calld=%p: destroying send_messages[%" PRIuPTR "]", chand_, this, idx); } - send_messages_[idx]->Destroy(); - send_messages_[idx] = nullptr; + Destruct(absl::exchange(send_messages_[idx].slices, nullptr)); } } @@ -2465,7 +2465,7 @@ RetryFilter::CallData::PendingBatch* RetryFilter::CallData::PendingBatchesAdd( if (batch->send_message) { pending_send_message_ = true; bytes_buffered_for_retry_ += - batch->payload->send_message.send_message->length(); + batch->payload->send_message.send_message->Length(); } if (batch->send_trailing_metadata) { pending_send_trailing_metadata_ = true; diff --git a/src/core/ext/filters/client_channel/subchannel_stream_client.cc b/src/core/ext/filters/client_channel/subchannel_stream_client.cc index 5995d63d6d8..7e3d3a9e8a4 100644 --- a/src/core/ext/filters/client_channel/subchannel_stream_client.cc +++ b/src/core/ext/filters/client_channel/subchannel_stream_client.cc @@ -20,15 +20,11 @@ #include #include -#include -#include #include #include -#include #include -#include #include #include "src/core/lib/gpr/time_precise.h" @@ -38,7 +34,6 @@ #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/resource_quota/api.h" #include "src/core/lib/resource_quota/resource_quota.h" -#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/transport/error_utils.h" #define SUBCHANNEL_STREAM_INITIAL_CONNECT_BACKOFF_SECONDS 1 @@ -252,14 +247,9 @@ void SubchannelStreamClient::CallState::StartCallLocked() { payload_.send_initial_metadata.peer_string = nullptr; batch_.send_initial_metadata = true; // Add send_message op. - grpc_slice request_slice = - subchannel_stream_client_->event_handler_->EncodeSendMessageLocked(); - grpc_slice_buffer slice_buffer; - grpc_slice_buffer_init(&slice_buffer); - grpc_slice_buffer_add(&slice_buffer, request_slice); - send_message_.emplace(&slice_buffer, 0); - grpc_slice_buffer_destroy_internal(&slice_buffer); - payload_.send_message.send_message.reset(&*send_message_); + send_message_.Append(Slice( + subchannel_stream_client_->event_handler_->EncodeSendMessageLocked())); + payload_.send_message.send_message = &send_message_; batch_.send_message = true; // Add send_trailing_metadata op. payload_.send_trailing_metadata.send_trailing_metadata = @@ -374,42 +364,18 @@ void SubchannelStreamClient::CallState::RecvInitialMetadataReady( self->call_->Unref(DEBUG_LOCATION, "recv_initial_metadata_ready"); } -void SubchannelStreamClient::CallState::DoneReadingRecvMessage( - grpc_error_handle error) { - recv_message_.reset(); - if (error != GRPC_ERROR_NONE) { - GRPC_ERROR_UNREF(error); - Cancel(); - grpc_slice_buffer_destroy_internal(&recv_message_buffer_); +void SubchannelStreamClient::CallState::RecvMessageReady() { + if (!recv_message_.has_value()) { call_->Unref(DEBUG_LOCATION, "recv_message_ready"); return; } - // Concatenate the slices to form a single string. - std::unique_ptr recv_message_deleter; - uint8_t* recv_message; - if (recv_message_buffer_.count == 1) { - recv_message = GRPC_SLICE_START_PTR(recv_message_buffer_.slices[0]); - } else { - recv_message = - static_cast(gpr_malloc(recv_message_buffer_.length)); - recv_message_deleter.reset(recv_message); - size_t offset = 0; - for (size_t i = 0; i < recv_message_buffer_.count; ++i) { - memcpy(recv_message + offset, - GRPC_SLICE_START_PTR(recv_message_buffer_.slices[i]), - GRPC_SLICE_LENGTH(recv_message_buffer_.slices[i])); - offset += GRPC_SLICE_LENGTH(recv_message_buffer_.slices[i]); - } - } // Report payload. { MutexLock lock(&subchannel_stream_client_->mu_); if (subchannel_stream_client_->event_handler_ != nullptr) { - absl::string_view serialized_message( - reinterpret_cast(recv_message), recv_message_buffer_.length); absl::Status status = subchannel_stream_client_->event_handler_->RecvMessageReadyLocked( - subchannel_stream_client_.get(), serialized_message); + subchannel_stream_client_.get(), recv_message_->JoinIntoString()); if (!status.ok()) { if (GPR_UNLIKELY(subchannel_stream_client_->tracer_ != nullptr)) { gpr_log(GPR_INFO, @@ -424,7 +390,7 @@ void SubchannelStreamClient::CallState::DoneReadingRecvMessage( } } seen_response_.store(true, std::memory_order_release); - grpc_slice_buffer_destroy_internal(&recv_message_buffer_); + recv_message_.reset(); // Start another recv_message batch. // This re-uses the ref we're holding. // Note: Can't just reuse batch_ here, since we don't know that all @@ -438,62 +404,11 @@ void SubchannelStreamClient::CallState::DoneReadingRecvMessage( StartBatch(&recv_message_batch_); } -grpc_error_handle -SubchannelStreamClient::CallState::PullSliceFromRecvMessage() { - grpc_slice slice; - grpc_error_handle error = recv_message_->Pull(&slice); - if (error == GRPC_ERROR_NONE) { - grpc_slice_buffer_add(&recv_message_buffer_, slice); - } - return error; -} - -void SubchannelStreamClient::CallState::ContinueReadingRecvMessage() { - while (recv_message_->Next(SIZE_MAX, &recv_message_ready_)) { - grpc_error_handle error = PullSliceFromRecvMessage(); - if (error != GRPC_ERROR_NONE) { - DoneReadingRecvMessage(error); - return; - } - if (recv_message_buffer_.length == recv_message_->length()) { - DoneReadingRecvMessage(GRPC_ERROR_NONE); - break; - } - } -} - -void SubchannelStreamClient::CallState::OnByteStreamNext( - void* arg, grpc_error_handle error) { - auto* self = static_cast(arg); - if (error != GRPC_ERROR_NONE) { - self->DoneReadingRecvMessage(GRPC_ERROR_REF(error)); - return; - } - error = self->PullSliceFromRecvMessage(); - if (error != GRPC_ERROR_NONE) { - self->DoneReadingRecvMessage(error); - return; - } - if (self->recv_message_buffer_.length == self->recv_message_->length()) { - self->DoneReadingRecvMessage(GRPC_ERROR_NONE); - } else { - self->ContinueReadingRecvMessage(); - } -} - void SubchannelStreamClient::CallState::RecvMessageReady( void* arg, grpc_error_handle /*error*/) { auto* self = static_cast(arg); GRPC_CALL_COMBINER_STOP(&self->call_combiner_, "recv_message_ready"); - if (self->recv_message_ == nullptr) { - self->call_->Unref(DEBUG_LOCATION, "recv_message_ready"); - return; - } - grpc_slice_buffer_init(&self->recv_message_buffer_); - GRPC_CLOSURE_INIT(&self->recv_message_ready_, OnByteStreamNext, self, - grpc_schedule_on_exec_ctx); - self->ContinueReadingRecvMessage(); - // Ref will continue to be held until we finish draining the byte stream. + self->RecvMessageReady(); } void SubchannelStreamClient::CallState::RecvTrailingMetadataReady( diff --git a/src/core/ext/filters/client_channel/subchannel_stream_client.h b/src/core/ext/filters/client_channel/subchannel_stream_client.h index 08f03f7586d..32d8f00fd69 100644 --- a/src/core/ext/filters/client_channel/subchannel_stream_client.h +++ b/src/core/ext/filters/client_channel/subchannel_stream_client.h @@ -46,7 +46,7 @@ #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/memory_quota.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -132,6 +132,8 @@ class SubchannelStreamClient void CallEndedLocked(bool retry) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&subchannel_stream_client_->mu_); + void RecvMessageReady(); + static void OnComplete(void* arg, grpc_error_handle error); static void RecvInitialMetadataReady(void* arg, grpc_error_handle error); static void RecvMessageReady(void* arg, grpc_error_handle error); @@ -139,11 +141,6 @@ class SubchannelStreamClient static void StartCancel(void* arg, grpc_error_handle error); static void OnCancelComplete(void* arg, grpc_error_handle error); - static void OnByteStreamNext(void* arg, grpc_error_handle error); - void ContinueReadingRecvMessage(); - grpc_error_handle PullSliceFromRecvMessage(); - void DoneReadingRecvMessage(grpc_error_handle error); - static void AfterCallStackDestruction(void* arg, grpc_error_handle error); RefCountedPtr subchannel_stream_client_; @@ -169,7 +166,7 @@ class SubchannelStreamClient grpc_metadata_batch send_initial_metadata_; // send_message - absl::optional send_message_; + SliceBuffer send_message_; // send_trailing_metadata grpc_metadata_batch send_trailing_metadata_; @@ -179,9 +176,8 @@ class SubchannelStreamClient grpc_closure recv_initial_metadata_ready_; // recv_message - OrphanablePtr recv_message_; + absl::optional recv_message_; grpc_closure recv_message_ready_; - grpc_slice_buffer recv_message_buffer_; std::atomic seen_response_{false}; // True if the cancel_stream batch has been started. diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.cc b/src/core/ext/filters/http/message_compress/message_compress_filter.cc index e31f34ddb7c..aab33eb6556 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.cc +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.cc @@ -23,33 +23,26 @@ #include #include -#include #include -#include #include "absl/meta/type_traits.h" -#include "absl/status/status.h" #include "absl/types/optional.h" +#include "absl/utility/utility.h" #include #include #include -#include -#include #include #include "src/core/lib/compression/compression_internal.h" #include "src/core/lib/compression/message_compress.h" #include "src/core/lib/debug/trace.h" -#include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/profiling/timers.h" -#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/call.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -107,66 +100,40 @@ class CallData { channeld->default_compression_algorithm()))) { compression_algorithm_ = channeld->default_compression_algorithm(); } - GRPC_CLOSURE_INIT(&start_send_message_batch_in_call_combiner_, - StartSendMessageBatch, elem, grpc_schedule_on_exec_ctx); + GRPC_CLOSURE_INIT(&forward_send_message_batch_in_call_combiner_, + ForwardSendMessageBatch, elem, grpc_schedule_on_exec_ctx); } - ~CallData() { - if (state_initialized_) { - grpc_slice_buffer_destroy_internal(&slices_); - } - GRPC_ERROR_UNREF(cancel_error_); - } + ~CallData() { GRPC_ERROR_UNREF(cancel_error_); } void CompressStartTransportStreamOpBatch( grpc_call_element* elem, grpc_transport_stream_op_batch* batch); private: bool SkipMessageCompression(); - void InitializeState(grpc_call_element* elem); + void FinishSendMessage(grpc_call_element* elem); void ProcessSendInitialMetadata(grpc_call_element* elem, grpc_metadata_batch* initial_metadata); // Methods for processing a send_message batch - static void StartSendMessageBatch(void* elem_arg, grpc_error_handle unused); - static void OnSendMessageNextDone(void* elem_arg, grpc_error_handle error); - grpc_error_handle PullSliceFromSendMessage(); - void ContinueReadingSendMessage(grpc_call_element* elem); - void FinishSendMessage(grpc_call_element* elem); - void SendMessageBatchContinue(grpc_call_element* elem); static void FailSendMessageBatchInCallCombiner(void* calld_arg, grpc_error_handle error); - - static void SendMessageOnComplete(void* calld_arg, grpc_error_handle error); + static void ForwardSendMessageBatch(void* elem_arg, grpc_error_handle unused); grpc_core::CallCombiner* call_combiner_; grpc_compression_algorithm compression_algorithm_ = GRPC_COMPRESS_NONE; grpc_error_handle cancel_error_ = GRPC_ERROR_NONE; grpc_transport_stream_op_batch* send_message_batch_ = nullptr; bool seen_initial_metadata_ = false; - /* Set to true, if the fields below are initialized. */ - bool state_initialized_ = false; - grpc_closure start_send_message_batch_in_call_combiner_; - /* The fields below are only initialized when we compress the payload. - * Keep them at the bottom of the struct, so they don't pollute the - * cache-lines. */ - grpc_slice_buffer slices_; /**< Buffers up input slices to be compressed */ - // Allocate space for the replacement stream - std::aligned_storage::type - replacement_stream_; - grpc_closure* original_send_message_on_complete_ = nullptr; - grpc_closure send_message_on_complete_; - grpc_closure on_send_message_next_done_; + grpc_closure forward_send_message_batch_in_call_combiner_; }; // Returns true if we should skip message compression for the current message. bool CallData::SkipMessageCompression() { // If the flags of this message indicate that it shouldn't be compressed, we // skip message compression. - uint32_t flags = - send_message_batch_->payload->send_message.send_message->flags(); + uint32_t flags = send_message_batch_->payload->send_message.flags; if (flags & (GRPC_WRITE_NO_COMPRESS | GRPC_WRITE_INTERNAL_COMPRESS)) { return true; } @@ -175,16 +142,6 @@ bool CallData::SkipMessageCompression() { return compression_algorithm_ == GRPC_COMPRESS_NONE; } -void CallData::InitializeState(grpc_call_element* elem) { - GPR_DEBUG_ASSERT(!state_initialized_); - state_initialized_ = true; - grpc_slice_buffer_init(&slices_); - GRPC_CLOSURE_INIT(&send_message_on_complete_, SendMessageOnComplete, this, - grpc_schedule_on_exec_ctx); - GRPC_CLOSURE_INIT(&on_send_message_next_done_, OnSendMessageNextDone, elem, - grpc_schedule_on_exec_ctx); -} - void CallData::ProcessSendInitialMetadata( grpc_call_element* elem, grpc_metadata_batch* initial_metadata) { ChannelData* channeld = static_cast(elem->channel_data); @@ -197,7 +154,6 @@ void CallData::ProcessSendInitialMetadata( break; case GRPC_COMPRESS_DEFLATE: case GRPC_COMPRESS_GZIP: - InitializeState(elem); initial_metadata->Set(grpc_core::GrpcEncodingMetadata(), compression_algorithm_); break; @@ -209,68 +165,46 @@ void CallData::ProcessSendInitialMetadata( channeld->enabled_compression_algorithms()); } -void CallData::SendMessageOnComplete(void* calld_arg, grpc_error_handle error) { - CallData* calld = static_cast(calld_arg); - grpc_slice_buffer_reset_and_unref_internal(&calld->slices_); - grpc_core::Closure::Run(DEBUG_LOCATION, - calld->original_send_message_on_complete_, - GRPC_ERROR_REF(error)); -} - -void CallData::SendMessageBatchContinue(grpc_call_element* elem) { - // Note: The call to grpc_call_next_op() results in yielding the - // call combiner, so we need to clear send_message_batch_ before we do that. - grpc_transport_stream_op_batch* send_message_batch = send_message_batch_; - send_message_batch_ = nullptr; - grpc_call_next_op(elem, send_message_batch); -} - void CallData::FinishSendMessage(grpc_call_element* elem) { - GPR_DEBUG_ASSERT(compression_algorithm_ != GRPC_COMPRESS_NONE); // Compress the data if appropriate. - grpc_slice_buffer tmp; - grpc_slice_buffer_init(&tmp); - uint32_t send_flags = - send_message_batch_->payload->send_message.send_message->flags(); - bool did_compress = grpc_msg_compress(compression_algorithm_, &slices_, &tmp); - if (did_compress) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { - const char* algo_name; - const size_t before_size = slices_.length; - const size_t after_size = tmp.length; - const float savings_ratio = 1.0f - static_cast(after_size) / - static_cast(before_size); - GPR_ASSERT( - grpc_compression_algorithm_name(compression_algorithm_, &algo_name)); - gpr_log(GPR_INFO, - "Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR - " bytes (%.2f%% savings)", - algo_name, before_size, after_size, 100 * savings_ratio); - } - grpc_slice_buffer_swap(&slices_, &tmp); - send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; - } else { - if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { - const char* algo_name; - GPR_ASSERT( - grpc_compression_algorithm_name(compression_algorithm_, &algo_name)); - gpr_log(GPR_INFO, - "Algorithm '%s' enabled but decided not to compress. Input size: " - "%" PRIuPTR, - algo_name, slices_.length); + if (!SkipMessageCompression()) { + grpc_core::SliceBuffer tmp; + uint32_t& send_flags = send_message_batch_->payload->send_message.flags; + grpc_core::SliceBuffer* payload = + send_message_batch_->payload->send_message.send_message; + bool did_compress = + grpc_msg_compress(compression_algorithm_, payload->c_slice_buffer(), + tmp.c_slice_buffer()); + if (did_compress) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { + const char* algo_name; + const size_t before_size = payload->Length(); + const size_t after_size = tmp.Length(); + const float savings_ratio = 1.0f - static_cast(after_size) / + static_cast(before_size); + GPR_ASSERT(grpc_compression_algorithm_name(compression_algorithm_, + &algo_name)); + gpr_log(GPR_INFO, + "Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR + " bytes (%.2f%% savings)", + algo_name, before_size, after_size, 100 * savings_ratio); + } + tmp.Swap(payload); + send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; + } else { + if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { + const char* algo_name; + GPR_ASSERT(grpc_compression_algorithm_name(compression_algorithm_, + &algo_name)); + gpr_log( + GPR_INFO, + "Algorithm '%s' enabled but decided not to compress. Input size: " + "%" PRIuPTR, + algo_name, payload->Length()); + } } } - grpc_slice_buffer_destroy_internal(&tmp); - // Swap out the original byte stream with our new one and send the - // batch down. - new (&replacement_stream_) - grpc_core::SliceBufferByteStream(&slices_, send_flags); - send_message_batch_->payload->send_message.send_message.reset( - reinterpret_cast( - &replacement_stream_)); - original_send_message_on_complete_ = send_message_batch_->on_complete; - send_message_batch_->on_complete = &send_message_on_complete_; - SendMessageBatchContinue(elem); + grpc_call_next_op(elem, absl::exchange(send_message_batch_, nullptr)); } void CallData::FailSendMessageBatchInCallCombiner(void* calld_arg, @@ -284,78 +218,11 @@ void CallData::FailSendMessageBatchInCallCombiner(void* calld_arg, } } -// Pulls a slice from the send_message byte stream and adds it to slices_. -grpc_error_handle CallData::PullSliceFromSendMessage() { - grpc_slice incoming_slice; - grpc_error_handle error = - send_message_batch_->payload->send_message.send_message->Pull( - &incoming_slice); - if (error == GRPC_ERROR_NONE) { - grpc_slice_buffer_add(&slices_, incoming_slice); - } - return error; -} - -// Reads as many slices as possible from the send_message byte stream. -// If all data has been read, invokes FinishSendMessage(). Otherwise, -// an async call to ByteStream::Next() has been started, which will -// eventually result in calling OnSendMessageNextDone(). -void CallData::ContinueReadingSendMessage(grpc_call_element* elem) { - if (slices_.length == - send_message_batch_->payload->send_message.send_message->length()) { - FinishSendMessage(elem); - return; - } - while (send_message_batch_->payload->send_message.send_message->Next( - ~static_cast(0), &on_send_message_next_done_)) { - grpc_error_handle error = PullSliceFromSendMessage(); - if (error != GRPC_ERROR_NONE) { - // Closure callback; does not take ownership of error. - FailSendMessageBatchInCallCombiner(this, error); - GRPC_ERROR_UNREF(error); - return; - } - if (slices_.length == - send_message_batch_->payload->send_message.send_message->length()) { - FinishSendMessage(elem); - break; - } - } -} - -// Async callback for ByteStream::Next(). -void CallData::OnSendMessageNextDone(void* elem_arg, grpc_error_handle error) { +void CallData::ForwardSendMessageBatch(void* elem_arg, + grpc_error_handle /*unused*/) { grpc_call_element* elem = static_cast(elem_arg); CallData* calld = static_cast(elem->call_data); - if (error != GRPC_ERROR_NONE) { - // Closure callback; does not take ownership of error. - FailSendMessageBatchInCallCombiner(calld, error); - return; - } - error = calld->PullSliceFromSendMessage(); - if (error != GRPC_ERROR_NONE) { - // Closure callback; does not take ownership of error. - FailSendMessageBatchInCallCombiner(calld, error); - GRPC_ERROR_UNREF(error); - return; - } - if (calld->slices_.length == calld->send_message_batch_->payload->send_message - .send_message->length()) { - calld->FinishSendMessage(elem); - } else { - calld->ContinueReadingSendMessage(elem); - } -} - -void CallData::StartSendMessageBatch(void* elem_arg, - grpc_error_handle /*unused*/) { - grpc_call_element* elem = static_cast(elem_arg); - CallData* calld = static_cast(elem->call_data); - if (calld->SkipMessageCompression()) { - calld->SendMessageBatchContinue(elem); - } else { - calld->ContinueReadingSendMessage(elem); - } + calld->FinishSendMessage(elem); } void CallData::CompressStartTransportStreamOpBatch( @@ -372,9 +239,6 @@ void CallData::CompressStartTransportStreamOpBatch( GRPC_CLOSURE_CREATE(FailSendMessageBatchInCallCombiner, this, grpc_schedule_on_exec_ctx), GRPC_ERROR_REF(cancel_error_), "failing send_message op"); - } else { - send_message_batch_->payload->send_message.send_message->Shutdown( - GRPC_ERROR_REF(cancel_error_)); } } } else if (!GRPC_ERROR_IS_NONE(cancel_error_)) { @@ -395,7 +259,7 @@ void CallData::CompressStartTransportStreamOpBatch( // the call stack) will release the call combiner for each batch it sees. if (send_message_batch_ != nullptr) { GRPC_CALL_COMBINER_START( - call_combiner_, &start_send_message_batch_in_call_combiner_, + call_combiner_, &forward_send_message_batch_in_call_combiner_, GRPC_ERROR_NONE, "starting send_message after send_initial_metadata"); } } @@ -411,7 +275,7 @@ void CallData::CompressStartTransportStreamOpBatch( call_combiner_, "send_message batch pending send_initial_metadata"); return; } - StartSendMessageBatch(elem, GRPC_ERROR_NONE); + FinishSendMessage(elem); } else { // Pass control down the stack. grpc_call_next_op(elem, batch); diff --git a/src/core/ext/filters/http/message_compress/message_decompress_filter.cc b/src/core/ext/filters/http/message_compress/message_decompress_filter.cc index 277989004df..14544f82894 100644 --- a/src/core/ext/filters/http/message_compress/message_decompress_filter.cc +++ b/src/core/ext/filters/http/message_compress/message_decompress_filter.cc @@ -23,18 +23,13 @@ #include #include -#include #include -#include -#include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/types/optional.h" #include -#include -#include #include #include @@ -42,13 +37,11 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/compression/message_compress.h" #include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/profiling/timers.h" -#include "src/core/lib/slice/slice_internal.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -83,9 +76,6 @@ class CallData { OnRecvInitialMetadataReady, this, grpc_schedule_on_exec_ctx); // Initialize state for recv_message_ready callback - grpc_slice_buffer_init(&recv_slices_); - GRPC_CLOSURE_INIT(&on_recv_message_next_done_, OnRecvMessageNextDone, this, - grpc_schedule_on_exec_ctx); GRPC_CLOSURE_INIT(&on_recv_message_ready_, OnRecvMessageReady, this, grpc_schedule_on_exec_ctx); // Initialize state for recv_trailing_metadata_ready callback @@ -102,8 +92,6 @@ class CallData { } } - ~CallData() { grpc_slice_buffer_destroy_internal(&recv_slices_); } - void DecompressStartTransportStreamOpBatch( grpc_call_element* elem, grpc_transport_stream_op_batch* batch); @@ -113,10 +101,6 @@ class CallData { // Methods for processing a receive message event void MaybeResumeOnRecvMessageReady(); static void OnRecvMessageReady(void* arg, grpc_error_handle error); - static void OnRecvMessageNextDone(void* arg, grpc_error_handle error); - grpc_error_handle PullSliceFromRecvMessage(); - void ContinueReadingRecvMessage(); - void FinishRecvMessage(); void ContinueRecvMessageReadyCallback(grpc_error_handle error); // Methods for processing a recv_trailing_metadata event @@ -134,17 +118,10 @@ class CallData { bool seen_recv_message_ready_ = false; int max_recv_message_length_; grpc_compression_algorithm algorithm_ = GRPC_COMPRESS_NONE; + absl::optional* recv_message_ = nullptr; + uint32_t* recv_message_flags_ = nullptr; grpc_closure on_recv_message_ready_; grpc_closure* original_recv_message_ready_ = nullptr; - grpc_closure on_recv_message_next_done_; - OrphanablePtr* recv_message_ = nullptr; - // recv_slices_ holds the slices read from the original recv_message stream. - // It is initialized during construction and reset when a new stream is - // created using it. - grpc_slice_buffer recv_slices_; - std::aligned_storage::type - recv_replacement_stream_; // Fields for handling recv_trailing_metadata_ready callback bool seen_recv_trailing_metadata_ready_ = false; grpc_closure on_recv_trailing_metadata_ready_; @@ -188,101 +165,46 @@ void CallData::OnRecvMessageReady(void* arg, grpc_error_handle error) { if (calld->algorithm_ != GRPC_COMPRESS_NONE) { // recv_message can be NULL if trailing metadata is received instead of // message, or it's possible that the message was not compressed. - if (*calld->recv_message_ == nullptr || - (*calld->recv_message_)->length() == 0 || - ((*calld->recv_message_)->flags() & GRPC_WRITE_INTERNAL_COMPRESS) == - 0) { + if (!calld->recv_message_->has_value() || + (*calld->recv_message_)->Length() == 0 || + ((*calld->recv_message_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) == 0)) { return calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_NONE); } if (calld->max_recv_message_length_ >= 0 && - (*calld->recv_message_)->length() > + (*calld->recv_message_)->Length() > static_cast(calld->max_recv_message_length_)) { GPR_DEBUG_ASSERT(GRPC_ERROR_IS_NONE(calld->error_)); calld->error_ = grpc_error_set_int( GRPC_ERROR_CREATE_FROM_CPP_STRING( absl::StrFormat("Received message larger than max (%u vs. %d)", - (*calld->recv_message_)->length(), + (*calld->recv_message_)->Length(), calld->max_recv_message_length_)), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED); return calld->ContinueRecvMessageReadyCallback( GRPC_ERROR_REF(calld->error_)); } - grpc_slice_buffer_destroy_internal(&calld->recv_slices_); - grpc_slice_buffer_init(&calld->recv_slices_); - return calld->ContinueReadingRecvMessage(); + SliceBuffer decompressed_slices; + if (grpc_msg_decompress(calld->algorithm_, + (*calld->recv_message_)->c_slice_buffer(), + decompressed_slices.c_slice_buffer()) == 0) { + GPR_DEBUG_ASSERT(GRPC_ERROR_IS_NONE(calld->error_)); + calld->error_ = GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat( + "Unexpected error decompressing data for algorithm with " + "enum value ", + calld->algorithm_)); + } else { + *calld->recv_message_flags_ = + (*calld->recv_message_flags_ & (~GRPC_WRITE_INTERNAL_COMPRESS)) | + GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED; + (*calld->recv_message_)->Swap(&decompressed_slices); + } + return calld->ContinueRecvMessageReadyCallback( + GRPC_ERROR_REF(calld->error_)); } } calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_REF(error)); } -void CallData::ContinueReadingRecvMessage() { - while ((*recv_message_) - ->Next((*recv_message_)->length() - recv_slices_.length, - &on_recv_message_next_done_)) { - grpc_error_handle error = PullSliceFromRecvMessage(); - if (error != GRPC_ERROR_NONE) { - return ContinueRecvMessageReadyCallback(error); - } - // We have read the entire message. - if (recv_slices_.length == (*recv_message_)->length()) { - return FinishRecvMessage(); - } - } -} - -grpc_error_handle CallData::PullSliceFromRecvMessage() { - grpc_slice incoming_slice; - grpc_error_handle error = (*recv_message_)->Pull(&incoming_slice); - if (error == GRPC_ERROR_NONE) { - grpc_slice_buffer_add(&recv_slices_, incoming_slice); - } - return error; -} - -void CallData::OnRecvMessageNextDone(void* arg, grpc_error_handle error) { - CallData* calld = static_cast(arg); - if (error != GRPC_ERROR_NONE) { - return calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_REF(error)); - } - error = calld->PullSliceFromRecvMessage(); - if (error != GRPC_ERROR_NONE) { - return calld->ContinueRecvMessageReadyCallback(error); - } - if (calld->recv_slices_.length == (*calld->recv_message_)->length()) { - calld->FinishRecvMessage(); - } else { - calld->ContinueReadingRecvMessage(); - } -} - -void CallData::FinishRecvMessage() { - grpc_slice_buffer decompressed_slices; - grpc_slice_buffer_init(&decompressed_slices); - if (grpc_msg_decompress(algorithm_, &recv_slices_, &decompressed_slices) == - 0) { - GPR_DEBUG_ASSERT(error_ == GRPC_ERROR_NONE); - error_ = GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("Unexpected error decompressing data for algorithm with " - "enum value ", - algorithm_)); - grpc_slice_buffer_destroy_internal(&decompressed_slices); - } else { - uint32_t recv_flags = - ((*recv_message_)->flags() & (~GRPC_WRITE_INTERNAL_COMPRESS)) | - GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED; - // Swap out the original receive byte stream with our new one and send the - // batch down. - // Initializing recv_replacement_stream_ with decompressed_slices removes - // all the slices from decompressed_slices leaving it empty. - new (&recv_replacement_stream_) - SliceBufferByteStream(&decompressed_slices, recv_flags); - recv_message_->reset( - reinterpret_cast(&recv_replacement_stream_)); - recv_message_ = nullptr; - } - ContinueRecvMessageReadyCallback(GRPC_ERROR_REF(error_)); -} - void CallData::ContinueRecvMessageReadyCallback(grpc_error_handle error) { MaybeResumeOnRecvTrailingMetadataReady(); // The surface will clean up the receiving stream if there is an error. @@ -334,6 +256,7 @@ void CallData::DecompressStartTransportStreamOpBatch( // Handle recv_message if (batch->recv_message) { recv_message_ = batch->payload->recv_message.recv_message; + recv_message_flags_ = batch->payload->recv_message.flags; original_recv_message_ready_ = batch->payload->recv_message.recv_message_ready; batch->payload->recv_message.recv_message_ready = &on_recv_message_ready_; diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc index 2f6d2fbc1a1..a9cca8cc436 100644 --- a/src/core/ext/filters/message_size/message_size_filter.cc +++ b/src/core/ext/filters/message_size/message_size_filter.cc @@ -39,13 +39,12 @@ #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/service_config/service_config_call_data.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/channel_init.h" #include "src/core/lib/surface/channel_stack_type.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/transport.h" static void recv_message_ready(void* user_data, grpc_error_handle error); @@ -194,7 +193,7 @@ struct call_data { // The error caused by a message that is too large, or GRPC_ERROR_NONE grpc_error_handle error = GRPC_ERROR_NONE; // Used by recv_message_ready. - grpc_core::OrphanablePtr* recv_message = nullptr; + absl::optional* recv_message = nullptr; // Original recv_message_ready callback, invoked after our own. grpc_closure* next_recv_message_ready = nullptr; // Original recv_trailing_metadata callback, invoked after our own. @@ -210,13 +209,13 @@ struct call_data { static void recv_message_ready(void* user_data, grpc_error_handle error) { grpc_call_element* elem = static_cast(user_data); call_data* calld = static_cast(elem->call_data); - if (*calld->recv_message != nullptr && calld->limits.max_recv_size >= 0 && - (*calld->recv_message)->length() > + if (calld->recv_message->has_value() && calld->limits.max_recv_size >= 0 && + (*calld->recv_message)->Length() > static_cast(calld->limits.max_recv_size)) { grpc_error_handle new_error = grpc_error_set_int( GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( "Received message larger than max (%u vs. %d)", - (*calld->recv_message)->length(), calld->limits.max_recv_size)), + (*calld->recv_message)->Length(), calld->limits.max_recv_size)), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED); error = grpc_error_add_child(GRPC_ERROR_REF(error), new_error); GRPC_ERROR_UNREF(calld->error); @@ -269,13 +268,13 @@ static void message_size_start_transport_stream_op_batch( call_data* calld = static_cast(elem->call_data); // Check max send message size. if (op->send_message && calld->limits.max_send_size >= 0 && - op->payload->send_message.send_message->length() > + op->payload->send_message.send_message->Length() > static_cast(calld->limits.max_send_size)) { grpc_transport_stream_op_batch_finish_with_failure( op, grpc_error_set_int(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( "Sent message larger than max (%u vs. %d)", - op->payload->send_message.send_message->length(), + op->payload->send_message.send_message->Length(), calld->limits.max_send_size)), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED), diff --git a/src/core/ext/transport/binder/transport/binder_stream.h b/src/core/ext/transport/binder/transport/binder_stream.h index b04076b1f85..6c753e8ca09 100644 --- a/src/core/ext/transport/binder/transport/binder_stream.h +++ b/src/core/ext/transport/binder/transport/binder_stream.h @@ -80,7 +80,6 @@ struct grpc_binder_stream { grpc_binder_transport* t; grpc_stream_refcount* refcount; grpc_core::Arena* arena; - grpc_core::ManualConstructor sbs; int tx_code; const bool is_client; bool is_closed; @@ -106,7 +105,7 @@ struct grpc_binder_stream { grpc_metadata_batch* recv_initial_metadata; grpc_closure* recv_initial_metadata_ready = nullptr; bool* trailing_metadata_available = nullptr; - grpc_core::OrphanablePtr* recv_message; + absl::optional* recv_message; grpc_closure* recv_message_ready = nullptr; bool* call_failed_before_recv_message = nullptr; grpc_metadata_batch* recv_trailing_metadata; diff --git a/src/core/ext/transport/binder/transport/binder_transport.cc b/src/core/ext/transport/binder/transport/binder_transport.cc index d478980f668..42cd830bca2 100644 --- a/src/core/ext/transport/binder/transport/binder_transport.cc +++ b/src/core/ext/transport/binder/transport/binder_transport.cc @@ -37,7 +37,6 @@ #include "src/core/ext/transport/binder/wire_format/wire_writer.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/slice/slice_internal.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -257,12 +256,9 @@ static void recv_message_locked(void* arg, grpc_error_handle /*error*/) { return absl_status_to_grpc_error(args->message.status()); } } - grpc_slice_buffer buf; - grpc_slice_buffer_init(&buf); - grpc_slice_buffer_add(&buf, grpc_slice_from_cpp_string(*args->message)); - - gbs->sbs.Init(&buf, 0); - gbs->recv_message->reset(gbs->sbs.get()); + grpc_core::SliceBuffer buf; + buf.Append(grpc_core::Slice(grpc_slice_from_cpp_string(*args->message))); + *gbs->recv_message = std::move(buf); return GRPC_ERROR_NONE; }(); @@ -412,7 +408,7 @@ static void perform_stream_op_locked(void* stream_op, if (gbs->is_closed) { if (op->send_message) { // Reset the send_message payload to prevent memory leaks. - op->payload->send_message.send_message.reset(); + op->payload->send_message.send_message->Clear(); } if (op->recv_initial_metadata) { grpc_core::ExecCtx::Run( @@ -454,27 +450,7 @@ static void perform_stream_op_locked(void* stream_op, } if (op->send_message) { gpr_log(GPR_INFO, "send_message"); - size_t remaining = op->payload->send_message.send_message->length(); - std::string message_data; - while (remaining > 0) { - grpc_slice message_slice; - // TODO(waynetu): Temporarily assume that the message is ready. - GPR_ASSERT( - op->payload->send_message.send_message->Next(SIZE_MAX, nullptr)); - grpc_error_handle error = - op->payload->send_message.send_message->Pull(&message_slice); - // TODO(waynetu): Cancel the stream if error is not GRPC_ERROR_NONE. - GPR_ASSERT(error == GRPC_ERROR_NONE); - uint8_t* p = GRPC_SLICE_START_PTR(message_slice); - size_t len = GRPC_SLICE_LENGTH(message_slice); - remaining -= len; - message_data += std::string(reinterpret_cast(p), len); - grpc_slice_unref_internal(message_slice); - } - tx->SetData(message_data); - // TODO(b/192369787): Are we supposed to reset here to avoid - // use-after-free issue in call.cc? - op->payload->send_message.send_message.reset(); + tx->SetData(op->payload->send_message.send_message->JoinIntoString()); } if (op->send_trailing_metadata) { diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index e17a0b36bff..03507c8c1eb 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -37,6 +37,7 @@ #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "absl/types/variant.h" #include #include @@ -61,9 +62,6 @@ #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/bitset.h" #include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/global_config_env.h" -#include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/status_helper.h" #include "src/core/lib/gprpp/time.h" @@ -75,16 +73,17 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/profiling/timers.h" +#include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/api.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/memory_quota.h" #include "src/core/lib/resource_quota/resource_quota.h" #include "src/core/lib/resource_quota/trace.h" #include "src/core/lib/slice/slice.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/transport/bdp_estimator.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/http2_errors.h" @@ -93,12 +92,6 @@ #include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport_impl.h" -GPR_GLOBAL_CONFIG_DEFINE_BOOL( - grpc_experimental_disable_flow_control, false, - "If set, flow control will be effectively disabled. Max out all values and " - "assume the remote peer does the same. Thus we can ignore any flow control " - "bookkeeping, error checking, and decision making"); - #define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024) #define MAX_WINDOW 0x7fffffffu #define MAX_WRITE_BUFFER_SIZE (64 * 1024 * 1024) @@ -151,8 +144,6 @@ static void read_action(void* t, grpc_error_handle error); static void read_action_locked(void* t, grpc_error_handle error); static void continue_read_action_locked(grpc_chttp2_transport* t); -static void complete_fetch(void* gs, grpc_error_handle error); -static void complete_fetch_locked(void* gs, grpc_error_handle error); // Set a transport level setting, and push it to our peer static void queue_setting_update(grpc_chttp2_transport* t, grpc_chttp2_setting_id id, uint32_t value); @@ -202,8 +193,6 @@ static void finish_keepalive_ping_locked(void* arg, grpc_error_handle error); static void keepalive_watchdog_fired(void* arg, grpc_error_handle error); static void keepalive_watchdog_fired_locked(void* arg, grpc_error_handle error); -static void reset_byte_stream(void* arg, grpc_error_handle error); - namespace grpc_core { namespace { @@ -280,8 +269,6 @@ grpc_chttp2_transport::~grpc_chttp2_transport() { write_cb_pool = next; } - flow_control.Destroy(); - GRPC_ERROR_UNREF(closed_with_error); gpr_free(ping_acks); if (grpc_core::test_only_destruct_callback != nullptr) { @@ -291,11 +278,9 @@ grpc_chttp2_transport::~grpc_chttp2_transport() { static const grpc_transport_vtable* get_vtable(void); -// Returns whether bdp is enabled -static bool read_channel_args(grpc_chttp2_transport* t, +static void read_channel_args(grpc_chttp2_transport* t, const grpc_channel_args* channel_args, bool is_client) { - bool enable_bdp = true; bool channelz_enabled = GRPC_ENABLE_CHANNELZ_DEFAULT; size_t i; int j; @@ -345,9 +330,6 @@ static bool read_channel_args(grpc_chttp2_transport* t, GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE)) { t->write_buffer_size = static_cast(grpc_channel_arg_get_integer( &channel_args->args[i], {0, 0, MAX_WRITE_BUFFER_SIZE})); - } else if (0 == - strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_BDP_PROBE)) { - enable_bdp = grpc_channel_arg_get_bool(&channel_args->args[i], true); } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_KEEPALIVE_TIME_MS)) { const int value = grpc_channel_arg_get_integer( @@ -439,7 +421,6 @@ static bool read_channel_args(grpc_chttp2_transport* t, grpc_core::channelz::SocketNode::Security::GetFromChannelArgs( channel_args)); } - return enable_bdp; } static void init_transport_keepalive_settings(grpc_chttp2_transport* t) { @@ -510,6 +491,10 @@ grpc_chttp2_transport::grpc_chttp2_transport( GRPC_CHANNEL_READY), is_client(is_client), next_stream_id(is_client ? 1 : 2), + flow_control(peer_string.c_str(), + grpc_channel_args_find_bool(channel_args, + GRPC_ARG_HTTP2_BDP_PROBE, true), + &memory_owner), deframe_state(is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0) { GPR_ASSERT(strlen(GRPC_CHTTP2_CLIENT_CONNECT_STRING) == GRPC_CHTTP2_CLIENT_CONNECT_STRLEN); @@ -551,19 +536,8 @@ grpc_chttp2_transport::grpc_chttp2_transport( configure_transport_ping_policy(this); init_transport_keepalive_settings(this); - bool enable_bdp = true; - if (channel_args) { - enable_bdp = read_channel_args(this, channel_args, is_client); - } - - static const bool kEnableFlowControl = - !GPR_GLOBAL_CONFIG_GET(grpc_experimental_disable_flow_control); - if (kEnableFlowControl) { - flow_control.Init(this, - enable_bdp); - } else { - flow_control.Init(this); - enable_bdp = false; + if (channel_args != nullptr) { + read_channel_args(this, channel_args, is_client); } // No pings allowed before receiving a header or data frame. @@ -576,9 +550,9 @@ grpc_chttp2_transport::grpc_chttp2_transport( init_keepalive_pings_if_enabled(this); - if (enable_bdp) { + if (flow_control.bdp_probe()) { bdp_ping_blocked = true; - grpc_chttp2_act_on_flowctl_action(flow_control->PeriodicUpdate(), this, + grpc_chttp2_act_on_flowctl_action(flow_control.PeriodicUpdate(), this, nullptr); } @@ -703,29 +677,23 @@ grpc_chttp2_stream::grpc_chttp2_stream(grpc_chttp2_transport* t, refcount(refcount), reffer(this), initial_metadata_buffer(arena), - trailing_metadata_buffer(arena) { + trailing_metadata_buffer(arena), + flow_control(&t->flow_control) { if (server_data) { id = static_cast(reinterpret_cast(server_data)); *t->accepting_stream = this; grpc_chttp2_stream_map_add(&t->stream_map, id, this); post_destructive_reclaimer(t); } - if (t->flow_control->flow_control_enabled()) { - flow_control.Init( - static_cast( - t->flow_control.get()), - this); - } else { - flow_control.Init(); - } grpc_slice_buffer_init(&frame_storage); - grpc_slice_buffer_init(&unprocessed_incoming_frames_buffer); grpc_slice_buffer_init(&flow_controlled_buffer); - GRPC_CLOSURE_INIT(&reset_byte_stream, ::reset_byte_stream, this, nullptr); } grpc_chttp2_stream::~grpc_chttp2_stream() { + grpc_chttp2_list_remove_stalled_by_stream(t, this); + grpc_chttp2_list_remove_stalled_by_transport(t, this); + if (t->channelz_socket != nullptr) { if ((t->is_client && eos_received) || (!t->is_client && eos_sent)) { t->channelz_socket->RecordStreamSucceeded(); @@ -739,7 +707,6 @@ grpc_chttp2_stream::~grpc_chttp2_stream() { GPR_ASSERT(grpc_chttp2_stream_map_find(&t->stream_map, id) == nullptr); } - grpc_slice_buffer_destroy_internal(&unprocessed_incoming_frames_buffer); grpc_slice_buffer_destroy_internal(&frame_storage); for (int i = 0; i < STREAM_LIST_COUNT; i++) { @@ -751,7 +718,6 @@ grpc_chttp2_stream::~grpc_chttp2_stream() { } GPR_ASSERT(send_initial_metadata_finished == nullptr); - GPR_ASSERT(fetching_send_message == nullptr); GPR_ASSERT(send_trailing_metadata_finished == nullptr); GPR_ASSERT(recv_initial_metadata_ready == nullptr); GPR_ASSERT(recv_message_ready == nullptr); @@ -759,8 +725,6 @@ grpc_chttp2_stream::~grpc_chttp2_stream() { grpc_slice_buffer_destroy_internal(&flow_controlled_buffer); GRPC_ERROR_UNREF(read_closed_error); GRPC_ERROR_UNREF(write_closed_error); - GRPC_ERROR_UNREF(byte_stream_error); - flow_control.Destroy(); GRPC_CHTTP2_UNREF_TRANSPORT(t, "stream"); grpc_core::ExecCtx::Run(DEBUG_LOCATION, destroy_stream_arg, GRPC_ERROR_NONE); } @@ -1326,94 +1290,6 @@ static bool contains_non_ok_status(grpc_metadata_batch* batch) { GRPC_STATUS_OK; } -static void maybe_become_writable_due_to_send_msg(grpc_chttp2_transport* t, - grpc_chttp2_stream* s) { - if (s->id != 0 && (!s->write_buffering || - s->flow_controlled_buffer.length > t->write_buffer_size)) { - grpc_chttp2_mark_stream_writable(t, s); - grpc_chttp2_initiate_write(t, GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE); - } -} - -static void add_fetched_slice_locked(grpc_chttp2_transport* t, - grpc_chttp2_stream* s) { - s->fetched_send_message_length += - static_cast GRPC_SLICE_LENGTH(s->fetching_slice); - grpc_slice_buffer_add(&s->flow_controlled_buffer, s->fetching_slice); - maybe_become_writable_due_to_send_msg(t, s); -} - -static void continue_fetching_send_locked(grpc_chttp2_transport* t, - grpc_chttp2_stream* s) { - for (;;) { - if (s->fetching_send_message == nullptr) { - // Stream was cancelled before message fetch completed - abort(); /* TODO(ctiller): what cleanup here? */ - } - if (s->fetched_send_message_length == s->fetching_send_message->length()) { - int64_t notify_offset = s->next_message_end_offset; - if (notify_offset <= s->flow_controlled_bytes_written) { - grpc_chttp2_complete_closure_step( - t, s, &s->fetching_send_message_finished, GRPC_ERROR_NONE, - "fetching_send_message_finished"); - } else { - grpc_chttp2_write_cb* cb = t->write_cb_pool; - if (cb == nullptr) { - cb = static_cast(gpr_malloc(sizeof(*cb))); - } else { - t->write_cb_pool = cb->next; - } - cb->call_at_byte = notify_offset; - cb->closure = s->fetching_send_message_finished; - s->fetching_send_message_finished = nullptr; - grpc_chttp2_write_cb** list = - s->fetching_send_message->flags() & GRPC_WRITE_THROUGH - ? &s->on_write_finished_cbs - : &s->on_flow_controlled_cbs; - cb->next = *list; - *list = cb; - } - s->fetching_send_message.reset(); - return; /* early out */ - } else if (s->fetching_send_message->Next( - UINT32_MAX, GRPC_CLOSURE_INIT(&s->complete_fetch_locked, - ::complete_fetch, s, - grpc_schedule_on_exec_ctx))) { - grpc_error_handle error = - s->fetching_send_message->Pull(&s->fetching_slice); - if (error != GRPC_ERROR_NONE) { - s->fetching_send_message.reset(); - grpc_chttp2_cancel_stream(t, s, error); - } else { - add_fetched_slice_locked(t, s); - } - } - } -} - -static void complete_fetch(void* gs, grpc_error_handle error) { - grpc_chttp2_stream* s = static_cast(gs); - s->t->combiner->Run(GRPC_CLOSURE_INIT(&s->complete_fetch_locked, - ::complete_fetch_locked, s, nullptr), - GRPC_ERROR_REF(error)); -} - -static void complete_fetch_locked(void* gs, grpc_error_handle error) { - grpc_chttp2_stream* s = static_cast(gs); - grpc_chttp2_transport* t = s->t; - if (error == GRPC_ERROR_NONE) { - error = s->fetching_send_message->Pull(&s->fetching_slice); - if (error == GRPC_ERROR_NONE) { - add_fetched_slice_locked(t, s); - continue_fetching_send_locked(t, s); - } - } - if (error != GRPC_ERROR_NONE) { - s->fetching_send_message.reset(); - grpc_chttp2_cancel_stream(t, s, error); - } -} - static void log_metadata(const grpc_metadata_batch* md_batch, uint32_t id, bool is_client, bool is_initial) { const std::string prefix = absl::StrCat( @@ -1508,8 +1384,7 @@ static void perform_stream_op_locked(void* stream_op, GPR_ASSERT(s->id != 0); grpc_chttp2_mark_stream_writable(t, s); if (!(op->send_message && - (op->payload->send_message.send_message->flags() & - GRPC_WRITE_BUFFER_HINT))) { + (op->payload->send_message.flags & GRPC_WRITE_BUFFER_HINT))) { grpc_chttp2_initiate_write( t, GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA); } @@ -1533,32 +1408,28 @@ static void perform_stream_op_locked(void* stream_op, GRPC_STATS_INC_HTTP2_OP_SEND_MESSAGE(); t->num_messages_in_next_write++; GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE( - op->payload->send_message.send_message->length()); + op->payload->send_message.send_message->Length()); on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE; - s->fetching_send_message_finished = add_closure_barrier(op->on_complete); + s->send_message_finished = add_closure_barrier(op->on_complete); + const uint32_t flags = op_payload->send_message.flags; if (s->write_closed) { op->payload->send_message.stream_write_closed = true; // We should NOT return an error here, so as to avoid a cancel OP being // started. The surface layer will notice that the stream has been closed // for writes and fail the send message op. - op->payload->send_message.send_message.reset(); - grpc_chttp2_complete_closure_step( - t, s, &s->fetching_send_message_finished, GRPC_ERROR_NONE, - "fetching_send_message_finished"); + grpc_chttp2_complete_closure_step(t, s, &s->send_message_finished, + GRPC_ERROR_NONE, + "fetching_send_message_finished"); } else { - GPR_ASSERT(s->fetching_send_message == nullptr); uint8_t* frame_hdr = grpc_slice_buffer_tiny_add( &s->flow_controlled_buffer, GRPC_HEADER_SIZE_IN_BYTES); - uint32_t flags = op_payload->send_message.send_message->flags(); frame_hdr[0] = (flags & GRPC_WRITE_INTERNAL_COMPRESS) != 0; - size_t len = op_payload->send_message.send_message->length(); + size_t len = op_payload->send_message.send_message->Length(); frame_hdr[1] = static_cast(len >> 24); frame_hdr[2] = static_cast(len >> 16); frame_hdr[3] = static_cast(len >> 8); frame_hdr[4] = static_cast(len); - s->fetching_send_message = - std::move(op_payload->send_message.send_message); - s->fetched_send_message_length = 0; + s->next_message_end_offset = s->flow_controlled_bytes_written + static_cast(s->flow_controlled_buffer.length) + @@ -1569,8 +1440,44 @@ static void perform_stream_op_locked(void* stream_op, } else { s->write_buffering = false; } - continue_fetching_send_locked(t, s); - maybe_become_writable_due_to_send_msg(t, s); + + grpc_slice* const slices = + op_payload->send_message.send_message->c_slice_buffer()->slices; + grpc_slice* const end = + slices + op_payload->send_message.send_message->Count(); + for (grpc_slice* slice = slices; slice != end; slice++) { + grpc_slice_buffer_add(&s->flow_controlled_buffer, + grpc_slice_ref_internal(*slice)); + } + + int64_t notify_offset = s->next_message_end_offset; + if (notify_offset <= s->flow_controlled_bytes_written) { + grpc_chttp2_complete_closure_step(t, s, &s->send_message_finished, + GRPC_ERROR_NONE, + "fetching_send_message_finished"); + } else { + grpc_chttp2_write_cb* cb = t->write_cb_pool; + if (cb == nullptr) { + cb = static_cast(gpr_malloc(sizeof(*cb))); + } else { + t->write_cb_pool = cb->next; + } + cb->call_at_byte = notify_offset; + cb->closure = s->send_message_finished; + s->send_message_finished = nullptr; + grpc_chttp2_write_cb** list = flags & GRPC_WRITE_THROUGH + ? &s->on_write_finished_cbs + : &s->on_flow_controlled_cbs; + cb->next = *list; + *list = cb; + } + + if (s->id != 0 && + (!s->write_buffering || + s->flow_controlled_buffer.length > t->write_buffer_size)) { + grpc_chttp2_mark_stream_writable(t, s); + grpc_chttp2_initiate_write(t, GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE); + } } } @@ -1624,28 +1531,14 @@ static void perform_stream_op_locked(void* stream_op, if (op->recv_message) { GRPC_STATS_INC_HTTP2_OP_RECV_MESSAGE(); - size_t before = 0; GPR_ASSERT(s->recv_message_ready == nullptr); - GPR_ASSERT(!s->pending_byte_stream); s->recv_message_ready = op_payload->recv_message.recv_message_ready; s->recv_message = op_payload->recv_message.recv_message; + s->recv_message->emplace(); + s->recv_message_flags = op_payload->recv_message.flags; s->call_failed_before_recv_message = op_payload->recv_message.call_failed_before_recv_message; - if (s->id != 0) { - if (!s->read_closed) { - before = s->frame_storage.length + - s->unprocessed_incoming_frames_buffer.length; - } - } - grpc_chttp2_maybe_complete_recv_message(t, s); - if (s->id != 0) { - if (!s->read_closed && s->frame_storage.length == 0) { - size_t after = s->unprocessed_incoming_frames_buffer_cached_length; - s->flow_control->IncomingByteStreamUpdate(GRPC_HEADER_SIZE_IN_BYTES, - before - after); - grpc_chttp2_act_on_flowctl_action(s->flow_control->MakeAction(), t, s); - } - } + grpc_chttp2_maybe_complete_recv_trailing_metadata(t, s); } if (op->recv_trailing_metadata) { @@ -2019,10 +1912,6 @@ void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_chttp2_transport* t, s->published_metadata[0] != GRPC_METADATA_NOT_PUBLISHED) { if (s->seen_error) { grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); - if (!s->pending_byte_stream) { - grpc_slice_buffer_reset_and_unref_internal( - &s->unprocessed_incoming_frames_buffer); - } } *s->recv_initial_metadata = std::move(s->initial_metadata_buffer); s->recv_initial_metadata->Set(grpc_core::PeerString(), t->peer_string); @@ -2039,47 +1928,62 @@ void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_chttp2_transport* t, } } -void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* /*t*/, +void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { + if (s->recv_message_ready == nullptr) return; + + grpc_core::chttp2::StreamFlowControl::IncomingUpdateContext upd( + &s->flow_control); grpc_error_handle error = GRPC_ERROR_NONE; - if (s->recv_message_ready != nullptr) { - *s->recv_message = nullptr; + + // Lambda is immediately invoked as a big scoped section that can be + // exited out of at any point by returning. + [&]() { if (s->final_metadata_requested && s->seen_error) { grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); - if (!s->pending_byte_stream) { - grpc_slice_buffer_reset_and_unref_internal( - &s->unprocessed_incoming_frames_buffer); - } - } - if (!s->pending_byte_stream) { - while (s->unprocessed_incoming_frames_buffer.length > 0 || - s->frame_storage.length > 0) { - if (s->unprocessed_incoming_frames_buffer.length == 0) { - grpc_slice_buffer_swap(&s->unprocessed_incoming_frames_buffer, - &s->frame_storage); - } - error = grpc_deframe_unprocessed_incoming_frames( - &s->data_parser, s, &s->unprocessed_incoming_frames_buffer, nullptr, - s->recv_message); - if (!GRPC_ERROR_IS_NONE(error)) { - s->seen_error = true; - grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); - grpc_slice_buffer_reset_and_unref_internal( - &s->unprocessed_incoming_frames_buffer); - break; - } else if (*s->recv_message != nullptr) { - break; + s->recv_message->reset(); + } else { + if (s->frame_storage.length != 0) { + while (true) { + GPR_ASSERT(s->frame_storage.length > 0); + uint32_t min_progress_size; + auto r = grpc_deframe_unprocessed_incoming_frames( + s, &min_progress_size, &**s->recv_message, s->recv_message_flags); + if (absl::holds_alternative(r)) { + if (s->read_closed) { + grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); + s->recv_message->reset(); + break; + } else { + upd.SetMinProgressSize(min_progress_size); + return; // Out of lambda to enclosing function + } + } else { + error = absl::get(r); + if (!GRPC_ERROR_IS_NONE(error)) { + s->seen_error = true; + grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); + break; + } else { + if (t->channelz_socket != nullptr) { + t->channelz_socket->RecordMessageReceived(); + } + break; + } + } } + } else if (s->read_closed) { + s->recv_message->reset(); + } else { + upd.SetMinProgressSize(GRPC_HEADER_SIZE_IN_BYTES); + return; // Out of lambda to enclosing function } } // save the length of the buffer before handing control back to application // threads. Needed to support correct flow control bookkeeping - s->unprocessed_incoming_frames_buffer_cached_length = - s->unprocessed_incoming_frames_buffer.length; - if (GRPC_ERROR_IS_NONE(error) && *s->recv_message != nullptr) { + if (GRPC_ERROR_IS_NONE(error) && s->recv_message->has_value()) { null_then_sched_closure(&s->recv_message_ready); } else if (s->published_metadata[1] != GRPC_METADATA_NOT_PUBLISHED) { - *s->recv_message = nullptr; if (s->call_failed_before_recv_message != nullptr) { *s->call_failed_before_recv_message = (s->published_metadata[1] != GRPC_METADATA_PUBLISHED_AT_CLOSE); @@ -2087,7 +1991,10 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* /*t*/, null_then_sched_closure(&s->recv_message_ready); } GRPC_ERROR_UNREF(error); - } + }(); + + upd.SetPendingSize(s->frame_storage.length); + grpc_chttp2_act_on_flowctl_action(upd.MakeAction(), t, s); } void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_chttp2_transport* t, @@ -2097,26 +2004,8 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_chttp2_transport* t, s->write_closed) { if (s->seen_error || !t->is_client) { grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); - if (!s->pending_byte_stream) { - grpc_slice_buffer_reset_and_unref_internal( - &s->unprocessed_incoming_frames_buffer); - } } - bool pending_data = s->pending_byte_stream || - s->unprocessed_incoming_frames_buffer.length > 0; - if (s->read_closed && s->frame_storage.length > 0 && !pending_data && - !s->seen_error && s->recv_trailing_metadata_finished != nullptr) { - // Maybe some SYNC_FLUSH data is left in frame_storage. Consume them and - // maybe decompress the next 5 bytes in the stream. - grpc_slice_buffer_move_first( - &s->frame_storage, - std::min(s->frame_storage.length, size_t(GRPC_HEADER_SIZE_IN_BYTES)), - &s->unprocessed_incoming_frames_buffer); - if (s->unprocessed_incoming_frames_buffer.length > 0) { - pending_data = true; - } - } - if (s->read_closed && s->frame_storage.length == 0 && !pending_data && + if (s->read_closed && s->frame_storage.length == 0 && s->recv_trailing_metadata_finished != nullptr) { grpc_transport_move_stats(&s->stats, s->collecting_stats); s->collecting_stats = nullptr; @@ -2136,20 +2025,6 @@ static void remove_stream(grpc_chttp2_transport* t, uint32_t id, t->incoming_stream = nullptr; grpc_chttp2_parsing_become_skip_parser(t); } - if (s->pending_byte_stream) { - if (s->on_next != nullptr) { - grpc_core::Chttp2IncomingByteStream* bs = s->data_parser.parsing_frame; - if (error == GRPC_ERROR_NONE) { - error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message"); - } - bs->PublishError(error); - bs->Unref(); - s->data_parser.parsing_frame = nullptr; - } else { - GRPC_ERROR_UNREF(s->byte_stream_error); - s->byte_stream_error = GRPC_ERROR_REF(error); - } - } if (grpc_chttp2_stream_map_size(&t->stream_map) == 0) { post_benign_reclaimer(t); @@ -2283,8 +2158,7 @@ void grpc_chttp2_fail_pending_writes(grpc_chttp2_transport* t, GRPC_ERROR_REF(error), "send_trailing_metadata_finished"); - s->fetching_send_message.reset(); - grpc_chttp2_complete_closure_step(t, s, &s->fetching_send_message_finished, + grpc_chttp2_complete_closure_step(t, s, &s->send_message_finished, GRPC_ERROR_REF(error), "fetching_send_message_finished"); flush_write_list(t, s, &s->on_write_finished_cbs, GRPC_ERROR_REF(error)); @@ -2554,8 +2428,11 @@ void grpc_chttp2_act_on_flowctl_action( const grpc_core::chttp2::FlowControlAction& action, grpc_chttp2_transport* t, grpc_chttp2_stream* s) { WithUrgency(t, action.send_stream_update(), - GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL, - [t, s]() { grpc_chttp2_mark_stream_writable(t, s); }); + GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL, [t, s]() { + if (s->id != 0) { + grpc_chttp2_mark_stream_writable(t, s); + } + }); WithUrgency(t, action.send_transport_update(), GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL, []() {}); WithUrgency(t, action.send_initial_window_update(), @@ -2703,13 +2580,12 @@ static void continue_read_action_locked(grpc_chttp2_transport* t) { grpc_schedule_on_exec_ctx); grpc_endpoint_read(t->ep, &t->read_buffer, &t->read_action_locked, urgent, /*min_progress_size=*/1); - grpc_chttp2_act_on_flowctl_action(t->flow_control->MakeAction(), t, nullptr); } // t is reffed prior to calling the first time, and once the callback chain // that kicks off finishes, it's unreffed void schedule_bdp_ping_locked(grpc_chttp2_transport* t) { - t->flow_control->bdp_estimator()->SchedulePing(); + t->flow_control.bdp_estimator()->SchedulePing(); send_ping_locked( t, GRPC_CLOSURE_INIT(&t->start_bdp_ping_locked, start_bdp_ping, t, @@ -2739,7 +2615,7 @@ static void start_bdp_ping_locked(void* tp, grpc_error_handle error) { if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_WAITING) { grpc_timer_cancel(&t->keepalive_ping_timer); } - t->flow_control->bdp_estimator()->StartPing(); + t->flow_control.bdp_estimator()->StartPing(); t->bdp_ping_started = true; } @@ -2770,8 +2646,8 @@ static void finish_bdp_ping_locked(void* tp, grpc_error_handle error) { } t->bdp_ping_started = false; grpc_core::Timestamp next_ping = - t->flow_control->bdp_estimator()->CompletePing(); - grpc_chttp2_act_on_flowctl_action(t->flow_control->PeriodicUpdate(), t, + t->flow_control.bdp_estimator()->CompletePing(); + grpc_chttp2_act_on_flowctl_action(t->flow_control.PeriodicUpdate(), t, nullptr); GPR_ASSERT(!t->have_next_bdp_ping_timer); t->have_next_bdp_ping_timer = true; @@ -2798,7 +2674,7 @@ static void next_bdp_ping_timer_expired_locked(void* tp, GRPC_CHTTP2_UNREF_TRANSPORT(t, "bdp_ping"); return; } - if (t->flow_control->bdp_estimator()->accumulator() == 0) { + if (t->flow_control.bdp_estimator()->accumulator() == 0) { // Block the bdp ping till we receive more data. t->bdp_ping_blocked = true; GRPC_CHTTP2_UNREF_TRANSPORT(t, "bdp_ping"); @@ -3034,187 +2910,6 @@ static void set_pollset_set(grpc_transport* gt, grpc_stream* /*gs*/, grpc_endpoint_add_to_pollset_set(t->ep, pollset_set); } -// -// BYTE STREAM -// - -static void reset_byte_stream(void* arg, grpc_error_handle error) { - grpc_chttp2_stream* s = static_cast(arg); - s->pending_byte_stream = false; - if (error == GRPC_ERROR_NONE) { - grpc_chttp2_maybe_complete_recv_message(s->t, s); - grpc_chttp2_maybe_complete_recv_trailing_metadata(s->t, s); - } else { - GPR_ASSERT(error != GRPC_ERROR_NONE); - grpc_core::ExecCtx::Run(DEBUG_LOCATION, s->on_next, GRPC_ERROR_REF(error)); - s->on_next = nullptr; - GRPC_ERROR_UNREF(s->byte_stream_error); - s->byte_stream_error = GRPC_ERROR_NONE; - grpc_chttp2_cancel_stream(s->t, s, GRPC_ERROR_REF(error)); - s->byte_stream_error = GRPC_ERROR_REF(error); - } -} - -namespace grpc_core { - -Chttp2IncomingByteStream::Chttp2IncomingByteStream( - grpc_chttp2_transport* transport, grpc_chttp2_stream* stream, - uint32_t frame_size, uint32_t flags) - : ByteStream(frame_size, flags), - transport_(transport), - stream_(stream), - refs_(2), - remaining_bytes_(frame_size) { - GRPC_ERROR_UNREF(stream->byte_stream_error); - stream->byte_stream_error = GRPC_ERROR_NONE; -} - -void Chttp2IncomingByteStream::OrphanLocked( - void* arg, grpc_error_handle /*error_ignored*/) { - Chttp2IncomingByteStream* bs = static_cast(arg); - grpc_chttp2_stream* s = bs->stream_; - grpc_chttp2_transport* t = s->t; - bs->Unref(); - s->pending_byte_stream = false; - grpc_chttp2_maybe_complete_recv_message(t, s); - grpc_chttp2_maybe_complete_recv_trailing_metadata(t, s); -} - -void Chttp2IncomingByteStream::Orphan() { - GPR_TIMER_SCOPE("incoming_byte_stream_destroy", 0); - transport_->combiner->Run( - GRPC_CLOSURE_INIT(&destroy_action_, - &Chttp2IncomingByteStream::OrphanLocked, this, nullptr), - GRPC_ERROR_NONE); -} - -void Chttp2IncomingByteStream::NextLocked(void* arg, - grpc_error_handle /*error_ignored*/) { - Chttp2IncomingByteStream* bs = static_cast(arg); - grpc_chttp2_transport* t = bs->transport_; - grpc_chttp2_stream* s = bs->stream_; - size_t cur_length = s->frame_storage.length; - if (!s->read_closed) { - s->flow_control->IncomingByteStreamUpdate(bs->next_action_.max_size_hint, - cur_length); - grpc_chttp2_act_on_flowctl_action(s->flow_control->MakeAction(), t, s); - } - GPR_ASSERT(s->unprocessed_incoming_frames_buffer.length == 0); - if (s->frame_storage.length > 0) { - grpc_slice_buffer_swap(&s->frame_storage, - &s->unprocessed_incoming_frames_buffer); - ExecCtx::Run(DEBUG_LOCATION, bs->next_action_.on_complete, GRPC_ERROR_NONE); - } else if (s->byte_stream_error != GRPC_ERROR_NONE) { - ExecCtx::Run(DEBUG_LOCATION, bs->next_action_.on_complete, - GRPC_ERROR_REF(s->byte_stream_error)); - if (s->data_parser.parsing_frame != nullptr) { - s->data_parser.parsing_frame->Unref(); - s->data_parser.parsing_frame = nullptr; - } - } else if (s->read_closed) { - if (bs->remaining_bytes_ != 0) { - s->byte_stream_error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - "Truncated message", &s->read_closed_error, 1); - ExecCtx::Run(DEBUG_LOCATION, bs->next_action_.on_complete, - GRPC_ERROR_REF(s->byte_stream_error)); - if (s->data_parser.parsing_frame != nullptr) { - s->data_parser.parsing_frame->Unref(); - s->data_parser.parsing_frame = nullptr; - } - } else { - // Should never reach here. - GPR_ASSERT(false); - } - } else { - s->on_next = bs->next_action_.on_complete; - } - bs->Unref(); -} - -bool Chttp2IncomingByteStream::Next(size_t max_size_hint, - grpc_closure* on_complete) { - GPR_TIMER_SCOPE("incoming_byte_stream_next", 0); - if (stream_->unprocessed_incoming_frames_buffer.length > 0) { - return true; - } else { - Ref(); - next_action_.max_size_hint = max_size_hint; - next_action_.on_complete = on_complete; - transport_->combiner->Run( - GRPC_CLOSURE_INIT(&next_action_.closure, - &Chttp2IncomingByteStream::NextLocked, this, nullptr), - GRPC_ERROR_NONE); - return false; - } -} - -grpc_error_handle Chttp2IncomingByteStream::Pull(grpc_slice* slice) { - GPR_TIMER_SCOPE("incoming_byte_stream_pull", 0); - grpc_error_handle error; - if (stream_->unprocessed_incoming_frames_buffer.length > 0) { - error = grpc_deframe_unprocessed_incoming_frames( - &stream_->data_parser, stream_, - &stream_->unprocessed_incoming_frames_buffer, slice, nullptr); - if (error != GRPC_ERROR_NONE) { - return error; - } - } else { - error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message"); - stream_->t->combiner->Run(&stream_->reset_byte_stream, - GRPC_ERROR_REF(error)); - return error; - } - return GRPC_ERROR_NONE; -} - -void Chttp2IncomingByteStream::PublishError(grpc_error_handle error) { - GPR_ASSERT(error != GRPC_ERROR_NONE); - ExecCtx::Run(DEBUG_LOCATION, stream_->on_next, GRPC_ERROR_REF(error)); - stream_->on_next = nullptr; - GRPC_ERROR_UNREF(stream_->byte_stream_error); - stream_->byte_stream_error = GRPC_ERROR_REF(error); - grpc_chttp2_cancel_stream(transport_, stream_, GRPC_ERROR_REF(error)); -} - -grpc_error_handle Chttp2IncomingByteStream::Push(const grpc_slice& slice, - grpc_slice* slice_out) { - if (remaining_bytes_ < GRPC_SLICE_LENGTH(slice)) { - grpc_error_handle error = - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Too many bytes in stream"); - transport_->combiner->Run(&stream_->reset_byte_stream, - GRPC_ERROR_REF(error)); - grpc_slice_unref_internal(slice); - return error; - } else { - remaining_bytes_ -= static_cast GRPC_SLICE_LENGTH(slice); - if (slice_out != nullptr) { - *slice_out = slice; - } - return GRPC_ERROR_NONE; - } -} - -grpc_error_handle Chttp2IncomingByteStream::Finished(grpc_error_handle error, - bool reset_on_error) { - if (error == GRPC_ERROR_NONE) { - if (remaining_bytes_ != 0) { - error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message"); - } - } - if (error != GRPC_ERROR_NONE && reset_on_error) { - transport_->combiner->Run(&stream_->reset_byte_stream, - GRPC_ERROR_REF(error)); - } - Unref(); - return error; -} - -void Chttp2IncomingByteStream::Shutdown(grpc_error_handle error) { - GRPC_ERROR_UNREF(Finished(error, true /* reset_on_error */)); -} - -} // namespace grpc_core - // // RESOURCE QUOTAS // diff --git a/src/core/ext/transport/chttp2/transport/flow_control.cc b/src/core/ext/transport/chttp2/transport/flow_control.cc index 54aad55a994..2a0217c910b 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.cc +++ b/src/core/ext/transport/chttp2/transport/flow_control.cc @@ -23,18 +23,18 @@ #include #include +#include #include +#include #include +#include +#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" -#include -#include #include -#include "src/core/ext/transport/chttp2/transport/frame.h" -#include "src/core/ext/transport/chttp2/transport/internal.h" -#include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/resource_quota/memory_quota.h" @@ -47,144 +47,63 @@ namespace chttp2 { TestOnlyTransportTargetWindowEstimatesMocker* g_test_only_transport_target_window_estimates_mocker; -bool g_test_only_transport_flow_control_window_check; - namespace { -constexpr const int kTracePadding = 30; constexpr const int64_t kMaxWindowUpdateSize = (1u << 31) - 1; -char* fmt_int64_diff_str(int64_t old_val, int64_t new_val) { - std::string str; - if (old_val != new_val) { - str = absl::StrFormat("%" PRId64 " -> %" PRId64 "", old_val, new_val); - } else { - str = absl::StrFormat("%" PRId64 "", old_val); - } - return gpr_leftpad(str.c_str(), ' ', kTracePadding); -} - -char* fmt_uint32_diff_str(uint32_t old_val, uint32_t new_val) { - std::string str; - if (old_val != new_val) { - str = absl::StrFormat("%" PRIu32 " -> %" PRIu32 "", old_val, new_val); - } else { - str = absl::StrFormat("%" PRIu32 "", old_val); - } - return gpr_leftpad(str.c_str(), ' ', kTracePadding); -} } // namespace -void FlowControlTrace::Init(const char* reason, TransportFlowControl* tfc, - StreamFlowControl* sfc) { - tfc_ = tfc; - sfc_ = sfc; - reason_ = reason; - remote_window_ = tfc->remote_window(); - target_window_ = tfc->target_window(); - announced_window_ = tfc->announced_window(); - if (sfc != nullptr) { - remote_window_delta_ = sfc->remote_window_delta(); - local_window_delta_ = sfc->local_window_delta(); - announced_window_delta_ = sfc->announced_window_delta(); - } -} - -void FlowControlTrace::Finish() { - uint32_t acked_local_window = - tfc_->transport()->settings[GRPC_SENT_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - uint32_t remote_window = - tfc_->transport()->settings[GRPC_PEER_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - char* trw_str = fmt_int64_diff_str(remote_window_, tfc_->remote_window()); - char* tlw_str = fmt_int64_diff_str(target_window_, tfc_->target_window()); - char* taw_str = - fmt_int64_diff_str(announced_window_, tfc_->announced_window()); - char* srw_str; - char* slw_str; - char* saw_str; - if (sfc_ != nullptr) { - srw_str = fmt_int64_diff_str(remote_window_delta_ + remote_window, - sfc_->remote_window_delta() + remote_window); - slw_str = - fmt_int64_diff_str(local_window_delta_ + acked_local_window, - sfc_->local_window_delta() + acked_local_window); - saw_str = - fmt_int64_diff_str(announced_window_delta_ + acked_local_window, - sfc_->announced_window_delta() + acked_local_window); - } else { - srw_str = gpr_leftpad("", ' ', kTracePadding); - slw_str = gpr_leftpad("", ' ', kTracePadding); - saw_str = gpr_leftpad("", ' ', kTracePadding); - } - gpr_log(GPR_DEBUG, - "%p[%u][%s] | %s | trw:%s, tlw:%s, taw:%s, srw:%s, slw:%s, saw:%s", - tfc_, sfc_ != nullptr ? sfc_->stream()->id : 0, - tfc_->transport()->is_client ? "cli" : "svr", reason_, trw_str, - tlw_str, taw_str, srw_str, slw_str, saw_str); - gpr_free(trw_str); - gpr_free(tlw_str); - gpr_free(taw_str); - gpr_free(srw_str); - gpr_free(slw_str); - gpr_free(saw_str); -} - const char* FlowControlAction::UrgencyString(Urgency u) { switch (u) { case Urgency::NO_ACTION_NEEDED: - return "no action"; + return "no-action"; case Urgency::UPDATE_IMMEDIATELY: - return "update immediately"; + return "now"; case Urgency::QUEUE_UPDATE: - return "queue update"; + return "queue"; default: GPR_UNREACHABLE_CODE(return "unknown"); } GPR_UNREACHABLE_CODE(return "unknown"); } -void FlowControlAction::Trace(grpc_chttp2_transport* t) const { - char* iw_str = fmt_uint32_diff_str( - t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], - initial_window_size_); - char* mf_str = fmt_uint32_diff_str( - t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], - max_frame_size_); - gpr_log(GPR_DEBUG, "t[%s], s[%s], iw:%s:%s mf:%s:%s", - UrgencyString(send_transport_update_), - UrgencyString(send_stream_update_), - UrgencyString(send_initial_window_update_), iw_str, - UrgencyString(send_max_frame_size_update_), mf_str); - gpr_free(iw_str); - gpr_free(mf_str); +std::ostream& operator<<(std::ostream& out, FlowControlAction::Urgency u) { + return out << FlowControlAction::UrgencyString(u); } -TransportFlowControlDisabled::TransportFlowControlDisabled( - grpc_chttp2_transport* t) { - remote_window_ = kMaxWindow; - target_initial_window_size_ = kMaxWindow; - announced_window_ = kMaxWindow; - t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] = - kFrameSize; - t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] = - kFrameSize; - t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] = - kFrameSize; - t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] = - kMaxWindow; - t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] = - kMaxWindow; - t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] = - kMaxWindow; +std::string FlowControlAction::DebugString() const { + std::vector segments; + if (send_transport_update_ != Urgency::NO_ACTION_NEEDED) { + segments.push_back( + absl::StrCat("t:", UrgencyString(send_transport_update_))); + } + if (send_stream_update_ != Urgency::NO_ACTION_NEEDED) { + segments.push_back(absl::StrCat("s:", UrgencyString(send_stream_update_))); + } + if (send_initial_window_update_ != Urgency::NO_ACTION_NEEDED) { + segments.push_back( + absl::StrCat("iw=", initial_window_size_, ":", + UrgencyString(send_initial_window_update_))); + } + if (send_max_frame_size_update_ != Urgency::NO_ACTION_NEEDED) { + segments.push_back( + absl::StrCat("mf=", max_frame_size_, ":", + UrgencyString(send_max_frame_size_update_))); + } + if (segments.empty()) return "no action"; + return absl::StrJoin(segments, ","); } -TransportFlowControl::TransportFlowControl(const grpc_chttp2_transport* t, - bool enable_bdp_probe) - : t_(t), +std::ostream& operator<<(std::ostream& out, const FlowControlAction& action) { + return out << action.DebugString(); +} + +TransportFlowControl::TransportFlowControl(const char* name, + bool enable_bdp_probe, + MemoryOwner* memory_owner) + : memory_owner_(memory_owner), enable_bdp_probe_(enable_bdp_probe), - bdp_estimator_(t->peer_string.c_str()), + bdp_estimator_(name), pid_controller_(PidController::Args() .set_gain_p(4) .set_gain_i(8) @@ -196,7 +115,6 @@ TransportFlowControl::TransportFlowControl(const grpc_chttp2_transport* t, last_pid_update_(ExecCtx::Get()->Now()) {} uint32_t TransportFlowControl::MaybeSendUpdate(bool writing_anyway) { - FlowControlTrace trace("t updt sent", this, nullptr); const uint32_t target_announced_window = static_cast(target_window()); if ((writing_anyway || announced_window_ <= target_announced_window / 2) && @@ -210,121 +128,55 @@ uint32_t TransportFlowControl::MaybeSendUpdate(bool writing_anyway) { return 0; } -grpc_error_handle TransportFlowControl::ValidateRecvData( - int64_t incoming_frame_size) { - if (incoming_frame_size > announced_window_) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( - "frame of size %" PRId64 " overflows local window of %" PRId64, - incoming_frame_size, announced_window_)); - } - return GRPC_ERROR_NONE; -} +StreamFlowControl::StreamFlowControl(TransportFlowControl* tfc) : tfc_(tfc) {} -StreamFlowControl::StreamFlowControl(TransportFlowControl* tfc, - const grpc_chttp2_stream* s) - : tfc_(tfc), s_(s) {} - -grpc_error_handle StreamFlowControl::RecvData(int64_t incoming_frame_size) { - FlowControlTrace trace(" data recv", tfc_, this); - - grpc_error_handle error = GRPC_ERROR_NONE; - error = tfc_->ValidateRecvData(incoming_frame_size); - if (error != GRPC_ERROR_NONE) return error; - - uint32_t sent_init_window = - tfc_->transport()->settings[GRPC_SENT_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - uint32_t acked_init_window = - tfc_->transport()->settings[GRPC_ACKED_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - - int64_t acked_stream_window = announced_window_delta_ + acked_init_window; - int64_t sent_stream_window = announced_window_delta_ + sent_init_window; - if (incoming_frame_size > acked_stream_window) { - if (incoming_frame_size <= sent_stream_window) { - gpr_log(GPR_ERROR, - "Incoming frame of size %" PRId64 - " exceeds local window size of %" PRId64 - ".\n" - "The (un-acked, future) window size would be %" PRId64 - " which is not exceeded.\n" - "This would usually cause a disconnection, but allowing it due to" - "broken HTTP2 implementations in the wild.\n" - "See (for example) https://github.com/netty/netty/issues/6520.", - incoming_frame_size, acked_stream_window, sent_stream_window); - } else { - return GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( +absl::Status StreamFlowControl::IncomingUpdateContext::RecvData( + int64_t incoming_frame_size) { + return tfc_upd_.RecvData(incoming_frame_size, [this, incoming_frame_size]() { + int64_t acked_stream_window = + sfc_->announced_window_delta_ + sfc_->tfc_->acked_init_window(); + if (incoming_frame_size > acked_stream_window) { + return absl::InternalError(absl::StrFormat( "frame of size %" PRId64 " overflows local window of %" PRId64, incoming_frame_size, acked_stream_window)); } - } - UpdateAnnouncedWindowDelta(tfc_, -incoming_frame_size); - local_window_delta_ -= incoming_frame_size; - tfc_->CommitRecvData(incoming_frame_size); - return GRPC_ERROR_NONE; + tfc_upd_.UpdateAnnouncedWindowDelta(&sfc_->announced_window_delta_, + -incoming_frame_size); + sfc_->min_progress_size_ -= + std::min(sfc_->min_progress_size_, incoming_frame_size); + return absl::OkStatus(); + }); } -uint32_t StreamFlowControl::MaybeSendUpdate() { - FlowControlTrace trace("s updt sent", tfc_, this); - // If a recently sent settings frame caused the stream's flow control window - // to go in the negative (or < GRPC_HEADER_SIZE_IN_BYTES), update the delta if - // one of the following conditions is satisfied - - // 1) There is a pending byte_stream and higher layers have expressed interest - // in reading additional data through the invokation of `Next()` where the - // bytes are to be available asynchronously. 2) There is a pending - // recv_message op. - // In these cases, we want to make sure that bytes are still flowing. - if (local_window_delta_ < GRPC_HEADER_SIZE_IN_BYTES) { - if (s_->on_next != nullptr) { - GPR_DEBUG_ASSERT(s_->pending_byte_stream); - IncomingByteStreamUpdate(GRPC_HEADER_SIZE_IN_BYTES, 0); - } else if (s_->recv_message != nullptr) { - IncomingByteStreamUpdate(GRPC_HEADER_SIZE_IN_BYTES, - s_->frame_storage.length); - } - } - if (local_window_delta_ > announced_window_delta_) { - uint32_t announce = static_cast( - Clamp(local_window_delta_ - announced_window_delta_, int64_t(0), - kMaxWindowUpdateSize)); - UpdateAnnouncedWindowDelta(tfc_, announce); - return announce; +absl::Status TransportFlowControl::IncomingUpdateContext::RecvData( + int64_t incoming_frame_size, absl::FunctionRef stream) { + if (incoming_frame_size > tfc_->announced_window_) { + return absl::InternalError(absl::StrFormat( + "frame of size %" PRId64 " overflows local window of %" PRId64, + incoming_frame_size, tfc_->announced_window_)); } - return 0; + absl::Status error = stream(); + if (!error.ok()) return error; + tfc_->announced_window_ -= incoming_frame_size; + return absl::OkStatus(); } -void StreamFlowControl::IncomingByteStreamUpdate(size_t max_size_hint, - size_t have_already) { - FlowControlTrace trace("app st recv", tfc_, this); - uint32_t max_recv_bytes; - - /* clamp max recv hint to an allowable size */ - if (max_size_hint >= kMaxWindowDelta) { - max_recv_bytes = kMaxWindowDelta; - } else { - max_recv_bytes = static_cast(max_size_hint); - } - - /* account for bytes already received but unknown to higher layers */ - if (max_recv_bytes >= have_already) { - max_recv_bytes -= static_cast(have_already); - } else { - max_recv_bytes = 0; - } +int64_t TransportFlowControl::target_window() const { + // See comment above announced_stream_total_over_incoming_window_ for the + // logic behind this decision. + return static_cast( + std::min(static_cast((1u << 31) - 1), + announced_stream_total_over_incoming_window_ + + target_initial_window_size_)); +} - /* add some small lookahead to keep pipelines flowing */ - GPR_DEBUG_ASSERT( - max_recv_bytes <= - kMaxWindowUpdateSize - - tfc_->transport() - ->settings[GRPC_SENT_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); - if (local_window_delta_ < max_recv_bytes) { - uint32_t add_max_recv_bytes = - static_cast(max_recv_bytes - local_window_delta_); - local_window_delta_ += add_max_recv_bytes; +FlowControlAction TransportFlowControl::UpdateAction(FlowControlAction action) { + if (announced_window_ < target_window() / 2) { + action.set_send_transport_update( + FlowControlAction::Urgency::UPDATE_IMMEDIATELY); } + return action; } // Take in a target and modifies it based on the memory pressure of the system @@ -345,10 +197,9 @@ static double AdjustForMemoryPressure(double memory_pressure, double target) { } double TransportFlowControl::TargetLogBdp() { - return AdjustForMemoryPressure(t_->memory_owner.is_valid() - ? t_->memory_owner.InstantaneousPressure() - : 0.0, - 1 + log2(bdp_estimator_.EstimateBdp())); + return AdjustForMemoryPressure( + memory_owner_->is_valid() ? memory_owner_->InstantaneousPressure() : 0.0, + 1 + log2(bdp_estimator_.EstimateBdp())); } double TransportFlowControl::SmoothLogBdp(double value) { @@ -361,15 +212,17 @@ double TransportFlowControl::SmoothLogBdp(double value) { return pid_controller_.Update(bdp_error, dt > kMaxDt ? kMaxDt : dt); } -FlowControlAction::Urgency TransportFlowControl::DeltaUrgency( - int64_t value, grpc_chttp2_setting_id setting_id) { - int64_t delta = value - static_cast( - t_->settings[GRPC_LOCAL_SETTINGS][setting_id]); +void TransportFlowControl::UpdateSetting( + int64_t* desired_value, int64_t new_desired_value, + FlowControlAction* action, + FlowControlAction& (FlowControlAction::*set)(FlowControlAction::Urgency, + uint32_t)) { + int64_t delta = new_desired_value - *desired_value; // TODO(ncteisen): tune this - if (delta != 0 && (delta <= -value / 5 || delta >= value / 5)) { - return FlowControlAction::Urgency::QUEUE_UPDATE; - } else { - return FlowControlAction::Urgency::NO_ACTION_NEEDED; + if (delta != 0 && + (delta <= -*desired_value / 5 || delta >= *desired_value / 5)) { + *desired_value = new_desired_value; + (action->*set)(FlowControlAction::Urgency::QUEUE_UPDATE, *desired_value); } } @@ -389,46 +242,73 @@ FlowControlAction TransportFlowControl::PeriodicUpdate() { } // Though initial window 'could' drop to 0, we keep the floor at // kMinInitialWindowSize - target_initial_window_size_ = static_cast(Clamp( - target, double(kMinInitialWindowSize), double(kMaxInitialWindowSize))); - action.set_send_initial_window_update( - DeltaUrgency(target_initial_window_size_, - GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE), - static_cast(target_initial_window_size_)); + UpdateSetting( + &target_initial_window_size_, + static_cast(Clamp(target, double(kMinInitialWindowSize), + double(kMaxInitialWindowSize))), + &action, &FlowControlAction::set_send_initial_window_update); // get bandwidth estimate and update max_frame accordingly. double bw_dbl = bdp_estimator_.EstimateBandwidth(); // we target the max of BDP or bandwidth in microseconds. - int32_t frame_size = static_cast(Clamp( - std::max( - static_cast(Clamp(bw_dbl, 0.0, double(INT_MAX))) / 1000, - static_cast(target_initial_window_size_)), - 16384, 16777215)); - action.set_send_max_frame_size_update( - DeltaUrgency(static_cast(frame_size), - GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE), - frame_size); + UpdateSetting( + &target_frame_size_, + static_cast(Clamp( + std::max(static_cast(Clamp(bw_dbl, 0.0, double(INT_MAX))) / + 1000, + static_cast(target_initial_window_size_)), + 16384, 16777215)), + &action, &FlowControlAction::set_send_max_frame_size_update); } return UpdateAction(action); } +uint32_t StreamFlowControl::MaybeSendUpdate() { + TransportFlowControl::IncomingUpdateContext tfc_upd(tfc_); + const uint32_t announce = DesiredAnnounceSize(); + pending_size_ = absl::nullopt; + tfc_upd.UpdateAnnouncedWindowDelta(&announced_window_delta_, announce); + GPR_ASSERT(DesiredAnnounceSize() == 0); + tfc_upd.MakeAction(); + return announce; +} + +uint32_t StreamFlowControl::DesiredAnnounceSize() const { + int64_t desired_window_delta = [this]() { + if (min_progress_size_ == 0) { + if (pending_size_.has_value() && + announced_window_delta_ < -*pending_size_) { + return -*pending_size_; + } else { + return announced_window_delta_; + } + } else { + return std::min(min_progress_size_, kMaxWindowDelta); + } + }(); + return Clamp(desired_window_delta - announced_window_delta_, int64_t(0), + kMaxWindowUpdateSize); +} + FlowControlAction StreamFlowControl::UpdateAction(FlowControlAction action) { - // TODO(ncteisen): tune this - if (!s_->read_closed) { - uint32_t sent_init_window = - tfc_->transport()->settings[GRPC_SENT_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - if (local_window_delta_ > announced_window_delta_ && - announced_window_delta_ + sent_init_window <= sent_init_window / 2) { + const int64_t desired_announce_size = DesiredAnnounceSize(); + if (desired_announce_size > 0) { + if ((min_progress_size_ > 0 && announced_window_delta_ < 0) || + desired_announce_size >= 8192) { action.set_send_stream_update( FlowControlAction::Urgency::UPDATE_IMMEDIATELY); - } else if (local_window_delta_ > announced_window_delta_) { + } else { action.set_send_stream_update(FlowControlAction::Urgency::QUEUE_UPDATE); } } - return action; } +void StreamFlowControl::IncomingUpdateContext::SetPendingSize( + int64_t pending_size) { + GPR_ASSERT(pending_size >= 0); + sfc_->pending_size_ = pending_size; +} + } // namespace chttp2 } // namespace grpc_core diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h index c50aa5f1e23..723c9551008 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.h +++ b/src/core/ext/transport/chttp2/transport/flow_control.h @@ -22,22 +22,23 @@ #include #include -#include -#include +#include +#include +#include "absl/functional/function_ref.h" #include "absl/status/status.h" +#include "absl/types/optional.h" +#include "absl/utility/utility.h" + +#include -#include "src/core/ext/transport/chttp2/transport/http2_settings.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/time.h" -#include "src/core/lib/iomgr/error.h" +#include "src/core/lib/resource_quota/memory_quota.h" #include "src/core/lib/transport/bdp_estimator.h" #include "src/core/lib/transport/pid_controller.h" -struct grpc_chttp2_transport; -struct grpc_chttp2_stream; - extern grpc_core::TraceFlag grpc_flowctl_trace; namespace grpc { @@ -50,18 +51,19 @@ namespace grpc_core { namespace chttp2 { static constexpr uint32_t kDefaultWindow = 65535; +static constexpr uint32_t kDefaultFrameSize = 16384; static constexpr int64_t kMaxWindow = static_cast((1u << 31) - 1); // TODO(ncteisen): Tune this static constexpr uint32_t kFrameSize = 1024 * 1024; static constexpr const uint32_t kMinInitialWindowSize = 128; static constexpr const uint32_t kMaxInitialWindowSize = (1u << 30); // The maximum per-stream flow control window delta to advertise. -static constexpr const uint32_t kMaxWindowDelta = (1u << 20); +static constexpr const int64_t kMaxWindowDelta = (1u << 20); class TransportFlowControl; class StreamFlowControl; -extern bool g_test_only_transport_flow_control_window_check; +enum class StallEdge { kNoChange, kStalled, kUnstalled }; // Encapsulates a collections of actions the transport needs to take with // regard to flow control. Each action comes with urgencies that tell the @@ -111,7 +113,20 @@ class FlowControlAction { } static const char* UrgencyString(Urgency u); - void Trace(grpc_chttp2_transport* t) const; + std::string DebugString() const; + + void AssertEmpty() { GPR_ASSERT(*this == FlowControlAction()); } + + bool operator==(const FlowControlAction& other) const { + return send_stream_update_ == other.send_stream_update_ && + send_transport_update_ == other.send_transport_update_ && + send_initial_window_update_ == other.send_initial_window_update_ && + send_max_frame_size_update_ == other.send_max_frame_size_update_ && + (send_initial_window_update_ == Urgency::NO_ACTION_NEEDED || + initial_window_size_ == other.initial_window_size_) && + (send_max_frame_size_update_ == Urgency::NO_ACTION_NEEDED || + max_frame_size_ == other.max_frame_size_); + } private: Urgency send_stream_update_ = Urgency::NO_ACTION_NEEDED; @@ -122,120 +137,16 @@ class FlowControlAction { uint32_t max_frame_size_ = 0; }; -class FlowControlTrace { - public: - FlowControlTrace(const char* reason, TransportFlowControl* tfc, - StreamFlowControl* sfc) { - if (enabled_) Init(reason, tfc, sfc); - } - - ~FlowControlTrace() { - if (enabled_) Finish(); - } - - private: - void Init(const char* reason, TransportFlowControl* tfc, - StreamFlowControl* sfc); - void Finish(); - - const bool enabled_ = GRPC_TRACE_FLAG_ENABLED(grpc_flowctl_trace); - - TransportFlowControl* tfc_; - StreamFlowControl* sfc_; - const char* reason_; - int64_t remote_window_; - int64_t target_window_; - int64_t announced_window_; - int64_t remote_window_delta_; - int64_t local_window_delta_; - int64_t announced_window_delta_; -}; - -// Fat interface with all methods a flow control implementation needs to -// support. -class TransportFlowControlBase { - public: - TransportFlowControlBase() {} - virtual ~TransportFlowControlBase() {} - - // Is flow control enabled? This is needed in other codepaths like the checks - // in parsing and in writing. - virtual bool flow_control_enabled() const = 0; - - // Called to check if the transport needs to send a WINDOW_UPDATE frame - virtual uint32_t MaybeSendUpdate(bool /* writing_anyway */) = 0; - - // Using the protected members, returns and Action to be taken by the - // tranport. - virtual FlowControlAction MakeAction() = 0; - - // Using the protected members, returns and Action to be taken by the - // tranport. Also checks for updates to our BDP estimate and acts - // accordingly. - virtual FlowControlAction PeriodicUpdate() = 0; - - // Called to do bookkeeping when a stream owned by this transport sends - // data on the wire - virtual void StreamSentData(int64_t /* size */) = 0; - - // Called to do bookkeeping when a stream owned by this transport receives - // data from the wire. Also does error checking for frame size. - virtual grpc_error_handle RecvData(int64_t /* incoming_frame_size */) = 0; - - // Called to do bookkeeping when we receive a WINDOW_UPDATE frame. - virtual void RecvUpdate(uint32_t /* size */) = 0; - - // Returns the BdpEstimator held by this object. Caller is responsible for - // checking for nullptr. TODO(ncteisen): consider fully encapsulating all - // bdp estimator actions inside TransportFlowControl - virtual BdpEstimator* bdp_estimator() { return nullptr; } - - // Getters - int64_t remote_window() const { return remote_window_; } - virtual int64_t target_window() const { return target_initial_window_size_; } - int64_t announced_window() const { return announced_window_; } - - // Used in certain benchmarks in which we don't want FlowControl to be a - // factor - virtual void TestOnlyForceHugeWindow() {} - - protected: - friend class grpc::testing::TrickledCHTTP2; - int64_t remote_window_ = kDefaultWindow; - int64_t target_initial_window_size_ = kDefaultWindow; - int64_t announced_window_ = kDefaultWindow; -}; - -// Implementation of flow control that does NOTHING. Always returns maximum -// values, never initiates writes, and assumes that the remote peer is doing -// the same. To be used to narrow down on flow control as the cause of negative -// performance. -class TransportFlowControlDisabled final : public TransportFlowControlBase { - public: - // Maxes out all values - explicit TransportFlowControlDisabled(grpc_chttp2_transport* t); - - bool flow_control_enabled() const override { return false; } - - // Never do anything. - uint32_t MaybeSendUpdate(bool /* writing_anyway */) override { return 0; } - FlowControlAction MakeAction() override { return FlowControlAction(); } - FlowControlAction PeriodicUpdate() override { return FlowControlAction(); } - void StreamSentData(int64_t /* size */) override {} - grpc_error_handle RecvData(int64_t /* incoming_frame_size */) override { - return GRPC_ERROR_NONE; - } - void RecvUpdate(uint32_t /* size */) override {} -}; +std::ostream& operator<<(std::ostream& out, FlowControlAction::Urgency urgency); +std::ostream& operator<<(std::ostream& out, const FlowControlAction& action); // Implementation of flow control that abides to HTTP/2 spec and attempts // to be as performant as possible. -class TransportFlowControl final : public TransportFlowControlBase { +class TransportFlowControl final { public: - TransportFlowControl(const grpc_chttp2_transport* t, bool enable_bdp_probe); - ~TransportFlowControl() override {} - - bool flow_control_enabled() const override { return true; } + explicit TransportFlowControl(const char* name, bool enable_bdp_probe, + MemoryOwner* memory_owner); + ~TransportFlowControl() {} bool bdp_probe() const { return enable_bdp_probe_; } @@ -243,85 +154,112 @@ class TransportFlowControl final : public TransportFlowControlBase { // else returns zero; writing_anyway indicates if a write would happen // regardless of the send - if it is false and this function returns non-zero, // this announce will cause a write to occur - uint32_t MaybeSendUpdate(bool writing_anyway) override; + uint32_t MaybeSendUpdate(bool writing_anyway); + + // Track an update to the incoming flow control counters - that is how many + // tokens we report to our peer that we're willing to accept. + // Instantiators *must* call MakeAction before destruction of this value. + class IncomingUpdateContext { + public: + explicit IncomingUpdateContext(TransportFlowControl* tfc) : tfc_(tfc) {} + ~IncomingUpdateContext() { GPR_ASSERT(tfc_ == nullptr); } + + IncomingUpdateContext(const IncomingUpdateContext&) = delete; + IncomingUpdateContext& operator=(const IncomingUpdateContext&) = delete; + + // Reads the flow control data and returns an actionable struct that will + // tell chttp2 exactly what it needs to do + FlowControlAction MakeAction() { + return absl::exchange(tfc_, nullptr)->UpdateAction(FlowControlAction()); + } - // Reads the flow control data and returns and actionable struct that will - // tell chttp2 exactly what it needs to do - FlowControlAction MakeAction() override { - return UpdateAction(FlowControlAction()); - } + // Notify of data receipt. Returns OkStatus if the data was accepted, + // else an error status if the connection should be closed. + absl::Status RecvData( + int64_t incoming_frame_size, absl::FunctionRef stream = + []() { return absl::OkStatus(); }); + + // Update a stream announce window delta, keeping track of how much total + // positive delta is present on the transport. + void UpdateAnnouncedWindowDelta(int64_t* delta, int64_t change) { + if (change == 0) return; + if (*delta > 0) { + tfc_->announced_stream_total_over_incoming_window_ -= *delta; + } + *delta += change; + if (*delta > 0) { + tfc_->announced_stream_total_over_incoming_window_ += *delta; + } + } + + private: + TransportFlowControl* tfc_; + }; + + // Track an update to the outgoing flow control counters - that is how many + // tokens our peer has said we can send. + class OutgoingUpdateContext { + public: + explicit OutgoingUpdateContext(TransportFlowControl* tfc) : tfc_(tfc) {} + void StreamSentData(int64_t size) { tfc_->remote_window_ -= size; } + + // we have received a WINDOW_UPDATE frame for a transport + void RecvUpdate(uint32_t size) { tfc_->remote_window_ += size; } + + // Finish the update and check whether we became stalled or unstalled. + StallEdge Finish() { + bool is_stalled = tfc_->remote_window_ <= 0; + if (is_stalled != was_stalled_) { + return is_stalled ? StallEdge::kStalled : StallEdge::kUnstalled; + } else { + return StallEdge::kNoChange; + } + } + + private: + TransportFlowControl* tfc_; + const bool was_stalled_ = tfc_->remote_window_ <= 0; + }; // Call periodically (at a low-ish rate, 100ms - 10s makes sense) // to perform more complex flow control calculations and return an action // to let chttp2 change its parameters - FlowControlAction PeriodicUpdate() override; + FlowControlAction PeriodicUpdate(); - void StreamSentData(int64_t size) override { remote_window_ -= size; } + int64_t target_window() const; + int64_t target_frame_size() const { return target_frame_size_; } - grpc_error_handle ValidateRecvData(int64_t incoming_frame_size); - void CommitRecvData(int64_t incoming_frame_size) { - announced_window_ -= incoming_frame_size; - } + BdpEstimator* bdp_estimator() { return &bdp_estimator_; } - grpc_error_handle RecvData(int64_t incoming_frame_size) override { - FlowControlTrace trace(" data recv", this, nullptr); - grpc_error_handle error = ValidateRecvData(incoming_frame_size); - if (error != GRPC_ERROR_NONE) return error; - CommitRecvData(incoming_frame_size); - return GRPC_ERROR_NONE; - } + uint32_t acked_init_window() const { return acked_init_window_; } - // we have received a WINDOW_UPDATE frame for a transport - void RecvUpdate(uint32_t size) override { - FlowControlTrace trace("t updt recv", this, nullptr); - remote_window_ += size; - } + void SetAckedInitialWindow(uint32_t value) { acked_init_window_ = value; } - // See comment above announced_stream_total_over_incoming_window_ for the - // logic behind this decision. - int64_t target_window() const override { - return static_cast( - std::min(static_cast((1u << 31) - 1), - announced_stream_total_over_incoming_window_ + - target_initial_window_size_)); - } - - const grpc_chttp2_transport* transport() const { return t_; } + // Getters + int64_t remote_window() const { return remote_window_; } + int64_t announced_window() const { return announced_window_; } - void PreUpdateAnnouncedWindowOverIncomingWindow(int64_t delta) { - if (delta > 0) { - announced_stream_total_over_incoming_window_ -= delta; - } + int64_t announced_stream_total_over_incoming_window() const { + return announced_stream_total_over_incoming_window_; } - void PostUpdateAnnouncedWindowOverIncomingWindow(int64_t delta) { + void RemoveAnnouncedWindowDelta(int64_t delta) { if (delta > 0) { - announced_stream_total_over_incoming_window_ += delta; + announced_stream_total_over_incoming_window_ -= delta; } } - BdpEstimator* bdp_estimator() override { return &bdp_estimator_; } - - void TestOnlyForceHugeWindow() override { - announced_window_ = 1024 * 1024 * 1024; - remote_window_ = 1024 * 1024 * 1024; - } - private: double TargetLogBdp(); double SmoothLogBdp(double value); - FlowControlAction::Urgency DeltaUrgency(int64_t value, - grpc_chttp2_setting_id setting_id); + static void UpdateSetting(int64_t* desired_value, int64_t new_desired_value, + FlowControlAction* action, + FlowControlAction& (FlowControlAction::*set)( + FlowControlAction::Urgency, uint32_t)); - FlowControlAction UpdateAction(FlowControlAction action) { - if (announced_window_ < target_window() / 2) { - action.set_send_transport_update( - FlowControlAction::Urgency::UPDATE_IMMEDIATELY); - } - return action; - } + FlowControlAction UpdateAction(FlowControlAction action); - const grpc_chttp2_transport* const t_; + MemoryOwner* const memory_owner_; /** calculating what we should give for local window: we track the total amount of flow control over initial window size @@ -342,140 +280,87 @@ class TransportFlowControl final : public TransportFlowControlBase { /* pid controller */ PidController pid_controller_; Timestamp last_pid_update_; + + int64_t remote_window_ = kDefaultWindow; + int64_t target_initial_window_size_ = kDefaultWindow; + int64_t target_frame_size_ = kDefaultFrameSize; + int64_t announced_window_ = kDefaultWindow; + uint32_t acked_init_window_ = kDefaultWindow; }; -// Fat interface with all methods a stream flow control implementation needs -// to support. -class StreamFlowControlBase { +// Implementation of flow control that abides to HTTP/2 spec and attempts +// to be as performant as possible. +class StreamFlowControl final { public: - StreamFlowControlBase() {} - virtual ~StreamFlowControlBase() {} - - // Updates an action using the protected members. - virtual FlowControlAction UpdateAction(FlowControlAction /* action */) { - abort(); + explicit StreamFlowControl(TransportFlowControl* tfc); + ~StreamFlowControl() { + tfc_->RemoveAnnouncedWindowDelta(announced_window_delta_); } - // Using the protected members, returns an Action for this stream to be - // taken by the tranport. - virtual FlowControlAction MakeAction() = 0; - - // Bookkeeping for when data is sent on this stream. - virtual void SentData(int64_t /* outgoing_frame_size */) = 0; - - // Bookkeeping and error checking for when data is received by this stream. - virtual grpc_error_handle RecvData(int64_t /* incoming_frame_size */) = 0; + // Track an update to the incoming flow control counters - that is how many + // tokens we report to our peer that we're willing to accept. + // Instantiators *must* call MakeAction before destruction of this value. + class IncomingUpdateContext { + public: + explicit IncomingUpdateContext(StreamFlowControl* sfc) + : tfc_upd_(sfc->tfc_), sfc_(sfc) {} - // Called to check if this stream needs to send a WINDOW_UPDATE frame. - virtual uint32_t MaybeSendUpdate() = 0; - - // Bookkeeping for receiving a WINDOW_UPDATE from for this stream. - virtual void RecvUpdate(uint32_t /* size */) = 0; - - // Bookkeeping for when a call pulls bytes out of the transport. At this - // point we consider the data 'used' and can thus let out peer know we are - // ready for more data. - virtual void IncomingByteStreamUpdate(size_t /* max_size_hint */, - size_t /* have_already */) { - abort(); - } - - // Used in certain benchmarks in which we don't want FlowControl to be a - // factor - virtual void TestOnlyForceHugeWindow() {} - - // Getters - int64_t remote_window_delta() const { return remote_window_delta_; } - int64_t local_window_delta() const { return local_window_delta_; } - int64_t announced_window_delta() const { return announced_window_delta_; } + FlowControlAction MakeAction() { + return sfc_->UpdateAction(tfc_upd_.MakeAction()); + } - protected: - friend class grpc::testing::TrickledCHTTP2; - int64_t remote_window_delta_ = 0; - int64_t local_window_delta_ = 0; - int64_t announced_window_delta_ = 0; -}; + // we have received data from the wire + absl::Status RecvData(int64_t incoming_frame_size); -// Implementation of flow control that does NOTHING. Always returns maximum -// values, never initiates writes, and assumes that the remote peer is doing -// the same. To be used to narrow down on flow control as the cause of negative -// performance. -class StreamFlowControlDisabled : public StreamFlowControlBase { - public: - FlowControlAction UpdateAction(FlowControlAction action) override { - return action; - } - FlowControlAction MakeAction() override { return FlowControlAction(); } - void SentData(int64_t /* outgoing_frame_size */) override {} - grpc_error_handle RecvData(int64_t /* incoming_frame_size */) override { - return GRPC_ERROR_NONE; - } - uint32_t MaybeSendUpdate() override { return 0; } - void RecvUpdate(uint32_t /* size */) override {} - void IncomingByteStreamUpdate(size_t /* max_size_hint */, - size_t /* have_already */) override {} -}; + // the application is asking for a certain amount of bytes + void SetMinProgressSize(uint32_t min_progress_size) { + sfc_->min_progress_size_ = min_progress_size; + } -// Implementation of flow control that abides to HTTP/2 spec and attempts -// to be as performant as possible. -class StreamFlowControl final : public StreamFlowControlBase { - public: - StreamFlowControl(TransportFlowControl* tfc, const grpc_chttp2_stream* s); - ~StreamFlowControl() override { - tfc_->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_); - } + void SetPendingSize(int64_t pending_size); - FlowControlAction UpdateAction(FlowControlAction action) override; - FlowControlAction MakeAction() override { - return UpdateAction(tfc_->MakeAction()); - } + private: + TransportFlowControl::IncomingUpdateContext tfc_upd_; + StreamFlowControl* const sfc_; + }; - // we have sent data on the wire, we must track this in our bookkeeping for - // the remote peer's flow control. - void SentData(int64_t outgoing_frame_size) override { - FlowControlTrace tracer(" data sent", tfc_, this); - tfc_->StreamSentData(outgoing_frame_size); - remote_window_delta_ -= outgoing_frame_size; - } + // Track an update to the outgoing flow control counters - that is how many + // tokens our peer has said we can send. + class OutgoingUpdateContext { + public: + explicit OutgoingUpdateContext(StreamFlowControl* sfc) + : tfc_upd_(sfc->tfc_), sfc_(sfc) {} + // we have received a WINDOW_UPDATE frame for a stream + void RecvUpdate(uint32_t size) { sfc_->remote_window_delta_ += size; } + // we have sent data on the wire, we must track this in our bookkeeping for + // the remote peer's flow control. + void SentData(int64_t outgoing_frame_size) { + tfc_upd_.StreamSentData(outgoing_frame_size); + sfc_->remote_window_delta_ -= outgoing_frame_size; + } - // we have received data from the wire - grpc_error_handle RecvData(int64_t incoming_frame_size) override; + private: + TransportFlowControl::OutgoingUpdateContext tfc_upd_; + StreamFlowControl* const sfc_; + }; // returns an announce if we should send a stream update to our peer, else // returns zero - uint32_t MaybeSendUpdate() override; - - // we have received a WINDOW_UPDATE frame for a stream - void RecvUpdate(uint32_t size) override { - FlowControlTrace trace("s updt recv", tfc_, this); - remote_window_delta_ += size; - } - - // the application is asking for a certain amount of bytes - void IncomingByteStreamUpdate(size_t max_size_hint, - size_t have_already) override; + uint32_t MaybeSendUpdate(); int64_t remote_window_delta() const { return remote_window_delta_; } - int64_t local_window_delta() const { return local_window_delta_; } int64_t announced_window_delta() const { return announced_window_delta_; } - - const grpc_chttp2_stream* stream() const { return s_; } - - void TestOnlyForceHugeWindow() override { - announced_window_delta_ = 1024 * 1024 * 1024; - local_window_delta_ = 1024 * 1024 * 1024; - remote_window_delta_ = 1024 * 1024 * 1024; - } + uint32_t min_progress_size() const { return min_progress_size_; } private: TransportFlowControl* const tfc_; - const grpc_chttp2_stream* const s_; + int64_t min_progress_size_ = 0; + int64_t remote_window_delta_ = 0; + int64_t announced_window_delta_ = 0; + absl::optional pending_size_; - void UpdateAnnouncedWindowDelta(TransportFlowControl* tfc, int64_t change) { - tfc->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_); - announced_window_delta_ += change; - tfc->PostUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_); - } + FlowControlAction UpdateAction(FlowControlAction action); + uint32_t DesiredAnnounceSize() const; }; class TestOnlyTransportTargetWindowEstimatesMocker { diff --git a/src/core/ext/transport/chttp2/transport/frame_data.cc b/src/core/ext/transport/chttp2/transport/frame_data.cc index 8cb4354f37f..4bc9f136811 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.cc +++ b/src/core/ext/transport/chttp2/transport/frame_data.cc @@ -20,9 +20,8 @@ #include "src/core/ext/transport/chttp2/transport/frame_data.h" -#include +#include -#include "absl/base/attributes.h" #include "absl/status/status.h" #include "absl/strings/str_format.h" @@ -30,33 +29,16 @@ #include #include "src/core/ext/transport/chttp2/transport/internal.h" -#include "src/core/lib/channel/channelz.h" -#include "src/core/lib/gpr/string.h" -#include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/memory.h" -#include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_refcount.h" -#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/transport/transport.h" -grpc_chttp2_data_parser::~grpc_chttp2_data_parser() { - if (parsing_frame != nullptr) { - GRPC_ERROR_UNREF(parsing_frame->Finished( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Parser destroyed"), false)); - } - GRPC_ERROR_UNREF(error); -} - -grpc_error_handle grpc_chttp2_data_parser_begin_frame( - grpc_chttp2_data_parser* /*parser*/, uint8_t flags, uint32_t stream_id, - grpc_chttp2_stream* s) { +absl::Status grpc_chttp2_data_parser_begin_frame(uint8_t flags, + uint32_t stream_id, + grpc_chttp2_stream* s) { if (flags & ~GRPC_CHTTP2_DATA_FLAG_END_STREAM) { - return grpc_error_set_int(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( - "unsupported data flags: 0x%02x", flags)), - GRPC_ERROR_INT_STREAM_ID, - static_cast(stream_id)); + return absl::InternalError(absl::StrFormat( + "unsupported data flags: 0x%02x stream: %d", flags, stream_id)); } if (flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM) { @@ -66,7 +48,7 @@ grpc_error_handle grpc_chttp2_data_parser_begin_frame( s->received_last_frame = false; } - return GRPC_ERROR_NONE; + return absl::OkStatus(); } void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer* inbuf, @@ -97,188 +79,58 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer* inbuf, stats->data_bytes += write_bytes; } -grpc_error_handle grpc_deframe_unprocessed_incoming_frames( - grpc_chttp2_data_parser* p, grpc_chttp2_stream* s, - grpc_slice_buffer* slices, grpc_slice* slice_out, - grpc_core::OrphanablePtr* stream_out) { +grpc_core::Poll grpc_deframe_unprocessed_incoming_frames( + grpc_chttp2_stream* s, uint32_t* min_progress_size, + grpc_core::SliceBuffer* stream_out, uint32_t* message_flags) { + grpc_slice_buffer* slices = &s->frame_storage; grpc_error_handle error = GRPC_ERROR_NONE; - grpc_chttp2_transport* t = s->t; - while (slices->count > 0) { - uint8_t* beg = nullptr; - uint8_t* end = nullptr; - uint8_t* cur = nullptr; - - grpc_slice* slice = grpc_slice_buffer_peek_first(slices); - beg = GRPC_SLICE_START_PTR(*slice); - end = GRPC_SLICE_END_PTR(*slice); - cur = beg; - uint32_t message_flags; + if (slices->length < 5) { + if (min_progress_size != nullptr) *min_progress_size = 5 - slices->length; + return grpc_core::Pending{}; + } - if (cur == end) { - grpc_slice_buffer_remove_first(slices); - continue; - } + uint8_t header[5]; + grpc_slice_buffer_copy_first_into_buffer(slices, 5, header); - switch (p->state) { - case GRPC_CHTTP2_DATA_ERROR: - p->state = GRPC_CHTTP2_DATA_ERROR; - grpc_slice_buffer_remove_first(slices); - return GRPC_ERROR_REF(p->error); - case GRPC_CHTTP2_DATA_FH_0: - s->stats.incoming.framing_bytes++; - p->frame_type = *cur; - switch (p->frame_type) { - case 0: - p->is_frame_compressed = false; /* GPR_FALSE */ - break; - case 1: - p->is_frame_compressed = true; /* GPR_TRUE */ - break; - default: - p->error = GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrFormat("Bad GRPC frame type 0x%02x", p->frame_type)); - p->error = grpc_error_set_int(p->error, GRPC_ERROR_INT_STREAM_ID, - static_cast(s->id)); - grpc_core::UniquePtr dmp( - grpc_dump_slice(*slice, GPR_DUMP_HEX | GPR_DUMP_ASCII)); - p->error = grpc_error_set_str(p->error, GRPC_ERROR_STR_RAW_BYTES, - dmp.get()); - p->error = - grpc_error_set_int(p->error, GRPC_ERROR_INT_OFFSET, cur - beg); - p->state = GRPC_CHTTP2_DATA_ERROR; - grpc_slice_buffer_remove_first(slices); - return GRPC_ERROR_REF(p->error); - } - if (++cur == end) { - p->state = GRPC_CHTTP2_DATA_FH_1; - grpc_slice_buffer_remove_first(slices); - continue; - } - ABSL_FALLTHROUGH_INTENDED; - case GRPC_CHTTP2_DATA_FH_1: - s->stats.incoming.framing_bytes++; - p->frame_size = (static_cast(*cur)) << 24; - if (++cur == end) { - p->state = GRPC_CHTTP2_DATA_FH_2; - grpc_slice_buffer_remove_first(slices); - continue; - } - ABSL_FALLTHROUGH_INTENDED; - case GRPC_CHTTP2_DATA_FH_2: - s->stats.incoming.framing_bytes++; - p->frame_size |= (static_cast(*cur)) << 16; - if (++cur == end) { - p->state = GRPC_CHTTP2_DATA_FH_3; - grpc_slice_buffer_remove_first(slices); - continue; - } - ABSL_FALLTHROUGH_INTENDED; - case GRPC_CHTTP2_DATA_FH_3: - s->stats.incoming.framing_bytes++; - p->frame_size |= (static_cast(*cur)) << 8; - if (++cur == end) { - p->state = GRPC_CHTTP2_DATA_FH_4; - grpc_slice_buffer_remove_first(slices); - continue; - } - ABSL_FALLTHROUGH_INTENDED; - case GRPC_CHTTP2_DATA_FH_4: - s->stats.incoming.framing_bytes++; - GPR_ASSERT(stream_out != nullptr); - GPR_ASSERT(p->parsing_frame == nullptr); - p->frame_size |= (static_cast(*cur)); - if (t->channelz_socket != nullptr) { - t->channelz_socket->RecordMessageReceived(); - } - p->state = GRPC_CHTTP2_DATA_FRAME; - ++cur; - message_flags = 0; - if (p->is_frame_compressed) { - message_flags |= GRPC_WRITE_INTERNAL_COMPRESS; - } - p->parsing_frame = new grpc_core::Chttp2IncomingByteStream( - t, s, p->frame_size, message_flags); - stream_out->reset(p->parsing_frame); - if (p->parsing_frame->remaining_bytes() == 0) { - GRPC_ERROR_UNREF(p->parsing_frame->Finished(GRPC_ERROR_NONE, true)); - p->parsing_frame = nullptr; - p->state = GRPC_CHTTP2_DATA_FH_0; - } - s->pending_byte_stream = true; - if (cur != end) { - grpc_slice_buffer_sub_first(slices, static_cast(cur - beg), - static_cast(end - beg)); - } else { - grpc_slice_buffer_remove_first(slices); - } - return GRPC_ERROR_NONE; - case GRPC_CHTTP2_DATA_FRAME: { - GPR_ASSERT(p->parsing_frame != nullptr); - GPR_ASSERT(slice_out != nullptr); - if (cur == end) { - grpc_slice_buffer_remove_first(slices); - continue; - } - uint32_t remaining = static_cast(end - cur); - if (remaining == p->frame_size) { - s->stats.incoming.data_bytes += remaining; - if (GRPC_ERROR_NONE != - (error = p->parsing_frame->Push( - grpc_slice_sub(*slice, static_cast(cur - beg), - static_cast(end - beg)), - slice_out))) { - grpc_slice_buffer_remove_first(slices); - return error; - } - if (GRPC_ERROR_NONE != - (error = p->parsing_frame->Finished(GRPC_ERROR_NONE, true))) { - grpc_slice_buffer_remove_first(slices); - return error; - } - p->parsing_frame = nullptr; - p->state = GRPC_CHTTP2_DATA_FH_0; - grpc_slice_buffer_remove_first(slices); - return GRPC_ERROR_NONE; - } else if (remaining < p->frame_size) { - s->stats.incoming.data_bytes += remaining; - if (GRPC_ERROR_NONE != - (error = p->parsing_frame->Push( - grpc_slice_sub(*slice, static_cast(cur - beg), - static_cast(end - beg)), - slice_out))) { - return error; - } - p->frame_size -= remaining; - grpc_slice_buffer_remove_first(slices); - return GRPC_ERROR_NONE; - } else { - GPR_ASSERT(remaining > p->frame_size); - s->stats.incoming.data_bytes += p->frame_size; - if (GRPC_ERROR_NONE != - p->parsing_frame->Push( - grpc_slice_sub( - *slice, static_cast(cur - beg), - static_cast(cur + p->frame_size - beg)), - slice_out)) { - grpc_slice_buffer_remove_first(slices); - return error; - } - if (GRPC_ERROR_NONE != - (error = p->parsing_frame->Finished(GRPC_ERROR_NONE, true))) { - grpc_slice_buffer_remove_first(slices); - return error; - } - p->parsing_frame = nullptr; - p->state = GRPC_CHTTP2_DATA_FH_0; - cur += p->frame_size; - grpc_slice_buffer_sub_first(slices, static_cast(cur - beg), - static_cast(end - beg)); - return GRPC_ERROR_NONE; - } + switch (header[0]) { + case 0: + if (message_flags != nullptr) *message_flags = 0; + break; + case 1: + if (message_flags != nullptr) { + *message_flags = GRPC_WRITE_INTERNAL_COMPRESS; } + break; + default: + error = GRPC_ERROR_CREATE_FROM_CPP_STRING( + absl::StrFormat("Bad GRPC frame type 0x%02x", header[0])); + error = grpc_error_set_int(error, GRPC_ERROR_INT_STREAM_ID, + static_cast(s->id)); + return error; + } + + uint64_t length = (static_cast(header[1]) << 24) | + (static_cast(header[2]) << 16) | + (static_cast(header[3]) << 8) | + static_cast(header[4]); + + if (slices->length < length + 5) { + if (min_progress_size != nullptr) { + *min_progress_size = length + 5 - slices->length; } + return grpc_core::Pending{}; + } + + if (min_progress_size != nullptr) *min_progress_size = 0; + + if (stream_out != nullptr) { + s->stats.incoming.framing_bytes += 5; + s->stats.incoming.data_bytes += length; + grpc_slice_buffer_move_first_into_buffer(slices, 5, header); + grpc_slice_buffer_move_first(slices, length, stream_out->c_slice_buffer()); } + return GRPC_ERROR_NONE; } @@ -287,20 +139,9 @@ grpc_error_handle grpc_chttp2_data_parser_parse(void* /*parser*/, grpc_chttp2_stream* s, const grpc_slice& slice, int is_last) { - if (!s->pending_byte_stream) { - grpc_slice_ref_internal(slice); - grpc_slice_buffer_add(&s->frame_storage, slice); - grpc_chttp2_maybe_complete_recv_message(t, s); - } else if (s->on_next) { - GPR_ASSERT(s->frame_storage.length == 0); - grpc_slice_ref_internal(slice); - grpc_slice_buffer_add(&s->unprocessed_incoming_frames_buffer, slice); - grpc_core::ExecCtx::Run(DEBUG_LOCATION, s->on_next, GRPC_ERROR_NONE); - s->on_next = nullptr; - } else { - grpc_slice_ref_internal(slice); - grpc_slice_buffer_add(&s->frame_storage, slice); - } + grpc_slice_ref_internal(slice); + grpc_slice_buffer_add(&s->frame_storage, slice); + grpc_chttp2_maybe_complete_recv_message(t, s); if (is_last && s->received_last_frame) { grpc_chttp2_mark_stream_closed( diff --git a/src/core/ext/transport/chttp2/transport/frame_data.h b/src/core/ext/transport/chttp2/transport/frame_data.h index d32bba83069..e1805edcb6a 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.h +++ b/src/core/ext/transport/chttp2/transport/frame_data.h @@ -25,45 +25,20 @@ #include +#include "absl/status/status.h" + #include #include "src/core/ext/transport/chttp2/transport/frame.h" -#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/error.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/promise/poll.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/transport.h" -typedef enum { - GRPC_CHTTP2_DATA_FH_0, - GRPC_CHTTP2_DATA_FH_1, - GRPC_CHTTP2_DATA_FH_2, - GRPC_CHTTP2_DATA_FH_3, - GRPC_CHTTP2_DATA_FH_4, - GRPC_CHTTP2_DATA_FRAME, - GRPC_CHTTP2_DATA_ERROR -} grpc_chttp2_stream_state; - -namespace grpc_core { -class Chttp2IncomingByteStream; -} // namespace grpc_core - -struct grpc_chttp2_data_parser { - grpc_chttp2_data_parser() = default; - ~grpc_chttp2_data_parser(); - - grpc_chttp2_stream_state state = GRPC_CHTTP2_DATA_FH_0; - uint8_t frame_type = 0; - uint32_t frame_size = 0; - grpc_error_handle error = GRPC_ERROR_NONE; - - bool is_frame_compressed = false; - grpc_core::Chttp2IncomingByteStream* parsing_frame = nullptr; -}; - /* start processing a new data frame */ -grpc_error_handle grpc_chttp2_data_parser_begin_frame( - grpc_chttp2_data_parser* parser, uint8_t flags, uint32_t stream_id, - grpc_chttp2_stream* s); +absl::Status grpc_chttp2_data_parser_begin_frame(uint8_t flags, + uint32_t stream_id, + grpc_chttp2_stream* s); /* handle a slice of a data frame - is_last indicates the last slice of a frame */ @@ -78,9 +53,8 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer* inbuf, grpc_transport_one_way_stats* stats, grpc_slice_buffer* outbuf); -grpc_error_handle grpc_deframe_unprocessed_incoming_frames( - grpc_chttp2_data_parser* p, grpc_chttp2_stream* s, - grpc_slice_buffer* slices, grpc_slice* slice_out, - grpc_core::OrphanablePtr* stream_out); +grpc_core::Poll grpc_deframe_unprocessed_incoming_frames( + grpc_chttp2_stream* s, uint32_t* min_progress_size, + grpc_core::SliceBuffer* stream_out, uint32_t* message_flags); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_DATA_H */ diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.cc b/src/core/ext/transport/chttp2/transport/frame_settings.cc index dfbfde9ee7e..d0157475ba5 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.cc +++ b/src/core/ext/transport/chttp2/transport/frame_settings.cc @@ -35,11 +35,9 @@ #include "src/core/ext/transport/chttp2/transport/frame_goaway.h" #include "src/core/ext/transport/chttp2/transport/hpack_encoder.h" #include "src/core/ext/transport/chttp2/transport/internal.h" -#include "src/core/ext/transport/chttp2/transport/stream_map.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/iomgr/exec_ctx.h" static uint8_t* fill_header(uint8_t* out, uint32_t length, uint8_t flags) { @@ -119,22 +117,6 @@ grpc_error_handle grpc_chttp2_settings_parser_begin_frame( } } -namespace { - -void StreamFlowControlWindowCheck(void* user_data, uint32_t /* key */, - void* stream) { - bool* error = static_cast(user_data); - grpc_chttp2_stream* s = static_cast(stream); - if ((s->t->settings[GRPC_PEER_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] + - s->t->initial_window_update + s->flow_control->remote_window_delta()) > - ((1u << 31) - 1)) { - *error = true; - } -} - -} // namespace - grpc_error_handle grpc_chttp2_settings_parser_parse(void* p, grpc_chttp2_transport* t, grpc_chttp2_stream* /*s*/, @@ -219,12 +201,6 @@ grpc_error_handle grpc_chttp2_settings_parser_parse(void* p, if (grpc_wire_id_to_setting_id(parser->id, &id)) { const grpc_chttp2_setting_parameters* sp = &grpc_chttp2_settings_parameters[id]; - // If flow control is disabled we skip these. - if (!t->flow_control->flow_control_enabled() && - (id == GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE || - id == GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE)) { - continue; - } if (parser->value < sp->min_value || parser->value > sp->max_value) { switch (sp->invalid_value_behavior) { case GRPC_CHTTP2_CLAMP_INVALID_VALUE: @@ -250,23 +226,6 @@ grpc_error_handle grpc_chttp2_settings_parser_parse(void* p, t->is_client ? "cli" : "svr", static_cast(t->initial_window_update)); } - if (grpc_core::chttp2:: - g_test_only_transport_flow_control_window_check) { - bool error = false; - if (parser->value > grpc_core::chttp2::kMaxInitialWindowSize || - parser->value < grpc_core::chttp2::kMinInitialWindowSize) { - error = true; - } else { - grpc_chttp2_stream_map_for_each( - &t->stream_map, StreamFlowControlWindowCheck, &error); - } - if (error) { - grpc_chttp2_goaway_append( - t->last_new_stream_id, sp->error_value, - grpc_slice_from_static_string("HTTP2 settings error"), - &t->qbuf); - } - } } parser->incoming_settings[id] = parser->value; if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) { diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.cc b/src/core/ext/transport/chttp2/transport/frame_window_update.cc index 790c654cb59..eb65e86d523 100644 --- a/src/core/ext/transport/chttp2/transport/frame_window_update.cc +++ b/src/core/ext/transport/chttp2/transport/frame_window_update.cc @@ -29,7 +29,6 @@ #include "src/core/ext/transport/chttp2/transport/flow_control.h" #include "src/core/ext/transport/chttp2/transport/internal.h" -#include "src/core/lib/gprpp/manual_constructor.h" grpc_slice grpc_chttp2_window_update_create( uint32_t id, uint32_t window_delta, grpc_transport_one_way_stats* stats) { @@ -98,13 +97,9 @@ grpc_error_handle grpc_chttp2_window_update_parser_parse( if (t->incoming_stream_id != 0) { if (s != nullptr) { - s->flow_control->RecvUpdate(received_update); - if (grpc_core::chttp2:: - g_test_only_transport_flow_control_window_check && - s->flow_control->remote_window_delta() > - grpc_core::chttp2::kMaxWindowDelta) { - GPR_ASSERT(false); - } + grpc_core::chttp2::StreamFlowControl::OutgoingUpdateContext( + &s->flow_control) + .RecvUpdate(received_update); if (grpc_chttp2_list_remove_stalled_by_stream(t, s)) { grpc_chttp2_mark_stream_writable(t, s); grpc_chttp2_initiate_write( @@ -112,10 +107,10 @@ grpc_error_handle grpc_chttp2_window_update_parser_parse( } } } else { - bool was_zero = t->flow_control->remote_window() <= 0; - t->flow_control->RecvUpdate(received_update); - bool is_zero = t->flow_control->remote_window() <= 0; - if (was_zero && !is_zero) { + grpc_core::chttp2::TransportFlowControl::OutgoingUpdateContext upd( + &t->flow_control); + upd.RecvUpdate(received_update); + if (upd.Finish() == grpc_core::chttp2::StallEdge::kUnstalled) { grpc_chttp2_initiate_write( t, GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED); } diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index d859766eadf..4af3c4ec9db 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -27,6 +27,7 @@ #include #include "absl/strings/string_view.h" +#include "absl/types/optional.h" #include #include @@ -34,7 +35,6 @@ #include "src/core/ext/transport/chttp2/transport/flow_control.h" #include "src/core/ext/transport/chttp2/transport/frame.h" -#include "src/core/ext/transport/chttp2/transport/frame_data.h" #include "src/core/ext/transport/chttp2/transport/frame_goaway.h" #include "src/core/ext/transport/chttp2/transport/frame_ping.h" #include "src/core/ext/transport/chttp2/transport/frame_rst_stream.h" @@ -48,8 +48,6 @@ #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/bitset.h" #include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/time.h" @@ -60,7 +58,7 @@ #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/memory_quota.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -230,76 +228,6 @@ typedef struct grpc_chttp2_write_cb { struct grpc_chttp2_write_cb* next; } grpc_chttp2_write_cb; -namespace grpc_core { - -class Chttp2IncomingByteStream : public ByteStream { - public: - Chttp2IncomingByteStream(grpc_chttp2_transport* transport, - grpc_chttp2_stream* stream, uint32_t frame_size, - uint32_t flags); - - void Orphan() override; - - bool Next(size_t max_size_hint, grpc_closure* on_complete) override; - grpc_error_handle Pull(grpc_slice* slice) override; - void Shutdown(grpc_error_handle error) override; - - // TODO(roth): When I converted this class to C++, I wanted to make it - // inherit from RefCounted or InternallyRefCounted instead of continuing - // to use its own custom ref-counting code. However, that would require - // using multiple inheritance, which sucks in general. And to make matters - // worse, it causes problems with our New<> and Delete<> wrappers. - // Specifically, unless RefCounted is first in the list of parent classes, - // it will see a different value of the address of the object than the one - // we actually allocated, in which case gpr_free() will be called on a - // different address than the one we got from gpr_malloc(), thus causing a - // crash. Given the fragility of depending on that, as well as a desire to - // avoid multiple inheritance in general, I've decided to leave this - // alone for now. We can revisit this once we're able to link against - // libc++, at which point we can eliminate New<> and Delete<> and - // switch to std::shared_ptr<>. - void Ref() { refs_.Ref(); } - void Unref() { - if (GPR_UNLIKELY(refs_.Unref())) { - delete this; - } - } - - void PublishError(grpc_error_handle error); - - grpc_error_handle Push(const grpc_slice& slice, grpc_slice* slice_out); - - grpc_error_handle Finished(grpc_error_handle error, bool reset_on_error); - - uint32_t remaining_bytes() const { return remaining_bytes_; } - - private: - static void NextLocked(void* arg, grpc_error_handle error_ignored); - static void OrphanLocked(void* arg, grpc_error_handle error_ignored); - - grpc_chttp2_transport* transport_; // Immutable. - grpc_chttp2_stream* stream_; // Immutable. - - RefCount refs_; - - /* Accessed only by transport thread when stream->pending_byte_stream == false - * Accessed only by application thread when stream->pending_byte_stream == - * true */ - uint32_t remaining_bytes_; - - /* Accessed only by transport thread when stream->pending_byte_stream == false - * Accessed only by application thread when stream->pending_byte_stream == - * true */ - struct { - grpc_closure closure; - size_t max_size_hint; - grpc_closure* on_complete; - } next_action_; - grpc_closure destroy_action_; -}; - -} // namespace grpc_core - typedef enum { GRPC_CHTTP2_KEEPALIVE_STATE_WAITING, GRPC_CHTTP2_KEEPALIVE_STATE_PINGING, @@ -428,11 +356,7 @@ struct grpc_chttp2_transport { /** parser for goaway frames */ grpc_chttp2_goaway_parser goaway_parser; - grpc_core::PolymorphicManualConstructor< - grpc_core::chttp2::TransportFlowControlBase, - grpc_core::chttp2::TransportFlowControl, - grpc_core::chttp2::TransportFlowControlDisabled> - flow_control; + grpc_core::chttp2::TransportFlowControl flow_control; /** initial window change. This is tracked as we parse settings frames from * the remote peer. If there is a positive delta, then we will make all * streams readable since they may have become unstalled */ @@ -566,19 +490,16 @@ struct grpc_chttp2_stream { bool* sent_trailing_metadata_op = nullptr; grpc_closure* send_trailing_metadata_finished = nullptr; - grpc_core::OrphanablePtr fetching_send_message; - uint32_t fetched_send_message_length = 0; - grpc_slice fetching_slice = grpc_empty_slice(); int64_t next_message_end_offset; int64_t flow_controlled_bytes_written = 0; int64_t flow_controlled_bytes_flowed = 0; - grpc_closure complete_fetch_locked; - grpc_closure* fetching_send_message_finished = nullptr; + grpc_closure* send_message_finished = nullptr; grpc_metadata_batch* recv_initial_metadata; grpc_closure* recv_initial_metadata_ready = nullptr; bool* trailing_metadata_available = nullptr; - grpc_core::OrphanablePtr* recv_message = nullptr; + absl::optional* recv_message = nullptr; + uint32_t* recv_message_flags = nullptr; bool* call_failed_before_recv_message = nullptr; grpc_closure* recv_message_ready = nullptr; grpc_metadata_batch* recv_trailing_metadata; @@ -615,22 +536,7 @@ struct grpc_chttp2_stream { grpc_metadata_batch initial_metadata_buffer; grpc_metadata_batch trailing_metadata_buffer; - grpc_slice_buffer frame_storage; /* protected by t combiner */ - - grpc_closure* on_next = nullptr; /* protected by t combiner */ - bool pending_byte_stream = false; /* protected by t combiner */ - // cached length of buffer to be used by the transport thread in cases where - // stream->pending_byte_stream == true. The value is saved before - // application threads are allowed to modify - // unprocessed_incoming_frames_buffer - size_t unprocessed_incoming_frames_buffer_cached_length = 0; - /* Accessed only by transport thread when stream->pending_byte_stream == false - * Accessed only by application thread when stream->pending_byte_stream == - * true */ - grpc_slice_buffer unprocessed_incoming_frames_buffer; - grpc_closure reset_byte_stream; - grpc_error_handle byte_stream_error = - GRPC_ERROR_NONE; /* protected by t combiner */ + grpc_slice_buffer frame_storage; /* protected by t combiner */ bool received_last_frame = false; /* protected by t combiner */ grpc_core::Timestamp deadline = grpc_core::Timestamp::InfFuture(); @@ -639,22 +545,13 @@ struct grpc_chttp2_stream { grpc_error_handle forced_close_error = GRPC_ERROR_NONE; /** how many header frames have we received? */ uint8_t header_frames_received = 0; - /** parsing state for data frames */ - /* Accessed only by transport thread when stream->pending_byte_stream == false - * Accessed only by application thread when stream->pending_byte_stream == - * true */ - grpc_chttp2_data_parser data_parser; /** number of bytes received - reset at end of parse thread execution */ int64_t received_bytes = 0; bool sent_initial_metadata = false; bool sent_trailing_metadata = false; - grpc_core::PolymorphicManualConstructor< - grpc_core::chttp2::StreamFlowControlBase, - grpc_core::chttp2::StreamFlowControl, - grpc_core::chttp2::StreamFlowControlDisabled> - flow_control; + grpc_core::chttp2::StreamFlowControl flow_control; grpc_slice_buffer flow_controlled_buffer; diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc index d91af3801be..6a45381e542 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.cc +++ b/src/core/ext/transport/chttp2/transport/parsing.cc @@ -47,11 +47,11 @@ #include "src/core/ext/transport/chttp2/transport/stream_map.h" #include "src/core/lib/channel/channelz.h" #include "src/core/lib/debug/trace.h" -#include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/time.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/transport/bdp_estimator.h" +#include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/http2_errors.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -218,10 +218,9 @@ grpc_error_handle grpc_chttp2_perform_read(grpc_chttp2_transport* t, return GRPC_ERROR_NONE; } goto dts_fh_0; /* loop */ - } else if (t->flow_control->flow_control_enabled() && - t->incoming_frame_size > - t->settings[GRPC_ACKED_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) { + } else if (t->incoming_frame_size > + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) { return GRPC_ERROR_CREATE_FROM_CPP_STRING( absl::StrFormat("Frame size %d is larger than max frame size %d", t->incoming_frame_size, @@ -385,7 +384,7 @@ void grpc_chttp2_parsing_become_skip_parser(grpc_chttp2_transport* t) { static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) { // Update BDP accounting since we have received a data frame. - grpc_core::BdpEstimator* bdp_est = t->flow_control->bdp_estimator(); + grpc_core::BdpEstimator* bdp_est = t->flow_control.bdp_estimator(); if (bdp_est) { if (t->bdp_ping_blocked) { t->bdp_ping_blocked = false; @@ -396,17 +395,21 @@ static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) { } grpc_chttp2_stream* s = grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); - grpc_error_handle err = GRPC_ERROR_NONE; + absl::Status status; grpc_core::chttp2::FlowControlAction action; if (s == nullptr) { - err = t->flow_control->RecvData(t->incoming_frame_size); - action = t->flow_control->MakeAction(); + grpc_core::chttp2::TransportFlowControl::IncomingUpdateContext upd( + &t->flow_control); + status = upd.RecvData(t->incoming_frame_size); + action = upd.MakeAction(); } else { - err = s->flow_control->RecvData(t->incoming_frame_size); - action = s->flow_control->MakeAction(); + grpc_core::chttp2::StreamFlowControl::IncomingUpdateContext upd( + &s->flow_control); + status = upd.RecvData(t->incoming_frame_size); + action = upd.MakeAction(); } grpc_chttp2_act_on_flowctl_action(action, t, s); - if (err != GRPC_ERROR_NONE) { + if (!status.ok()) { goto error_handler; } if (s == nullptr) { @@ -414,33 +417,29 @@ static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) { } s->received_bytes += t->incoming_frame_size; s->stats.incoming.framing_bytes += 9; - if (err == GRPC_ERROR_NONE && s->read_closed) { + if (s->read_closed) { return init_non_header_skip_frame_parser(t); } - if (err == GRPC_ERROR_NONE) { - err = grpc_chttp2_data_parser_begin_frame( - &s->data_parser, t->incoming_frame_flags, s->id, s); - } + status = + grpc_chttp2_data_parser_begin_frame(t->incoming_frame_flags, s->id, s); error_handler: - intptr_t unused; - if (err == GRPC_ERROR_NONE) { + if (status.ok()) { t->incoming_stream = s; /* t->parser = grpc_chttp2_data_parser_parse;*/ t->parser = grpc_chttp2_data_parser_parse; - t->parser_data = &s->data_parser; + t->parser_data = nullptr; t->ping_state.last_ping_sent_time = grpc_core::Timestamp::InfPast(); return GRPC_ERROR_NONE; - } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, &unused)) { + } else if (s != nullptr) { /* handle stream errors by closing the stream */ - if (s != nullptr) { - grpc_chttp2_mark_stream_closed(t, s, true, false, err); - } + grpc_chttp2_mark_stream_closed(t, s, true, false, + absl_status_to_grpc_error(status)); grpc_chttp2_add_rst_stream_to_next_write(t, t->incoming_stream_id, GRPC_HTTP2_PROTOCOL_ERROR, &s->stats.outgoing); return init_non_header_skip_frame_parser(t); } else { - return err; + return absl_status_to_grpc_error(status); } } @@ -569,6 +568,10 @@ static grpc_error_handle init_header_frame_parser(grpc_chttp2_transport* t, gpr_log(GPR_ERROR, "too many header frames received"); return init_header_skip_frame_parser(t, priority_type); } + if (frame_type == HPackParser::LogInfo::kTrailers && !t->header_eof) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Trailing metadata frame received without an end-o-stream"); + } t->hpack_parser.BeginFrame( incoming_metadata_buffer, t->settings[GRPC_ACKED_SETTINGS] @@ -648,6 +651,9 @@ static grpc_error_handle init_settings_frame_parser(grpc_chttp2_transport* t) { t->hpack_parser.hpack_table()->SetMaxBytes( t->settings[GRPC_ACKED_SETTINGS] [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]); + t->flow_control.SetAckedInitialWindow( + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); t->sent_local_settings = false; } t->parser = grpc_chttp2_settings_parser_parse; diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.cc b/src/core/ext/transport/chttp2/transport/stream_lists.cc index aaa73975b5c..92c36a60934 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.cc +++ b/src/core/ext/transport/chttp2/transport/stream_lists.cc @@ -20,12 +20,10 @@ #include -#include "src/core/ext/transport/chttp2/transport/flow_control.h" #include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/ext/transport/chttp2/transport/internal.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/bitset.h" -#include "src/core/lib/gprpp/manual_constructor.h" static const char* stream_list_id_string(grpc_chttp2_stream_list_id id) { switch (id) { @@ -189,7 +187,6 @@ void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport* t, void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { - GPR_ASSERT(t->flow_control->flow_control_enabled()); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } @@ -205,7 +202,6 @@ void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport* t, void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { - GPR_ASSERT(t->flow_control->flow_control_enabled()); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc index 1cc39b7135d..e676672e7d9 100644 --- a/src/core/ext/transport/chttp2/transport/writing.cc +++ b/src/core/ext/transport/chttp2/transport/writing.cc @@ -22,7 +22,6 @@ #include #include -#include #include #include "absl/types/optional.h" @@ -50,7 +49,6 @@ #include "src/core/lib/debug/stats.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/time.h" @@ -220,14 +218,14 @@ static void report_stall(grpc_chttp2_transport* t, grpc_chttp2_stream* s, s->flow_controlled_buffer.length, s->flow_controlled_bytes_flowed, t->settings[GRPC_ACKED_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], - t->flow_control->remote_window(), + t->flow_control.remote_window(), static_cast(std::max( int64_t(0), - s->flow_control->remote_window_delta() + + s->flow_control.remote_window_delta() + static_cast( t->settings[GRPC_PEER_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]))), - s->flow_control->remote_window_delta()); + s->flow_control.remote_window_delta()); } } @@ -304,7 +302,7 @@ class WriteContext { void FlushWindowUpdates() { uint32_t transport_announce = - t_->flow_control->MaybeSendUpdate(t_->outbuf.count > 0); + t_->flow_control.MaybeSendUpdate(t_->outbuf.count > 0); if (transport_announce) { grpc_transport_one_way_stats throwaway_stats; grpc_slice_buffer_add( @@ -392,7 +390,7 @@ class DataSendContext { uint32_t stream_remote_window() const { return static_cast(std::max( int64_t(0), - s_->flow_control->remote_window_delta() + + s_->flow_control.remote_window_delta() + static_cast( t_->settings[GRPC_PEER_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]))); @@ -402,7 +400,7 @@ class DataSendContext { return static_cast(std::min( t_->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], static_cast(std::min(int64_t(stream_remote_window()), - t_->flow_control->remote_window())))); + t_->flow_control.remote_window())))); } bool AnyOutgoing() const { return max_outgoing() > 0; } @@ -411,12 +409,11 @@ class DataSendContext { uint32_t send_bytes = static_cast( std::min(size_t(max_outgoing()), s_->flow_controlled_buffer.length)); is_last_frame_ = send_bytes == s_->flow_controlled_buffer.length && - s_->fetching_send_message == nullptr && s_->send_trailing_metadata != nullptr && s_->send_trailing_metadata->empty(); grpc_chttp2_encode_data(s_->id, &s_->flow_controlled_buffer, send_bytes, is_last_frame_, &s_->stats.outgoing, &t_->outbuf); - s_->flow_control->SentData(send_bytes); + sfc_upd_.SentData(send_bytes); s_->sending_bytes += send_bytes; } @@ -436,6 +433,8 @@ class DataSendContext { WriteContext* write_context_; grpc_chttp2_transport* t_; grpc_chttp2_stream* s_; + grpc_core::chttp2::StreamFlowControl::OutgoingUpdateContext sfc_upd_{ + &s_->flow_control}; const size_t sending_bytes_before_; bool is_last_frame_ = false; }; @@ -445,11 +444,9 @@ class StreamWriteContext { StreamWriteContext(WriteContext* write_context, grpc_chttp2_stream* s) : write_context_(write_context), t_(write_context->transport()), s_(s) { GRPC_CHTTP2_IF_TRACING( - gpr_log(GPR_INFO, "W:%p %s[%d] im-(sent,send)=(%d,%d) announce=%d", t_, + gpr_log(GPR_INFO, "W:%p %s[%d] im-(sent,send)=(%d,%d)", t_, t_->is_client ? "CLIENT" : "SERVER", s->id, - s->sent_initial_metadata, s->send_initial_metadata != nullptr, - (int)(s->flow_control->local_window_delta() - - s->flow_control->announced_window_delta()))); + s->sent_initial_metadata, s->send_initial_metadata != nullptr)); } void FlushInitialMetadata() { @@ -462,8 +459,7 @@ class StreamWriteContext { // trailing metadata. This results in a Trailers-Only response, // which is required for retries, as per: // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid - if (!t_->is_client && s_->fetching_send_message == nullptr && - s_->flow_controlled_buffer.length == 0 && + if (!t_->is_client && s_->flow_controlled_buffer.length == 0 && s_->send_trailing_metadata != nullptr && is_default_initial_metadata(s_->send_initial_metadata)) { ConvertInitialMetadataToTrailingMetadata(); @@ -495,8 +491,10 @@ class StreamWriteContext { } void FlushWindowUpdates() { + if (s_->read_closed) return; + /* send any window updates */ - const uint32_t stream_announce = s_->flow_control->MaybeSendUpdate(); + const uint32_t stream_announce = s_->flow_control.MaybeSendUpdate(); if (stream_announce == 0) return; grpc_slice_buffer_add( @@ -516,7 +514,7 @@ class StreamWriteContext { DataSendContext data_send_context(write_context_, t_, s_); if (!data_send_context.AnyOutgoing()) { - if (t_->flow_control->remote_window() <= 0) { + if (t_->flow_control.remote_window() <= 0) { report_stall(t_, s_, "transport"); grpc_chttp2_list_add_stalled_by_transport(t_, s_); } else if (data_send_context.stream_remote_window() <= 0) { @@ -547,7 +545,6 @@ class StreamWriteContext { if (!s_->sent_initial_metadata) return; if (s_->send_trailing_metadata == nullptr) return; - if (s_->fetching_send_message != nullptr) return; if (s_->flow_controlled_buffer.length != 0) return; GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "sending trailing_metadata")); @@ -635,7 +632,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( ctx.FlushQueuedBuffers(); ctx.EnactHpackSettings(); - if (t->flow_control->remote_window() > 0) { + if (t->flow_control.remote_window() > 0) { ctx.UpdateStreamsNoLongerStalled(); } diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.cc b/src/core/ext/transport/cronet/transport/cronet_transport.cc index 2fe5fdf311e..19e0f721c67 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.cc +++ b/src/core/ext/transport/cronet/transport/cronet_transport.cc @@ -24,20 +24,18 @@ #include #include -#include #include #include #include -#include "absl/status/status.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" #include "third_party/objective_c/Cronet/bidirectional_stream_c.h" #include -#include #include #include #include @@ -48,8 +46,6 @@ #include "src/core/ext/transport/cronet/transport/cronet_status.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/error.h" @@ -58,14 +54,15 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/surface/validate_metadata.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport_impl.h" +// IWYU pragma: no_include + #define GRPC_HEADER_SIZE_IN_BYTES 5 #define GRPC_FLUSH_READ_SIZE 4096 @@ -134,9 +131,7 @@ typedef struct grpc_cronet_transport grpc_cronet_transport; http://www.catb.org/esr/structure-packing/#_structure_reordering: */ struct read_state { explicit read_state(grpc_core::Arena* arena) - : trailing_metadata(arena), initial_metadata(arena) { - grpc_slice_buffer_init(&read_slice_buffer); - } + : trailing_metadata(arena), initial_metadata(arena) {} /* vars to store data coming from server */ char* read_buffer = nullptr; @@ -150,8 +145,7 @@ struct read_state { bool read_stream_closed = false; /* vars for holding data destined for the application */ - grpc_core::ManualConstructor sbs; - grpc_slice_buffer read_slice_buffer; + grpc_core::SliceBuffer read_slice_buffer; /* vars for trailing metadata */ grpc_metadata_batch trailing_metadata; @@ -1089,38 +1083,17 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { result = NO_ACTION_POSSIBLE; CRONET_LOG(GPR_DEBUG, "Stream is either cancelled, failed or finished"); } else { - grpc_slice_buffer write_slice_buffer; - grpc_slice slice; - grpc_slice_buffer_init(&write_slice_buffer); - while (write_slice_buffer.length < - stream_op->payload->send_message.send_message->length()) { - /* TODO(roth): When we add support for incremental sending,this code - * will need to be changed to support asynchronous delivery of the - * send_message payload. */ - if (!stream_op->payload->send_message.send_message->Next( - stream_op->payload->send_message.send_message->length(), - nullptr)) { - /* Should never reach here */ - GPR_ASSERT(false); - } - if (GRPC_ERROR_NONE != - stream_op->payload->send_message.send_message->Pull(&slice)) { - /* Should never reach here */ - GPR_ASSERT(false); - } - grpc_slice_buffer_add(&write_slice_buffer, slice); - } size_t write_buffer_size; - create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, - &write_buffer_size, - stream_op->payload->send_message.send_message->flags()); + create_grpc_frame( + stream_op->payload->send_message.send_message->c_slice_buffer(), + &stream_state->ws.write_buffer, &write_buffer_size, + stream_op->payload->send_message.flags); if (write_buffer_size > 0) { CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, %p)", s->cbs, stream_state->ws.write_buffer); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, static_cast(write_buffer_size), false); - grpc_slice_buffer_destroy_internal(&write_slice_buffer); if (t->use_packet_coalescing) { if (!stream_op->send_trailing_metadata) { CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); @@ -1140,7 +1113,6 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { } stream_state->state_op_done[OP_SEND_MESSAGE] = true; oas->state.state_op_done[OP_SEND_MESSAGE] = true; - stream_op->payload->send_message.send_message.reset(); } else if (stream_op->send_trailing_metadata && op_can_be_run(stream_op, s, &oas->state, OP_SEND_TRAILING_METADATA)) { @@ -1252,16 +1224,14 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { stream_state->rs.remaining_bytes = 0; CRONET_LOG(GPR_DEBUG, "read operation complete. Empty response."); /* Clean up read_slice_buffer in case there is unread data. */ - grpc_slice_buffer_destroy_internal( - &stream_state->rs.read_slice_buffer); - grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer); + stream_state->rs.read_slice_buffer.Clear(); uint32_t flags = 0; if (stream_state->rs.compressed) { flags |= GRPC_WRITE_INTERNAL_COMPRESS; } - stream_state->rs.sbs.Init(&stream_state->rs.read_slice_buffer, flags); - stream_op->payload->recv_message.recv_message->reset( - stream_state->rs.sbs.get()); + *stream_op->payload->recv_message.flags = flags; + *stream_op->payload->recv_message.recv_message = + std::move(stream_state->rs.read_slice_buffer); grpc_core::ExecCtx::Run( DEBUG_LOCATION, stream_op->payload->recv_message.recv_message_ready, @@ -1300,17 +1270,16 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { static_cast(stream_state->rs.length_field)); null_and_maybe_free_read_buffer(s); /* Clean up read_slice_buffer in case there is unread data. */ - grpc_slice_buffer_destroy_internal(&stream_state->rs.read_slice_buffer); - grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer); - grpc_slice_buffer_add(&stream_state->rs.read_slice_buffer, - read_data_slice); + stream_state->rs.read_slice_buffer.Clear(); + stream_state->rs.read_slice_buffer.Append( + grpc_core::Slice(read_data_slice)); uint32_t flags = 0; if (stream_state->rs.compressed) { flags = GRPC_WRITE_INTERNAL_COMPRESS; } - stream_state->rs.sbs.Init(&stream_state->rs.read_slice_buffer, flags); - stream_op->payload->recv_message.recv_message->reset( - stream_state->rs.sbs.get()); + *stream_op->payload->recv_message.flags = flags; + *stream_op->payload->recv_message.recv_message = + std::move(stream_state->rs.read_slice_buffer); grpc_core::ExecCtx::Run( DEBUG_LOCATION, stream_op->payload->recv_message.recv_message_ready, GRPC_ERROR_NONE); @@ -1426,8 +1395,6 @@ inline stream_obj::stream_obj(grpc_transport* gt, grpc_stream* gs, inline stream_obj::~stream_obj() { null_and_maybe_free_read_buffer(this); - /* Clean up read_slice_buffer in case there is unread data. */ - grpc_slice_buffer_destroy_internal(&state.rs.read_slice_buffer); GRPC_ERROR_UNREF(state.cancel_error); } diff --git a/src/core/ext/transport/inproc/inproc_transport.cc b/src/core/ext/transport/inproc/inproc_transport.cc index f6f9f6d500f..3b554775a3e 100644 --- a/src/core/ext/transport/inproc/inproc_transport.cc +++ b/src/core/ext/transport/inproc/inproc_transport.cc @@ -21,7 +21,6 @@ #include "src/core/ext/transport/inproc/inproc_transport.h" #include -#include #include #include @@ -34,11 +33,10 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "absl/utility/utility.h" #include #include -#include -#include #include #include #include @@ -50,8 +48,6 @@ #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/time.h" #include "src/core/lib/iomgr/closure.h" @@ -62,12 +58,11 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/surface/server.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -92,6 +87,10 @@ void fill_in_metadata(inproc_stream* s, const grpc_metadata_batch* metadata, uint32_t flags, grpc_metadata_batch* out_md, uint32_t* outflags, bool* markfilled); +void ResetSendMessage(grpc_transport_stream_op_batch* batch) { + absl::exchange(batch->payload->send_message.send_message, nullptr)->Clear(); +} + struct shared_mu { shared_mu() { // Share one lock between both sides since both sides get affected @@ -226,10 +225,6 @@ struct inproc_stream { GRPC_ERROR_UNREF(cancel_self_error); GRPC_ERROR_UNREF(cancel_other_error); - if (recv_inited) { - grpc_slice_buffer_destroy_internal(&recv_message); - } - t->unref(); } @@ -283,10 +278,6 @@ struct inproc_stream { grpc_transport_stream_op_batch* recv_message_op = nullptr; grpc_transport_stream_op_batch* recv_trailing_md_op = nullptr; - grpc_slice_buffer recv_message; - grpc_core::ManualConstructor recv_stream; - bool recv_inited = false; - bool initial_md_sent = false; bool trailing_md_sent = false; bool initial_md_recvd = false; @@ -360,8 +351,8 @@ void fill_in_metadata(inproc_stream* s, const grpc_metadata_batch* metadata, } // TODO(ctiller): copy the metadata batch, don't rely on a bespoke copy - // function. Can only do this once mdelems are out of the way though, too many - // edge cases otherwise. + // function. Can only do this once mdelems are out of the way though, too + // many edge cases otherwise. out_md->Clear(); CopySink sink(out_md); metadata->Encode(&sink); @@ -477,8 +468,8 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) { if (s->recv_initial_md_op) { grpc_error_handle err; if (!s->t->is_client) { - // If this is a server, provide initial metadata with a path and authority - // since it expects that as well as no error yet + // If this is a server, provide initial metadata with a path and + // authority since it expects that as well as no error yet grpc_metadata_batch fake_md(s->arena); fake_md.Set(grpc_core::HttpPathMetadata(), grpc_core::Slice::FromStaticString("/")); @@ -537,7 +528,7 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) { s->recv_message_op = nullptr; } if (s->send_message_op) { - s->send_message_op->payload->send_message.send_message.reset(); + ResetSendMessage(s->send_message_op); complete_if_batch_end_locked( s, error, s->send_message_op, "fail_helper scheduling send-message-on-complete"); @@ -579,35 +570,11 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) { // synchronously. That assumption is true today but may not always be // true in the future. void message_transfer_locked(inproc_stream* sender, inproc_stream* receiver) { - size_t remaining = - sender->send_message_op->payload->send_message.send_message->length(); - if (receiver->recv_inited) { - grpc_slice_buffer_destroy_internal(&receiver->recv_message); - } - grpc_slice_buffer_init(&receiver->recv_message); - receiver->recv_inited = true; - do { - grpc_slice message_slice; - grpc_closure unused; - GPR_ASSERT( - sender->send_message_op->payload->send_message.send_message->Next( - SIZE_MAX, &unused)); - grpc_error_handle error = - sender->send_message_op->payload->send_message.send_message->Pull( - &message_slice); - if (error != GRPC_ERROR_NONE) { - cancel_stream_locked(sender, GRPC_ERROR_REF(error)); - break; - } - GPR_ASSERT(error == GRPC_ERROR_NONE); - remaining -= GRPC_SLICE_LENGTH(message_slice); - grpc_slice_buffer_add(&receiver->recv_message, message_slice); - } while (remaining > 0); - sender->send_message_op->payload->send_message.send_message.reset(); - - receiver->recv_stream.Init(&receiver->recv_message, 0); - receiver->recv_message_op->payload->recv_message.recv_message->reset( - receiver->recv_stream.get()); + *receiver->recv_message_op->payload->recv_message.recv_message = + std::move(*sender->send_message_op->payload->send_message.send_message); + *receiver->recv_message_op->payload->recv_message.flags = + sender->send_message_op->payload->send_message.flags; + INPROC_LOG(GPR_INFO, "message_transfer_locked %p scheduling message-ready", receiver); grpc_core::ExecCtx::Run( @@ -656,7 +623,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { maybe_process_ops_locked(other, GRPC_ERROR_NONE); } else if (!s->t->is_client && s->trailing_md_sent) { // A server send will never be matched if the server already sent status - s->send_message_op->payload->send_message.send_message.reset(); + ResetSendMessage(s->send_message_op); complete_if_batch_end_locked( s, GRPC_ERROR_NONE, s->send_message_op, "op_state_machine scheduling send-message-on-complete case 1"); @@ -797,7 +764,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { if (s->recv_message_op != nullptr) { // This message needs to be wrapped up because it will never be // satisfied - *s->recv_message_op->payload->recv_message.recv_message = nullptr; + s->recv_message_op->payload->recv_message.recv_message->reset(); INPROC_LOG(GPR_INFO, "op_state_machine %p scheduling message-ready", s); grpc_core::ExecCtx::Run( DEBUG_LOCATION, @@ -811,7 +778,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { if ((s->trailing_md_sent || s->t->is_client) && s->send_message_op) { // Nothing further will try to receive from this stream, so finish off // any outstanding send_message op - s->send_message_op->payload->send_message.send_message.reset(); + ResetSendMessage(s->send_message_op); s->send_message_op->payload->send_message.stream_write_closed = true; complete_if_batch_end_locked( s, new_err, s->send_message_op, @@ -831,7 +798,8 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { // We should schedule the recv_trailing_md_op completion if // 1. this stream is the client-side // 2. this stream is the server-side AND has already sent its trailing md - // (If the server hasn't already sent its trailing md, it doesn't have + // (If the server hasn't already sent its trailing md, it doesn't + // have // a final status, so don't mark this op complete) if (s->t->is_client || s->trailing_md_sent) { grpc_core::ExecCtx::Run( @@ -876,7 +844,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { // No further message will come on this stream, so finish off the // recv_message_op INPROC_LOG(GPR_INFO, "op_state_machine %p scheduling message-ready", s); - *s->recv_message_op->payload->recv_message.recv_message = nullptr; + s->recv_message_op->payload->recv_message.recv_message->reset(); grpc_core::ExecCtx::Run( DEBUG_LOCATION, s->recv_message_op->payload->recv_message.recv_message_ready, @@ -889,7 +857,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) { if (s->trailing_md_recvd && s->send_message_op && s->t->is_client) { // Nothing further will try to receive from this stream, so finish off // any outstanding send_message op - s->send_message_op->payload->send_message.send_message.reset(); + ResetSendMessage(s->send_message_op); complete_if_batch_end_locked( s, new_err, s->send_message_op, "op_state_machine scheduling send-message-on-complete case 3"); @@ -1102,7 +1070,7 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs, // Consume any send message that was sent here but that we are not // pushing to the other side if (op->send_message) { - op->payload->send_message.send_message.reset(); + ResetSendMessage(op); } // Schedule op's closures that we didn't push to op state machine if (op->recv_initial_metadata) { @@ -1138,10 +1106,10 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs, GRPC_ERROR_REF(error)); } if (op->recv_trailing_metadata) { - INPROC_LOG( - GPR_INFO, - "perform_stream_op error %p scheduling trailing-metadata-ready %s", - s, grpc_error_std_string(error).c_str()); + INPROC_LOG(GPR_INFO, + "perform_stream_op error %p scheduling " + "trailing-metadata-ready %s", + s, grpc_error_std_string(error).c_str()); grpc_core::ExecCtx::Run( DEBUG_LOCATION, op->payload->recv_trailing_metadata.recv_trailing_metadata_ready, diff --git a/src/core/lib/channel/call_tracer.h b/src/core/lib/channel/call_tracer.h index 58b4bd1d386..c473c354ba1 100644 --- a/src/core/lib/channel/call_tracer.h +++ b/src/core/lib/channel/call_tracer.h @@ -29,7 +29,7 @@ #include #include "src/core/lib/iomgr/error.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -58,13 +58,13 @@ class CallTracer { virtual void RecordOnDoneSendInitialMetadata(gpr_atm* peer_string) = 0; virtual void RecordSendTrailingMetadata( grpc_metadata_batch* send_trailing_metadata) = 0; - virtual void RecordSendMessage(const ByteStream& send_message) = 0; + virtual void RecordSendMessage(const SliceBuffer& send_message) = 0; // The `RecordReceivedInitialMetadata()` and `RecordReceivedMessage()` // methods should only be invoked when the metadata/message was // successfully received, i.e., without any error. virtual void RecordReceivedInitialMetadata( grpc_metadata_batch* recv_initial_metadata, uint32_t flags) = 0; - virtual void RecordReceivedMessage(const ByteStream& recv_message) = 0; + virtual void RecordReceivedMessage(const SliceBuffer& recv_message) = 0; // If the call was cancelled before the recv_trailing_metadata op // was started, recv_trailing_metadata and transport_stream_stats // will be null. diff --git a/src/core/lib/gprpp/manual_constructor.h b/src/core/lib/gprpp/manual_constructor.h index 81c3031f02f..95e57cd0e4c 100644 --- a/src/core/lib/gprpp/manual_constructor.h +++ b/src/core/lib/gprpp/manual_constructor.h @@ -25,12 +25,9 @@ #include -#include #include #include -#include - #include "src/core/lib/gprpp/construct_destruct.h" namespace grpc_core { @@ -102,70 +99,6 @@ class max_align_of { } // namespace manual_ctor_impl -template -class PolymorphicManualConstructor { - public: - // No constructor or destructor because one of the most useful uses of - // this class is as part of a union, and members of a union could not have - // constructors or destructors till C++11. And, anyway, the whole point of - // this class is to bypass constructor and destructor. - - BaseType* get() { return reinterpret_cast(&space_); } - const BaseType* get() const { - return reinterpret_cast(&space_); - } - - BaseType* operator->() { return get(); } - const BaseType* operator->() const { return get(); } - - BaseType& operator*() { return *get(); } - const BaseType& operator*() const { return *get(); } - - template - void Init() { - FinishInit(new (&space_) DerivedType); - } - - // Init() constructs the Type instance using the given arguments - // (which are forwarded to Type's constructor). - // - // Note that Init() with no arguments performs default-initialization, - // not zero-initialization (i.e it behaves the same as "new Type;", not - // "new Type();"), so it will leave non-class types uninitialized. - template - void Init(Ts&&... args) { - FinishInit(new (&space_) DerivedType(std::forward(args)...)); - } - - // Init() that is equivalent to copy and move construction. - // Enables usage like this: - // ManualConstructor> v; - // v.Init({1, 2, 3}); - template - void Init(const DerivedType& x) { - FinishInit(new (&space_) DerivedType(x)); - } - template - void Init(DerivedType&& x) { - FinishInit(new (&space_) DerivedType(std::forward(x))); - } - - void Destroy() { get()->~BaseType(); } - - private: - template - void FinishInit(DerivedType* p) { - static_assert( - manual_ctor_impl::is_one_of::value, - "DerivedType must be one of the predeclared DerivedTypes"); - GPR_ASSERT(static_cast(p) == p); - } - - typename std::aligned_storage< - manual_ctor_impl::max_size_of::value, - manual_ctor_impl::max_align_of::value>::type space_; -}; - template class ManualConstructor { public: diff --git a/src/core/lib/slice/slice_buffer.cc b/src/core/lib/slice/slice_buffer.cc index b6f33237a8f..2d2ee9fc995 100644 --- a/src/core/lib/slice/slice_buffer.cc +++ b/src/core/lib/slice/slice_buffer.cc @@ -51,10 +51,21 @@ void SliceBuffer::Prepend(Slice slice) { grpc_slice_buffer_undo_take_first(&slice_buffer_, slice.TakeCSlice()); } -Slice SliceBuffer::RefSlice(size_t index) { +Slice SliceBuffer::RefSlice(size_t index) const { return Slice(grpc_slice_ref_internal(slice_buffer_.slices[index])); } +std::string SliceBuffer::JoinIntoString() const { + std::string result; + result.reserve(slice_buffer_.length); + for (size_t i = 0; i < slice_buffer_.count; i++) { + result.append(reinterpret_cast( + GRPC_SLICE_START_PTR(slice_buffer_.slices[i])), + GRPC_SLICE_LENGTH(slice_buffer_.slices[i])); + } + return result; +} + } // namespace grpc_core /* grow a buffer; requires GRPC_SLICE_BUFFER_INLINE_ELEMENTS > 1 */ @@ -370,6 +381,24 @@ void grpc_slice_buffer_move_first_into_buffer(grpc_slice_buffer* src, size_t n, } } +void grpc_slice_buffer_copy_first_into_buffer(grpc_slice_buffer* src, size_t n, + void* dst) { + uint8_t* dstp = static_cast(dst); + GPR_ASSERT(src->length >= n); + + for (size_t i = 0; i < src->count; i++) { + grpc_slice slice = src->slices[i]; + size_t slice_len = GRPC_SLICE_LENGTH(slice); + if (slice_len >= n) { + memcpy(dstp, GRPC_SLICE_START_PTR(slice), n); + return; + } + memcpy(dstp, GRPC_SLICE_START_PTR(slice), slice_len); + dstp += slice_len; + n -= slice_len; + } +} + void grpc_slice_buffer_trim_end(grpc_slice_buffer* sb, size_t n, grpc_slice_buffer* garbage) { GPR_ASSERT(n <= sb->length); diff --git a/src/core/lib/slice/slice_buffer.h b/src/core/lib/slice/slice_buffer.h index 9effb828e88..d08b08cc1f1 100644 --- a/src/core/lib/slice/slice_buffer.h +++ b/src/core/lib/slice/slice_buffer.h @@ -19,6 +19,8 @@ #include +#include + #include #include @@ -45,14 +47,15 @@ class SliceBuffer { SliceBuffer(const SliceBuffer& other) = delete; SliceBuffer(SliceBuffer&& other) noexcept { grpc_slice_buffer_init(&slice_buffer_); - grpc_slice_buffer_move_into(&slice_buffer_, &other.slice_buffer_); + grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_); } /// Upon destruction, the underlying raw slice buffer is cleaned out and all /// slices are unreffed. ~SliceBuffer() { grpc_slice_buffer_destroy(&slice_buffer_); } + SliceBuffer& operator=(const SliceBuffer&) = delete; SliceBuffer& operator=(SliceBuffer&& other) noexcept { - grpc_slice_buffer_move_into(&slice_buffer_, &other.slice_buffer_); + grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_); return *this; } @@ -65,7 +68,7 @@ class SliceBuffer { size_t AppendIndexed(Slice slice); /// Returns the number of slices held by the SliceBuffer. - size_t Count() { return slice_buffer_.count; } + size_t Count() const { return slice_buffer_.count; } /// Removes/deletes the last n bytes in the SliceBuffer. void RemoveLastNBytes(size_t n) { @@ -88,13 +91,37 @@ class SliceBuffer { /// Increased the ref-count of slice at the specified index and returns the /// associated slice. - Slice RefSlice(size_t index); + Slice RefSlice(size_t index) const; /// The total number of bytes held by the SliceBuffer - size_t Length() { return slice_buffer_.length; } + size_t Length() const { return slice_buffer_.length; } + + /// Swap with another slice buffer + void Swap(SliceBuffer* other) { + grpc_slice_buffer_swap(c_slice_buffer(), other->c_slice_buffer()); + } + + /// Concatenate all slices and return the resulting string. + std::string JoinIntoString() const; + + // Return a copy of the slice buffer + SliceBuffer Copy() const { + SliceBuffer copy; + for (size_t i = 0; i < Count(); i++) { + copy.Append(RefSlice(i)); + } + return copy; + } /// Return a pointer to the back raw grpc_slice_buffer - grpc_slice_buffer* RawSliceBuffer() { return &slice_buffer_; } + grpc_slice_buffer* c_slice_buffer() { return &slice_buffer_; } + + /// Return a pointer to the back raw grpc_slice_buffer + const grpc_slice_buffer* c_slice_buffer() const { return &slice_buffer_; } + + const grpc_slice& c_slice_at(size_t index) { + return slice_buffer_.slices[index]; + } private: /// The backing raw slice buffer. @@ -103,4 +130,8 @@ class SliceBuffer { } // namespace grpc_core +// Copy the first n bytes of src into memory pointed to by dst. +void grpc_slice_buffer_copy_first_into_buffer(grpc_slice_buffer* src, size_t n, + void* dst); + #endif // GRPC_CORE_LIB_SLICE_SLICE_BUFFER_H diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 3405842d46a..39a5d356307 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -25,7 +25,6 @@ #include #include -#include #include #include #include @@ -59,8 +58,6 @@ #include "src/core/lib/gpr/time_precise.h" #include "src/core/lib/gprpp/cpp_impl_of.h" #include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/call_combiner.h" @@ -68,6 +65,7 @@ #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/resource_quota/arena.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/surface/api_trace.h" @@ -76,7 +74,6 @@ #include "src/core/lib/surface/completion_queue.h" #include "src/core/lib/surface/server.h" #include "src/core/lib/surface/validate_metadata.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -309,8 +306,6 @@ class FilterStackCall final : public Call { void PostCompletion(); void FinishStep(); - void ContinueReceivingSlices(); - void ReceivingSliceReady(grpc_error_handle error); void ProcessDataAfterMetadata(); void ReceivingStreamReady(grpc_error_handle error); void ValidateFilteredMetadata(); @@ -399,13 +394,13 @@ class FilterStackCall final : public Call { /* Contexts for various subsystems (security, tracing, ...). */ grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {}; - ManualConstructor sending_stream_; + SliceBuffer send_slice_buffer_; + absl::optional receiving_slice_buffer_; + uint32_t receiving_stream_flags_; - OrphanablePtr receiving_stream_; bool call_failed_before_recv_message_ = false; grpc_byte_buffer** receiving_buffer_ = nullptr; grpc_slice receiving_slice_ = grpc_empty_slice(); - grpc_closure receiving_slice_ready_; grpc_closure receiving_stream_ready_; grpc_closure receiving_initial_metadata_ready_; grpc_closure receiving_trailing_metadata_ready_; @@ -652,7 +647,7 @@ void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) { auto* c = static_cast(call); c->recv_initial_metadata_.Clear(); c->recv_trailing_metadata_.Clear(); - c->receiving_stream_.reset(); + c->receiving_slice_buffer_.reset(); ParentCall* pc = c->parent_call(); if (pc != nullptr) { pc->~ParentCall(); @@ -1088,6 +1083,7 @@ void FilterStackCall::BatchControl::PostCompletion() { "Attempt to send message after stream was closed.")); } call->sending_message_ = false; + call->send_slice_buffer_.Clear(); } if (op_.send_trailing_metadata) { call->send_trailing_metadata_.Clear(); @@ -1133,95 +1129,27 @@ void FilterStackCall::BatchControl::FinishStep() { } } -void FilterStackCall::BatchControl::ContinueReceivingSlices() { - grpc_error_handle error; - FilterStackCall* call = call_; - for (;;) { - size_t remaining = call->receiving_stream_->length() - - (*call->receiving_buffer_)->data.raw.slice_buffer.length; - if (remaining == 0) { - call->receiving_message_ = false; - call->receiving_stream_.reset(); - FinishStep(); - return; - } - if (call->receiving_stream_->Next(remaining, - &call->receiving_slice_ready_)) { - error = call->receiving_stream_->Pull(&call->receiving_slice_); - if (error == GRPC_ERROR_NONE) { - grpc_slice_buffer_add( - &(*call->receiving_buffer_)->data.raw.slice_buffer, - call->receiving_slice_); - } else { - call->receiving_stream_.reset(); - grpc_byte_buffer_destroy(*call->receiving_buffer_); - *call->receiving_buffer_ = nullptr; - call->receiving_message_ = false; - FinishStep(); - GRPC_ERROR_UNREF(error); - return; - } - } else { - return; - } - } -} - -void FilterStackCall::BatchControl::ReceivingSliceReady( - grpc_error_handle error) { - FilterStackCall* call = call_; - bool release_error = false; - - if (error == GRPC_ERROR_NONE) { - grpc_slice slice; - error = call->receiving_stream_->Pull(&slice); - if (error == GRPC_ERROR_NONE) { - grpc_slice_buffer_add(&(*call->receiving_buffer_)->data.raw.slice_buffer, - slice); - ContinueReceivingSlices(); - } else { - /* Error returned by ByteStream::Pull() needs to be released manually */ - release_error = true; - } - } - - if (error != GRPC_ERROR_NONE) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures)) { - GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error)); - } - call->receiving_stream_.reset(); - grpc_byte_buffer_destroy(*call->receiving_buffer_); - *call->receiving_buffer_ = nullptr; - call->receiving_message_ = false; - FinishStep(); - if (release_error) { - GRPC_ERROR_UNREF(error); - } - } -} - void FilterStackCall::BatchControl::ProcessDataAfterMetadata() { FilterStackCall* call = call_; - if (call->receiving_stream_ == nullptr) { + if (!call->receiving_slice_buffer_.has_value()) { *call->receiving_buffer_ = nullptr; call->receiving_message_ = false; FinishStep(); } else { - call->test_only_last_message_flags_ = call->receiving_stream_->flags(); - if ((call->receiving_stream_->flags() & GRPC_WRITE_INTERNAL_COMPRESS) && + call->test_only_last_message_flags_ = call->receiving_stream_flags_; + if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) && (call->incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) { *call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create( nullptr, 0, call->incoming_compression_algorithm_); } else { *call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0); } - GRPC_CLOSURE_INIT( - &call->receiving_slice_ready_, - [](void* bctl, grpc_error_handle error) { - static_cast(bctl)->ReceivingSliceReady(error); - }, - this, grpc_schedule_on_exec_ctx); - ContinueReceivingSlices(); + grpc_slice_buffer_move_into( + call->receiving_slice_buffer_->c_slice_buffer(), + &(*call->receiving_buffer_)->data.raw.slice_buffer); + call->receiving_message_ = false; + call->receiving_slice_buffer_.reset(); + FinishStep(); } } @@ -1229,7 +1157,7 @@ void FilterStackCall::BatchControl::ReceivingStreamReady( grpc_error_handle error) { FilterStackCall* call = call_; if (!GRPC_ERROR_IS_NONE(error)) { - call->receiving_stream_.reset(); + call->receiving_slice_buffer_.reset(); if (batch_error_.ok()) { batch_error_.set(error); } @@ -1238,7 +1166,8 @@ void FilterStackCall::BatchControl::ReceivingStreamReady( /* If recv_state is kRecvNone, we will save the batch_control * object with rel_cas, and will not use it after the cas. Its corresponding * acq_load is in receiving_initial_metadata_ready() */ - if (!GRPC_ERROR_IS_NONE(error) || call->receiving_stream_ == nullptr || + if (!GRPC_ERROR_IS_NONE(error) || + !call->receiving_slice_buffer_.has_value() || !gpr_atm_rel_cas(&call->recv_state_, kRecvNone, reinterpret_cast(this))) { ProcessDataAfterMetadata(); @@ -1519,10 +1448,12 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, } stream_op->send_message = true; sending_message_ = true; - sending_stream_.Init( - &op->data.send_message.send_message->data.raw.slice_buffer, flags); - stream_op_payload->send_message.send_message.reset( - sending_stream_.get()); + send_slice_buffer_.Clear(); + grpc_slice_buffer_move_into( + &op->data.send_message.send_message->data.raw.slice_buffer, + send_slice_buffer_.c_slice_buffer()); + stream_op_payload->send_message.flags = flags; + stream_op_payload->send_message.send_message = &send_slice_buffer_; has_send_ops = true; break; } @@ -1659,8 +1590,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, } receiving_message_ = true; stream_op->recv_message = true; + receiving_slice_buffer_.reset(); receiving_buffer_ = op->data.recv_message.recv_message; - stream_op_payload->recv_message.recv_message = &receiving_stream_; + stream_op_payload->recv_message.recv_message = &receiving_slice_buffer_; + receiving_stream_flags_ = 0; + stream_op_payload->recv_message.flags = &receiving_stream_flags_; stream_op_payload->recv_message.call_failed_before_recv_message = &call_failed_before_recv_message_; GRPC_CLOSURE_INIT( @@ -1784,10 +1718,6 @@ done_with_error: } if (stream_op->send_message) { sending_message_ = false; - // No need to invoke call->sending_stream->Orphan() explicitly. - // stream_op_payload->send_message.send_message.reset() calls Deletor - // of call->sending_stream which in-turn invokes the Orphan() method. - stream_op_payload->send_message.send_message.reset(); } if (stream_op->send_trailing_metadata) { sent_final_op_ = false; diff --git a/src/core/lib/transport/byte_stream.cc b/src/core/lib/transport/byte_stream.cc deleted file mode 100644 index ac49481f4dc..00000000000 --- a/src/core/lib/transport/byte_stream.cc +++ /dev/null @@ -1,167 +0,0 @@ -/* - * - * Copyright 2015 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include - -#include "src/core/lib/transport/byte_stream.h" - -#include -#include - -#include "absl/status/status.h" - -#include -#include - -#include "src/core/lib/slice/slice_internal.h" -#include "src/core/lib/slice/slice_refcount.h" - -namespace grpc_core { - -// -// SliceBufferByteStream -// - -SliceBufferByteStream::SliceBufferByteStream(grpc_slice_buffer* slice_buffer, - uint32_t flags) - : ByteStream(static_cast(slice_buffer->length), flags) { - GPR_ASSERT(slice_buffer->length <= UINT32_MAX); - grpc_slice_buffer_init(&backing_buffer_); - grpc_slice_buffer_swap(slice_buffer, &backing_buffer_); - if (backing_buffer_.count == 0) { - grpc_slice_buffer_add_indexed(&backing_buffer_, grpc_empty_slice()); - GPR_ASSERT(backing_buffer_.count > 0); - } -} - -SliceBufferByteStream::~SliceBufferByteStream() {} - -void SliceBufferByteStream::Orphan() { - grpc_slice_buffer_destroy_internal(&backing_buffer_); - GRPC_ERROR_UNREF(shutdown_error_); - shutdown_error_ = GRPC_ERROR_NONE; - // Note: We do not actually delete the object here, since - // SliceBufferByteStream is usually allocated as part of a larger - // object and has an OrphanablePtr of itself passed down through the - // filter stack. -} - -bool SliceBufferByteStream::Next(size_t /*max_size_hint*/, - grpc_closure* /*on_complete*/) { - GPR_DEBUG_ASSERT(backing_buffer_.count > 0); - return true; -} - -grpc_error_handle SliceBufferByteStream::Pull(grpc_slice* slice) { - if (GPR_UNLIKELY(shutdown_error_ != GRPC_ERROR_NONE)) { - return GRPC_ERROR_REF(shutdown_error_); - } - *slice = grpc_slice_buffer_take_first(&backing_buffer_); - return GRPC_ERROR_NONE; -} - -void SliceBufferByteStream::Shutdown(grpc_error_handle error) { - GRPC_ERROR_UNREF(shutdown_error_); - shutdown_error_ = error; -} - -// -// ByteStreamCache -// - -ByteStreamCache::ByteStreamCache(OrphanablePtr underlying_stream) - : underlying_stream_(std::move(underlying_stream)), - length_(underlying_stream_->length()), - flags_(underlying_stream_->flags()) { - grpc_slice_buffer_init(&cache_buffer_); -} - -ByteStreamCache::~ByteStreamCache() { Destroy(); } - -void ByteStreamCache::Destroy() { - underlying_stream_.reset(); - if (cache_buffer_.length > 0) { - grpc_slice_buffer_destroy_internal(&cache_buffer_); - } -} - -// -// ByteStreamCache::CachingByteStream -// - -ByteStreamCache::CachingByteStream::CachingByteStream(ByteStreamCache* cache) - : ByteStream(cache->length_, cache->flags_), cache_(cache) {} - -ByteStreamCache::CachingByteStream::~CachingByteStream() {} - -void ByteStreamCache::CachingByteStream::Orphan() { - GRPC_ERROR_UNREF(shutdown_error_); - shutdown_error_ = GRPC_ERROR_NONE; - // Note: We do not actually delete the object here, since - // CachingByteStream is usually allocated as part of a larger - // object and has an OrphanablePtr of itself passed down through the - // filter stack. -} - -bool ByteStreamCache::CachingByteStream::Next(size_t max_size_hint, - grpc_closure* on_complete) { - if (shutdown_error_ != GRPC_ERROR_NONE) return true; - if (cursor_ < cache_->cache_buffer_.count) return true; - GPR_ASSERT(cache_->underlying_stream_ != nullptr); - return cache_->underlying_stream_->Next(max_size_hint, on_complete); -} - -grpc_error_handle ByteStreamCache::CachingByteStream::Pull(grpc_slice* slice) { - if (shutdown_error_ != GRPC_ERROR_NONE) { - return GRPC_ERROR_REF(shutdown_error_); - } - if (cursor_ < cache_->cache_buffer_.count) { - *slice = grpc_slice_ref_internal(cache_->cache_buffer_.slices[cursor_]); - ++cursor_; - offset_ += GRPC_SLICE_LENGTH(*slice); - return GRPC_ERROR_NONE; - } - GPR_ASSERT(cache_->underlying_stream_ != nullptr); - grpc_error_handle error = cache_->underlying_stream_->Pull(slice); - if (error == GRPC_ERROR_NONE) { - grpc_slice_buffer_add(&cache_->cache_buffer_, - grpc_slice_ref_internal(*slice)); - ++cursor_; - offset_ += GRPC_SLICE_LENGTH(*slice); - // Orphan the underlying stream if it's been drained. - if (offset_ == cache_->underlying_stream_->length()) { - cache_->underlying_stream_.reset(); - } - } - return error; -} - -void ByteStreamCache::CachingByteStream::Shutdown(grpc_error_handle error) { - GRPC_ERROR_UNREF(shutdown_error_); - shutdown_error_ = GRPC_ERROR_REF(error); - if (cache_->underlying_stream_ != nullptr) { - cache_->underlying_stream_->Shutdown(error); - } -} - -void ByteStreamCache::CachingByteStream::Reset() { - cursor_ = 0; - offset_ = 0; -} - -} // namespace grpc_core diff --git a/src/core/lib/transport/byte_stream.h b/src/core/lib/transport/byte_stream.h deleted file mode 100644 index 6c5d6e4d23b..00000000000 --- a/src/core/lib/transport/byte_stream.h +++ /dev/null @@ -1,170 +0,0 @@ -/* - * - * Copyright 2015 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H -#define GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H - -#include - -#include -#include - -#include - -#include "src/core/lib/gprpp/orphanable.h" -#include "src/core/lib/iomgr/closure.h" -#include "src/core/lib/iomgr/error.h" - -/** Internal bit flag for grpc_begin_message's \a flags signaling the use of - * compression for the message. (Does not apply for stream compression.) */ -#define GRPC_WRITE_INTERNAL_COMPRESS (0x80000000u) -/** Internal bit flag for determining whether the message was compressed and had - * to be decompressed by the message_decompress filter. (Does not apply for - * stream compression.) */ -#define GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED (0x40000000u) -/** Mask of all valid internal flags. */ -#define GRPC_WRITE_INTERNAL_USED_MASK \ - (GRPC_WRITE_INTERNAL_COMPRESS | GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED) - -namespace grpc_core { - -class ByteStream : public Orphanable { - public: - ~ByteStream() override {} - - // Returns true if the bytes are available immediately (in which case - // on_complete will not be called), or false if the bytes will be available - // asynchronously (in which case on_complete will be called when they - // are available). Should not be called if there is no data left on the - // stream. - // - // max_size_hint can be set as a hint as to the maximum number - // of bytes that would be acceptable to read. - virtual bool Next(size_t max_size_hint, grpc_closure* on_complete) = 0; - - // Returns the next slice in the byte stream when it is available, as - // indicated by Next(). - // - // Once a slice is returned into *slice, it is owned by the caller. - virtual grpc_error_handle Pull(grpc_slice* slice) = 0; - - // Shuts down the byte stream. - // - // If there is a pending call to on_complete from Next(), it will be - // invoked with the error passed to Shutdown(). - // - // The next call to Pull() (if any) will return the error passed to - // Shutdown(). - virtual void Shutdown(grpc_error_handle error) = 0; - - uint32_t length() const { return length_; } - uint32_t flags() const { return flags_; } - - void set_flags(uint32_t flags) { flags_ = flags; } - - protected: - ByteStream(uint32_t length, uint32_t flags) - : length_(length), flags_(flags) {} - - private: - const uint32_t length_; - uint32_t flags_; -}; - -// -// SliceBufferByteStream -// -// A ByteStream that wraps a slice buffer. -// - -class SliceBufferByteStream : public ByteStream { - public: - // Removes all slices in slice_buffer, leaving it empty. - SliceBufferByteStream(grpc_slice_buffer* slice_buffer, uint32_t flags); - - ~SliceBufferByteStream() override; - - void Orphan() override; - - bool Next(size_t max_size_hint, grpc_closure* on_complete) override; - grpc_error_handle Pull(grpc_slice* slice) override; - void Shutdown(grpc_error_handle error) override; - - private: - grpc_error_handle shutdown_error_ = GRPC_ERROR_NONE; - grpc_slice_buffer backing_buffer_; -}; - -// -// CachingByteStream -// -// A ByteStream that that wraps an underlying byte stream but caches -// the resulting slices in a slice buffer. If an initial attempt fails -// without fully draining the underlying stream, a new caching stream -// can be created from the same underlying cache, in which case it will -// return whatever is in the backing buffer before continuing to read the -// underlying stream. -// -// NOTE: No synchronization is done, so it is not safe to have multiple -// CachingByteStreams simultaneously drawing from the same underlying -// ByteStreamCache at the same time. -// - -class ByteStreamCache { - public: - class CachingByteStream : public ByteStream { - public: - explicit CachingByteStream(ByteStreamCache* cache); - - ~CachingByteStream() override; - - void Orphan() override; - - bool Next(size_t max_size_hint, grpc_closure* on_complete) override; - grpc_error_handle Pull(grpc_slice* slice) override; - void Shutdown(grpc_error_handle error) override; - - // Resets the byte stream to the start of the underlying stream. - void Reset(); - - private: - ByteStreamCache* cache_; - size_t cursor_ = 0; - size_t offset_ = 0; - grpc_error_handle shutdown_error_ = GRPC_ERROR_NONE; - }; - - explicit ByteStreamCache(OrphanablePtr underlying_stream); - - ~ByteStreamCache(); - - // Must not be destroyed while still in use by a CachingByteStream. - void Destroy(); - - grpc_slice_buffer* cache_buffer() { return &cache_buffer_; } - - private: - OrphanablePtr underlying_stream_; - uint32_t length_; - uint32_t flags_; - grpc_slice_buffer cache_buffer_; -}; - -} // namespace grpc_core - -#endif /* GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H */ diff --git a/src/core/lib/transport/transport.cc b/src/core/lib/transport/transport.cc index b7c4950acb7..35e28349aed 100644 --- a/src/core/lib/transport/transport.cc +++ b/src/core/lib/transport/transport.cc @@ -161,9 +161,6 @@ void grpc_transport_stream_op_batch_finish_with_failure( void grpc_transport_stream_op_batch_queue_finish_with_failure( grpc_transport_stream_op_batch* batch, grpc_error_handle error, grpc_core::CallCombinerClosureList* closures) { - if (batch->send_message) { - batch->payload->send_message.send_message.reset(); - } if (batch->cancel_stream) { GRPC_ERROR_UNREF(batch->payload->cancel_stream.cancel_error); } diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 069c53df87f..c9b4bc96f89 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -55,7 +55,7 @@ #include "src/core/lib/promise/latch.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport_fwd.h" @@ -70,6 +70,17 @@ struct grpc_transport_stream_op_batch_payload; #define GRPC_ARG_TRANSPORT "grpc.internal.transport" +/** Internal bit flag for grpc_begin_message's \a flags signaling the use of + * compression for the message. (Does not apply for stream compression.) */ +#define GRPC_WRITE_INTERNAL_COMPRESS (0x80000000u) +/** Internal bit flag for determining whether the message was compressed and had + * to be decompressed by the message_decompress filter. (Does not apply for + * stream compression.) */ +#define GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED (0x40000000u) +/** Mask of all valid internal flags. */ +#define GRPC_WRITE_INTERNAL_USED_MASK \ + (GRPC_WRITE_INTERNAL_COMPRESS | GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED) + namespace grpc_core { // TODO(ctiller): eliminate once MetadataHandle is constructable directly. namespace promise_filter_detail { @@ -331,12 +342,6 @@ struct grpc_transport_stream_op_batch_payload { explicit grpc_transport_stream_op_batch_payload( grpc_call_context_element* context) : context(context) {} - ~grpc_transport_stream_op_batch_payload() { - // We don't really own `send_message`, so release ownership and let the - // owner clean the data. - (void)send_message.send_message.release(); - } - struct { grpc_metadata_batch* send_initial_metadata = nullptr; /** Iff send_initial_metadata != NULL, flags associated with @@ -365,7 +370,8 @@ struct grpc_transport_stream_op_batch_payload { // the op gets down to the transport) takes ownership. // The batch's on_complete will not be called until after the byte // stream is orphaned. - grpc_core::OrphanablePtr send_message; + grpc_core::SliceBuffer* send_message; + uint32_t flags = 0; // Set by the transport if the stream has been closed for writes. If this // is set and send message op is present, we set the operation to be a // failure without sending a cancel OP down the stack. This is so that the @@ -404,10 +410,11 @@ struct grpc_transport_stream_op_batch_payload { } recv_initial_metadata; struct { - // Will be set by the transport to point to the byte stream - // containing a received message. - // Will be NULL if trailing metadata is received instead of a message. - grpc_core::OrphanablePtr* recv_message = nullptr; + // Will be set by the transport to point to the byte stream containing a + // received message. Will be nullopt if trailing metadata is received + // instead of a message. + absl::optional* recv_message = nullptr; + uint32_t* flags = nullptr; // Was this recv_message failed for reasons other than a clean end-of-stream bool* call_failed_before_recv_message = nullptr; /** Should be enqueued when one message is ready to be processed. */ diff --git a/src/core/lib/transport/transport_op_string.cc b/src/core/lib/transport/transport_op_string.cc index 8889f274e25..42ea6e34b58 100644 --- a/src/core/lib/transport/transport_op_string.cc +++ b/src/core/lib/transport/transport_op_string.cc @@ -33,7 +33,7 @@ #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/error.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -54,10 +54,9 @@ std::string grpc_transport_stream_op_batch_string( if (op->send_message) { if (op->payload->send_message.send_message != nullptr) { - out.push_back( - absl::StrFormat(" SEND_MESSAGE:flags=0x%08x:len=%d", - op->payload->send_message.send_message->flags(), - op->payload->send_message.send_message->length())); + out.push_back(absl::StrFormat( + " SEND_MESSAGE:flags=0x%08x:len=%d", op->payload->send_message.flags, + op->payload->send_message.send_message->Length())); } else { // This can happen when we check a batch after the transport has // processed and cleared the send_message op. diff --git a/src/cpp/common/channel_filter.h b/src/cpp/common/channel_filter.h index fed61128d0a..3f0e62e50f2 100644 --- a/src/cpp/common/channel_filter.h +++ b/src/cpp/common/channel_filter.h @@ -27,6 +27,8 @@ #include #include +#include "absl/types/optional.h" + #include #include #include @@ -35,12 +37,11 @@ #include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/context.h" -#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/polling_entity.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/channel_stack_type.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -152,22 +153,21 @@ class TransportStreamOpBatch { op_->payload->recv_initial_metadata.recv_initial_metadata_ready = closure; } - grpc_core::OrphanablePtr* send_message() const { - return op_->send_message ? &op_->payload->send_message.send_message + grpc_core::SliceBuffer* send_message() const { + return op_->send_message ? op_->payload->send_message.send_message : nullptr; } - void set_send_message( - grpc_core::OrphanablePtr send_message) { + + void set_send_message(grpc_core::SliceBuffer* send_message) { op_->send_message = true; - op_->payload->send_message.send_message = std::move(send_message); + op_->payload->send_message.send_message = send_message; } - grpc_core::OrphanablePtr* recv_message() const { + absl::optional* recv_message() const { return op_->recv_message ? op_->payload->recv_message.recv_message : nullptr; } - void set_recv_message( - grpc_core::OrphanablePtr* recv_message) { + void set_recv_message(absl::optional* recv_message) { op_->recv_message = true; op_->payload->recv_message.recv_message = recv_message; } diff --git a/src/cpp/ext/filters/census/client_filter.cc b/src/cpp/ext/filters/census/client_filter.cc index 9ad447b4237..410d7399807 100644 --- a/src/cpp/ext/filters/census/client_filter.cc +++ b/src/cpp/ext/filters/census/client_filter.cc @@ -50,8 +50,8 @@ #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_refcount.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" #include "src/cpp/ext/filters/census/context.h" @@ -137,12 +137,12 @@ void OpenCensusCallTracer::OpenCensusCallAttemptTracer:: } void OpenCensusCallTracer::OpenCensusCallAttemptTracer::RecordSendMessage( - const grpc_core::ByteStream& /*send_message*/) { + const grpc_core::SliceBuffer& /*send_message*/) { ++sent_message_count_; } void OpenCensusCallTracer::OpenCensusCallAttemptTracer::RecordReceivedMessage( - const grpc_core::ByteStream& /*recv_message*/) { + const grpc_core::SliceBuffer& /*recv_message*/) { ++recv_message_count_; } diff --git a/src/cpp/ext/filters/census/open_census_call_tracer.h b/src/cpp/ext/filters/census/open_census_call_tracer.h index 990630c17ed..f2f297f8600 100644 --- a/src/cpp/ext/filters/census/open_census_call_tracer.h +++ b/src/cpp/ext/filters/census/open_census_call_tracer.h @@ -38,7 +38,7 @@ #include "src/core/lib/iomgr/error.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" #include "src/cpp/ext/filters/census/context.h" @@ -58,12 +58,12 @@ class OpenCensusCallTracer : public grpc_core::CallTracer { void RecordSendTrailingMetadata( grpc_metadata_batch* /*send_trailing_metadata*/) override {} void RecordSendMessage( - const grpc_core::ByteStream& /*send_message*/) override; + const grpc_core::SliceBuffer& /*send_message*/) override; void RecordReceivedInitialMetadata( grpc_metadata_batch* /*recv_initial_metadata*/, uint32_t /*flags*/) override {} void RecordReceivedMessage( - const grpc_core::ByteStream& /*recv_message*/) override; + const grpc_core::SliceBuffer& /*recv_message*/) override; void RecordReceivedTrailingMetadata( absl::Status status, grpc_metadata_batch* recv_trailing_metadata, const grpc_transport_stream_stats* transport_stream_stats) override; diff --git a/src/cpp/ext/filters/census/server_filter.cc b/src/cpp/ext/filters/census/server_filter.cc index a64f339b2fa..4de1216093f 100644 --- a/src/cpp/ext/filters/census/server_filter.cc +++ b/src/cpp/ext/filters/census/server_filter.cc @@ -20,7 +20,6 @@ #include "src/cpp/ext/filters/census/server_filter.h" -#include #include #include "absl/strings/str_cat.h" @@ -82,7 +81,7 @@ void CensusServerCallData::OnDoneRecvMessageCb(void* user_data, GPR_ASSERT(calld != nullptr); GPR_ASSERT(channeld != nullptr); // Stream messages are no longer valid after receiving trailing metadata. - if ((*calld->recv_message_) != nullptr) { + if (calld->recv_message_->has_value()) { ++calld->recv_message_count_; } grpc_core::Closure::Run(DEBUG_LOCATION, calld->initial_on_done_recv_message_, diff --git a/src/cpp/ext/filters/census/server_filter.h b/src/cpp/ext/filters/census/server_filter.h index d5cf021d615..55e542cb5d7 100644 --- a/src/cpp/ext/filters/census/server_filter.h +++ b/src/cpp/ext/filters/census/server_filter.h @@ -28,6 +28,7 @@ #include "absl/strings/string_view.h" #include "absl/time/time.h" +#include "absl/types/optional.h" #include #include @@ -35,11 +36,10 @@ #include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/cpp/common/channel_filter.h" #include "src/cpp/ext/filters/census/context.h" @@ -101,7 +101,7 @@ class CensusServerCallData : public CallData { grpc_closure on_done_recv_message_; absl::Time start_time_; absl::Duration elapsed_time_; - grpc_core::OrphanablePtr* recv_message_; + absl::optional* recv_message_; uint64_t recv_message_count_; uint64_t sent_message_count_; // Buffer needed for grpc_slice to reference it when adding metatdata to diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 3d63c967592..c80ed184d6a 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -683,7 +683,6 @@ CORE_SOURCE_FILES = [ 'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', - 'src/core/lib/transport/byte_stream.cc', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/handshaker.cc', diff --git a/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py b/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py index 565a79814cd..34c6d15eb5e 100644 --- a/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py +++ b/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py @@ -374,18 +374,25 @@ class ChannelzServicerTest(unittest.TestCase): # Subchannel exists self.assertGreater(len(gc_resp.channel.subchannel_ref), 0) - gsc_resp = self._channelz_stub.GetSubchannel( - channelz_pb2.GetSubchannelRequest( - subchannel_id=gc_resp.channel.subchannel_ref[0].subchannel_id)) + while True: + gsc_resp = self._channelz_stub.GetSubchannel( + channelz_pb2.GetSubchannelRequest( + subchannel_id=gc_resp.channel.subchannel_ref[0]. + subchannel_id)) + if gsc_resp.subchannel.data.calls_started == gsc_resp.subchannel.data.calls_succeeded + gsc_resp.subchannel.data.calls_failed: + break self.assertEqual(gsc_resp.subchannel.data.calls_started, 1) - self.assertEqual(gsc_resp.subchannel.data.calls_succeeded, 1) self.assertEqual(gsc_resp.subchannel.data.calls_failed, 0) + self.assertEqual(gsc_resp.subchannel.data.calls_succeeded, 1) # Socket exists self.assertEqual(len(gsc_resp.subchannel.socket_ref), 1) - gs_resp = self._channelz_stub.GetSocket( - channelz_pb2.GetSocketRequest( - socket_id=gsc_resp.subchannel.socket_ref[0].socket_id)) + while True: + gs_resp = self._channelz_stub.GetSocket( + channelz_pb2.GetSocketRequest( + socket_id=gsc_resp.subchannel.socket_ref[0].socket_id)) + if gs_resp.socket.data.streams_started == gs_resp.socket.data.streams_succeeded + gs_resp.socket.data.streams_failed: + break self.assertEqual(gs_resp.socket.data.streams_started, 1) self.assertEqual(gs_resp.socket.data.streams_succeeded, 1) self.assertEqual(gs_resp.socket.data.streams_failed, 0) diff --git a/src/python/grpcio_tests/tests/unit/_compression_test.py b/src/python/grpcio_tests/tests/unit/_compression_test.py index d58f4dc06ba..30a5d6bf685 100644 --- a/src/python/grpcio_tests/tests/unit/_compression_test.py +++ b/src/python/grpcio_tests/tests/unit/_compression_test.py @@ -200,24 +200,15 @@ def _get_compression_ratios(client_function, first_channel_kwargs, first_server_handler, second_channel_kwargs, second_multicallable_kwargs, second_server_kwargs, second_server_handler, message): - try: - # This test requires the byte length of each connection to be deterministic. As - # it turns out, flow control puts bytes on the wire in a nondeterministic - # manner. We disable it here in order to measure compression ratios - # deterministically. - os.environ['GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL'] = 'true' - first_bytes_sent, first_bytes_received = _get_byte_counts( - first_channel_kwargs, first_multicallable_kwargs, client_function, - first_server_kwargs, first_server_handler, message) - second_bytes_sent, second_bytes_received = _get_byte_counts( - second_channel_kwargs, second_multicallable_kwargs, client_function, - second_server_kwargs, second_server_handler, message) - return ((second_bytes_sent - first_bytes_sent) / - float(first_bytes_sent), - (second_bytes_received - first_bytes_received) / - float(first_bytes_received)) - finally: - del os.environ['GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL'] + first_bytes_sent, first_bytes_received = _get_byte_counts( + first_channel_kwargs, first_multicallable_kwargs, client_function, + first_server_kwargs, first_server_handler, message) + second_bytes_sent, second_bytes_received = _get_byte_counts( + second_channel_kwargs, second_multicallable_kwargs, client_function, + second_server_kwargs, second_server_handler, message) + return ((second_bytes_sent - first_bytes_sent) / float(first_bytes_sent), + (second_bytes_received - first_bytes_received) / + float(first_bytes_received)) def _unary_unary_client(channel, multicallable_kwargs, message): @@ -302,37 +293,28 @@ class CompressionTest(unittest.TestCase): server_handler = _GenericHandler( functools.partial(set_call_compression, grpc.Compression.Gzip) ) if server_call_compression else _GenericHandler(None) - sent_ratio, received_ratio = _get_compression_ratios( - client_function, {}, {}, {}, _GenericHandler(None), channel_kwargs, - multicallable_kwargs, server_kwargs, server_handler, _REQUEST) - - if client_side_compressed: - self.assertCompressed(sent_ratio) - else: - self.assertNotCompressed(sent_ratio) - - if server_side_compressed: - self.assertCompressed(received_ratio) - else: - self.assertNotCompressed(received_ratio) + _get_compression_ratios(client_function, {}, {}, {}, + _GenericHandler(None), channel_kwargs, + multicallable_kwargs, server_kwargs, + server_handler, _REQUEST) def testDisableNextCompressionStreaming(self): server_kwargs = { 'compression': grpc.Compression.Deflate, } - _, received_ratio = _get_compression_ratios( - _stream_stream_client, {}, {}, {}, _GenericHandler(None), {}, {}, - server_kwargs, _GenericHandler(disable_next_compression), _REQUEST) - self.assertNotCompressed(received_ratio) + _get_compression_ratios(_stream_stream_client, {}, {}, {}, + _GenericHandler(None), {}, {}, server_kwargs, + _GenericHandler(disable_next_compression), + _REQUEST) def testDisableNextCompressionStreamingResets(self): server_kwargs = { 'compression': grpc.Compression.Deflate, } - _, received_ratio = _get_compression_ratios( - _stream_stream_client, {}, {}, {}, _GenericHandler(None), {}, {}, - server_kwargs, _GenericHandler(disable_first_compression), _REQUEST) - self.assertCompressed(received_ratio) + _get_compression_ratios(_stream_stream_client, {}, {}, {}, + _GenericHandler(None), {}, {}, server_kwargs, + _GenericHandler(disable_first_compression), + _REQUEST) def _get_compression_str(name, value): diff --git a/src/python/grpcio_tests/tests_aio/channelz/channelz_servicer_test.py b/src/python/grpcio_tests/tests_aio/channelz/channelz_servicer_test.py index e400dc87dc4..8c62826a641 100644 --- a/src/python/grpcio_tests/tests_aio/channelz/channelz_servicer_test.py +++ b/src/python/grpcio_tests/tests_aio/channelz/channelz_servicer_test.py @@ -375,21 +375,28 @@ class ChannelzServicerTest(AioTestBase): # Subchannel exists self.assertGreater(len(gc_resp.channel.subchannel_ref), 0) - gsc_resp = await self._channelz_stub.GetSubchannel( - channelz_pb2.GetSubchannelRequest( - subchannel_id=gc_resp.channel.subchannel_ref[0].subchannel_id)) + while True: + gsc_resp = await self._channelz_stub.GetSubchannel( + channelz_pb2.GetSubchannelRequest( + subchannel_id=gc_resp.channel.subchannel_ref[0]. + subchannel_id)) + if gsc_resp.subchannel.data.calls_started == gsc_resp.subchannel.data.calls_succeeded + gsc_resp.subchannel.data.calls_failed: + break self.assertEqual(gsc_resp.subchannel.data.calls_started, 1) - self.assertEqual(gsc_resp.subchannel.data.calls_succeeded, 1) self.assertEqual(gsc_resp.subchannel.data.calls_failed, 0) + self.assertEqual(gsc_resp.subchannel.data.calls_succeeded, 1) # Socket exists self.assertEqual(len(gsc_resp.subchannel.socket_ref), 1) - gs_resp = await self._channelz_stub.GetSocket( - channelz_pb2.GetSocketRequest( - socket_id=gsc_resp.subchannel.socket_ref[0].socket_id)) + while True: + gs_resp = await self._channelz_stub.GetSocket( + channelz_pb2.GetSocketRequest( + socket_id=gsc_resp.subchannel.socket_ref[0].socket_id)) + if gs_resp.socket.data.streams_started == gs_resp.socket.data.streams_succeeded + gs_resp.socket.data.streams_failed: + break self.assertEqual(gs_resp.socket.data.streams_started, 1) - self.assertEqual(gs_resp.socket.data.streams_succeeded, 1) self.assertEqual(gs_resp.socket.data.streams_failed, 0) + self.assertEqual(gs_resp.socket.data.streams_succeeded, 1) self.assertEqual(gs_resp.socket.data.messages_sent, test_constants.STREAM_LENGTH) self.assertEqual(gs_resp.socket.data.messages_received, diff --git a/test/core/bad_client/tests/duplicate_header.cc b/test/core/bad_client/tests/duplicate_header.cc index 28c5f310e4a..09c5cfc62d6 100644 --- a/test/core/bad_client/tests/duplicate_header.cc +++ b/test/core/bad_client/tests/duplicate_header.cc @@ -90,7 +90,7 @@ static void verifier(grpc_server* server, grpc_completion_queue* cq, nullptr); GPR_ASSERT(GRPC_CALL_OK == error); - CQ_EXPECT_COMPLETION(cqv, tag(102), 1); + CQ_EXPECT_COMPLETION_ANY_STATUS(cqv, tag(102)); cq_verify(cqv); memset(ops, 0, sizeof(ops)); diff --git a/test/core/end2end/fixtures/h2_full_no_retry.cc b/test/core/end2end/fixtures/h2_full_no_retry.cc new file mode 100644 index 00000000000..d69f502f273 --- /dev/null +++ b/test/core/end2end/fixtures/h2_full_no_retry.cc @@ -0,0 +1,118 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include +#include +#include +#include + +#include "src/core/ext/filters/client_channel/client_channel.h" +#include "src/core/ext/filters/http/server/http_server_filter.h" +#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" +#include "src/core/lib/channel/connected_channel.h" +#include "src/core/lib/gprpp/host_port.h" +#include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/server.h" +#include "test/core/end2end/end2end_tests.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" + +struct fullstack_fixture_data { + std::string localaddr; +}; + +static grpc_end2end_test_fixture chttp2_create_fixture_fullstack( + const grpc_channel_args* /*client_args*/, + const grpc_channel_args* /*server_args*/) { + grpc_end2end_test_fixture f; + int port = grpc_pick_unused_port_or_die(); + fullstack_fixture_data* ffd = new fullstack_fixture_data(); + memset(&f, 0, sizeof(f)); + + ffd->localaddr = grpc_core::JoinHostPort("localhost", port); + + f.fixture_data = ffd; + f.cq = grpc_completion_queue_create_for_next(nullptr); + + return f; +} + +void chttp2_init_client_fullstack(grpc_end2end_test_fixture* f, + const grpc_channel_args* client_args) { + fullstack_fixture_data* ffd = + static_cast(f->fixture_data); + grpc_channel_credentials* creds = grpc_insecure_credentials_create(); + grpc_arg no_retry = grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_ENABLE_RETRIES), 0); + client_args = grpc_channel_args_copy_and_add(client_args, &no_retry, 1); + f->client = grpc_channel_create(ffd->localaddr.c_str(), creds, client_args); + grpc_channel_args_destroy(client_args); + grpc_channel_credentials_release(creds); + GPR_ASSERT(f->client); +} + +void chttp2_init_server_fullstack(grpc_end2end_test_fixture* f, + const grpc_channel_args* server_args) { + fullstack_fixture_data* ffd = + static_cast(f->fixture_data); + if (f->server) { + grpc_server_destroy(f->server); + } + f->server = grpc_server_create(server_args, nullptr); + grpc_server_register_completion_queue(f->server, f->cq, nullptr); + grpc_server_credentials* server_creds = + grpc_insecure_server_credentials_create(); + GPR_ASSERT(grpc_server_add_http2_port(f->server, ffd->localaddr.c_str(), + server_creds)); + grpc_server_credentials_release(server_creds); + grpc_server_start(f->server); +} + +void chttp2_tear_down_fullstack(grpc_end2end_test_fixture* f) { + fullstack_fixture_data* ffd = + static_cast(f->fixture_data); + delete ffd; +} + +/* All test configurations */ +static grpc_end2end_test_config configs[] = { + {"chttp2/fullstack", + FEATURE_MASK_SUPPORTS_DELAYED_CONNECTION | + FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | + FEATURE_MASK_SUPPORTS_AUTHORITY_HEADER, + nullptr, chttp2_create_fixture_fullstack, chttp2_init_client_fullstack, + chttp2_init_server_fullstack, chttp2_tear_down_fullstack}, +}; + +int main(int argc, char** argv) { + size_t i; + + grpc::testing::TestEnvironment env(&argc, argv); + grpc_end2end_tests_pre_init(); + grpc_init(); + + for (i = 0; i < sizeof(configs) / sizeof(*configs); i++) { + grpc_end2end_tests(argc, argv, configs[i]); + } + + grpc_shutdown(); + + return 0; +} diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index e585de0aff7..a50d22cce88 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -36,8 +36,11 @@ def _fixture_options( supports_write_buffering = True, client_channel = True, supports_msvc = True, + supports_retry = None, flaky_tests = [], tags = []): + if supports_retry == None: + supports_retry = client_channel return struct( fullstack = fullstack, includes_proxy = includes_proxy, @@ -54,6 +57,7 @@ def _fixture_options( supports_msvc = supports_msvc, _platforms = _platforms, flaky_tests = flaky_tests, + supports_retry = supports_retry, tags = tags, ) @@ -73,6 +77,7 @@ END2END_FIXTURES = { tags = ["no_test_ios"], ), "h2_full": _fixture_options(), + "h2_full_no_retry": _fixture_options(supports_retry = False), "h2_full+pipe": _fixture_options(_platforms = ["linux"]), "h2_full+trace": _fixture_options(tracing = True), "h2_http_proxy": _fixture_options(supports_proxy_auth = True), @@ -155,6 +160,7 @@ def _test_options( needs_proxy_auth = False, needs_write_buffering = False, needs_client_channel = False, + needs_retry = False, short_name = None, exclude_pollers = []): return struct( @@ -170,6 +176,7 @@ def _test_options( needs_proxy_auth = needs_proxy_auth, needs_write_buffering = needs_write_buffering, needs_client_channel = needs_client_channel, + needs_retry = needs_retry, short_name = short_name, exclude_pollers = exclude_pollers, ) @@ -250,81 +257,89 @@ END2END_TESTS = { "registered_call": _test_options(), "request_with_flags": _test_options(proxyable = False), "request_with_payload": _test_options(), - "retry": _test_options(needs_client_channel = True), - "retry_cancellation": _test_options(needs_client_channel = True), - "retry_cancel_during_delay": _test_options(needs_client_channel = True), + "retry": _test_options(needs_client_channel = True, needs_retry = True), + "retry_cancellation": _test_options(needs_client_channel = True, needs_retry = True), + "retry_cancel_during_delay": _test_options(needs_client_channel = True, needs_retry = True), "retry_cancel_with_multiple_send_batches": _test_options( # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_cancel3", needs_client_channel = True, + needs_retry = True, ), "retry_cancel_after_first_attempt_starts": _test_options( # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_cancel4", needs_client_channel = True, + needs_retry = True, ), - "retry_disabled": _test_options(needs_client_channel = True), - "retry_exceeds_buffer_size_in_delay": _test_options(needs_client_channel = True), + "retry_disabled": _test_options(needs_client_channel = True, needs_retry = True), + "retry_exceeds_buffer_size_in_delay": _test_options(needs_client_channel = True, needs_retry = True), "retry_exceeds_buffer_size_in_initial_batch": _test_options( needs_client_channel = True, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_exceeds_buffer_size_in_init", + needs_retry = True, ), "retry_exceeds_buffer_size_in_subsequent_batch": _test_options( needs_client_channel = True, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_exceeds_buffer_size_in_subseq", + needs_retry = True, ), - "retry_lb_drop": _test_options(needs_client_channel = True), - "retry_lb_fail": _test_options(needs_client_channel = True), - "retry_non_retriable_status": _test_options(needs_client_channel = True), + "retry_lb_drop": _test_options(needs_client_channel = True, needs_retry = True), + "retry_lb_fail": _test_options(needs_client_channel = True, needs_retry = True), + "retry_non_retriable_status": _test_options(needs_client_channel = True, needs_retry = True), "retry_non_retriable_status_before_recv_trailing_metadata_started": _test_options( needs_client_channel = True, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_non_retriable_status2", + needs_retry = True, ), - "retry_per_attempt_recv_timeout": _test_options(needs_client_channel = True), + "retry_per_attempt_recv_timeout": _test_options(needs_client_channel = True, needs_retry = True), "retry_per_attempt_recv_timeout_on_last_attempt": _test_options( needs_client_channel = True, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_per_attempt_recv_timeout2", + needs_retry = True, ), - "retry_recv_initial_metadata": _test_options(needs_client_channel = True), - "retry_recv_message": _test_options(needs_client_channel = True), - "retry_recv_message_replay": _test_options(needs_client_channel = True), - "retry_recv_trailing_metadata_error": _test_options(needs_client_channel = True), - "retry_send_initial_metadata_refs": _test_options(needs_client_channel = True), - "retry_send_op_fails": _test_options(needs_client_channel = True), - "retry_send_recv_batch": _test_options(needs_client_channel = True), - "retry_server_pushback_delay": _test_options(needs_client_channel = True), - "retry_server_pushback_disabled": _test_options(needs_client_channel = True), - "retry_streaming": _test_options(needs_client_channel = True), - "retry_streaming_after_commit": _test_options(needs_client_channel = True), + "retry_recv_initial_metadata": _test_options(needs_client_channel = True, needs_retry = True), + "retry_recv_message": _test_options(needs_client_channel = True, needs_retry = True), + "retry_recv_message_replay": _test_options(needs_client_channel = True, needs_retry = True), + "retry_recv_trailing_metadata_error": _test_options(needs_client_channel = True, needs_retry = True), + "retry_send_initial_metadata_refs": _test_options(needs_client_channel = True, needs_retry = True), + "retry_send_op_fails": _test_options(needs_client_channel = True, needs_retry = True), + "retry_send_recv_batch": _test_options(needs_client_channel = True, needs_retry = True), + "retry_server_pushback_delay": _test_options(needs_client_channel = True, needs_retry = True), + "retry_server_pushback_disabled": _test_options(needs_client_channel = True, needs_retry = True), + "retry_streaming": _test_options(needs_client_channel = True, needs_retry = True), + "retry_streaming_after_commit": _test_options(needs_client_channel = True, needs_retry = True), "retry_streaming_succeeds_before_replay_finished": _test_options( needs_client_channel = True, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_streaming2", + needs_retry = True, ), - "retry_throttled": _test_options(needs_client_channel = True), - "retry_too_many_attempts": _test_options(needs_client_channel = True), - "retry_transparent_goaway": _test_options(needs_client_channel = True), - "retry_transparent_not_sent_on_wire": _test_options(needs_client_channel = True), + "retry_throttled": _test_options(needs_client_channel = True, needs_retry = True), + "retry_too_many_attempts": _test_options(needs_client_channel = True, needs_retry = True), + "retry_transparent_goaway": _test_options(needs_client_channel = True, needs_retry = True), + "retry_transparent_not_sent_on_wire": _test_options(needs_client_channel = True, needs_retry = True), "retry_transparent_max_concurrent_streams": _test_options( needs_client_channel = True, proxyable = False, # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # See b/151617965 short_name = "retry_transparent_mcs", + needs_retry = True, ), - "retry_unref_before_finish": _test_options(needs_client_channel = True), - "retry_unref_before_recv": _test_options(needs_client_channel = True), + "retry_unref_before_finish": _test_options(needs_client_channel = True, needs_retry = True), + "retry_unref_before_recv": _test_options(needs_client_channel = True, needs_retry = True), "server_finishes_request": _test_options(), "server_streaming": _test_options(needs_http2 = True), "shutdown_finishes_calls": _test_options(), @@ -375,6 +390,9 @@ def _compatible(fopt, topt): if topt.needs_client_channel: if not fopt.client_channel: return False + if topt.needs_retry: + if not fopt.supports_retry: + return False return True def _platform_support_tags(fopt): diff --git a/test/core/end2end/tests/request_with_flags.cc b/test/core/end2end/tests/request_with_flags.cc index 59e137aeca3..b277036dffb 100644 --- a/test/core/end2end/tests/request_with_flags.cc +++ b/test/core/end2end/tests/request_with_flags.cc @@ -25,7 +25,7 @@ #include #include "src/core/lib/gpr/useful.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/transport/transport.h" #include "test/core/end2end/cq_verifier.h" #include "test/core/end2end/end2end_tests.h" diff --git a/test/core/event_engine/test_suite/event_engine_test_utils.cc b/test/core/event_engine/test_suite/event_engine_test_utils.cc index 68b4018fd72..b7ccbce48e6 100644 --- a/test/core/event_engine/test_suite/event_engine_test_utils.cc +++ b/test/core/event_engine/test_suite/event_engine_test_utils.cc @@ -67,7 +67,7 @@ std::string ExtractSliceBufferIntoString(SliceBuffer* buf) { } std::string tmp(buf->Length(), '\0'); char* bytes = const_cast(tmp.c_str()); - grpc_slice_buffer_move_first_into_buffer(buf->RawSliceBuffer(), buf->Length(), + grpc_slice_buffer_move_first_into_buffer(buf->c_slice_buffer(), buf->Length(), bytes); return tmp; } diff --git a/test/core/gprpp/BUILD b/test/core/gprpp/BUILD index b61ac9f6e98..b5f48b9887d 100644 --- a/test/core/gprpp/BUILD +++ b/test/core/gprpp/BUILD @@ -84,18 +84,6 @@ grpc_cc_test( ], ) -grpc_cc_test( - name = "manual_constructor_test", - srcs = ["manual_constructor_test.cc"], - language = "C++", - uses_event_engine = False, - uses_polling = False, - deps = [ - "//:gpr", - "//test/core/util:grpc_test_util", - ], -) - grpc_cc_test( name = "bitset_test", srcs = ["bitset_test.cc"], diff --git a/test/core/gprpp/manual_constructor_test.cc b/test/core/gprpp/manual_constructor_test.cc deleted file mode 100644 index b5e5873bf3f..00000000000 --- a/test/core/gprpp/manual_constructor_test.cc +++ /dev/null @@ -1,100 +0,0 @@ -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -/* Test of gpr synchronization support. */ - -#include "src/core/lib/gprpp/manual_constructor.h" - -#include -#include - -#include - -#include -#include -#include - -#include "test/core/util/test_config.h" - -class A { - public: - A() {} - virtual ~A() {} - virtual const char* foo() { return "A_foo"; } - virtual const char* bar() { return "A_bar"; } -}; - -class B : public A { - public: - B() {} - ~B() override {} - const char* foo() override { return "B_foo"; } - char get_junk() { return junk[0]; } - - private: - char junk[1000]; -}; - -class C : public B { - public: - C() {} - ~C() override {} - const char* bar() override { return "C_bar"; } - char get_more_junk() { return more_junk[0]; } - - private: - char more_junk[1000]; -}; - -class D : public A { - public: - const char* bar() override { return "D_bar"; } -}; - -static void basic_test() { - grpc_core::PolymorphicManualConstructor poly; - poly.Init(); - GPR_ASSERT(!strcmp(poly->foo(), "B_foo")); - GPR_ASSERT(!strcmp(poly->bar(), "A_bar")); -} - -static void complex_test() { - grpc_core::PolymorphicManualConstructor polyB; - polyB.Init(); - GPR_ASSERT(!strcmp(polyB->foo(), "B_foo")); - GPR_ASSERT(!strcmp(polyB->bar(), "A_bar")); - - grpc_core::PolymorphicManualConstructor polyC; - polyC.Init(); - GPR_ASSERT(!strcmp(polyC->foo(), "B_foo")); - GPR_ASSERT(!strcmp(polyC->bar(), "C_bar")); - - grpc_core::PolymorphicManualConstructor polyD; - polyD.Init(); - GPR_ASSERT(!strcmp(polyD->foo(), "A_foo")); - GPR_ASSERT(!strcmp(polyD->bar(), "D_bar")); -} - -/* ------------------------------------------------- */ - -int main(int argc, char* argv[]) { - grpc::testing::TestEnvironment env(&argc, argv); - basic_test(); - complex_test(); - return 0; -} diff --git a/test/core/transport/BUILD b/test/core/transport/BUILD index 4909a1ec4e8..94264237979 100644 --- a/test/core/transport/BUILD +++ b/test/core/transport/BUILD @@ -35,22 +35,6 @@ grpc_cc_test( ], ) -grpc_cc_test( - name = "byte_stream_test", - srcs = ["byte_stream_test.cc"], - external_deps = [ - "gtest", - ], - language = "C++", - uses_event_engine = False, - uses_polling = False, - deps = [ - "//:gpr", - "//:grpc", - "//test/core/util:grpc_test_util", - ], -) - grpc_cc_test( name = "connectivity_state_test", srcs = ["connectivity_state_test.cc"], diff --git a/test/core/transport/binder/binder_transport_test.cc b/test/core/transport/binder/binder_transport_test.cc index 53cc3424be5..c73ef8b7b07 100644 --- a/test/core/transport/binder/binder_transport_test.cc +++ b/test/core/transport/binder/binder_transport_test.cc @@ -242,19 +242,13 @@ struct MakeSendInitialMetadata { struct MakeSendMessage { MakeSendMessage(const std::string& message, grpc_transport_stream_op_batch* op) { - grpc_slice_buffer send_buffer; - grpc_slice_buffer_init(&send_buffer); - grpc_slice send_slice = grpc_slice_from_cpp_string(message); - grpc_slice_buffer_add(&send_buffer, send_slice); - - send_stream.Init(&send_buffer, 0); - grpc_slice_buffer_destroy(&send_buffer); + send_stream.Append(grpc_core::Slice::FromCopiedString(message)); op->send_message = true; - op->payload->send_message.send_message.reset(send_stream.get()); + op->payload->send_message.send_message = &send_stream; } - grpc_core::ManualConstructor send_stream; + grpc_core::SliceBuffer send_stream; }; struct MakeSendTrailingMetadata { @@ -313,7 +307,7 @@ struct MakeRecvMessage { MockGrpcClosure ready; absl::Notification notification; - grpc_core::OrphanablePtr grpc_message; + absl::optional grpc_message; }; struct MakeRecvTrailingMetadata { @@ -565,13 +559,7 @@ TEST_F(BinderTransportTest, PerformRecvMessage) { grpc_core::ExecCtx::Get()->Flush(); recv_message.notification.WaitForNotification(); - EXPECT_TRUE(recv_message.grpc_message->Next(SIZE_MAX, nullptr)); - grpc_slice slice; - EXPECT_EQ(recv_message.grpc_message->Pull(&slice), GRPC_ERROR_NONE); - EXPECT_EQ(kMessage, - std::string(reinterpret_cast(GRPC_SLICE_START_PTR(slice)), - GRPC_SLICE_LENGTH(slice))); - grpc_slice_unref_internal(slice); + EXPECT_EQ(kMessage, recv_message.grpc_message->JoinIntoString()); } TEST_F(BinderTransportTest, PerformRecvTrailingMetadata) { @@ -629,13 +617,7 @@ TEST_F(BinderTransportTest, PerformRecvAll) { trailing_metadata.emplace_back("grpc-status", std::to_string(kStatus)); VerifyMetadataEqual(trailing_metadata, recv_trailing_metadata.grpc_trailing_metadata); - EXPECT_TRUE(recv_message.grpc_message->Next(SIZE_MAX, nullptr)); - grpc_slice slice; - EXPECT_EQ(recv_message.grpc_message->Pull(&slice), GRPC_ERROR_NONE); - EXPECT_EQ(kMessage, - std::string(reinterpret_cast(GRPC_SLICE_START_PTR(slice)), - GRPC_SLICE_LENGTH(slice))); - grpc_slice_unref_internal(slice); + EXPECT_EQ(kMessage, recv_message.grpc_message->JoinIntoString()); } TEST_F(BinderTransportTest, PerformAllOps) { @@ -706,13 +688,7 @@ TEST_F(BinderTransportTest, PerformAllOps) { VerifyMetadataEqual(AppendStatus(kRecvTrailingMetadata, kStatus), recv_trailing_metadata.grpc_trailing_metadata); - EXPECT_TRUE(recv_message.grpc_message->Next(SIZE_MAX, nullptr)); - grpc_slice slice; - EXPECT_EQ(recv_message.grpc_message->Pull(&slice), GRPC_ERROR_NONE); - EXPECT_EQ(kRecvMessage, - std::string(reinterpret_cast(GRPC_SLICE_START_PTR(slice)), - GRPC_SLICE_LENGTH(slice))); - grpc_slice_unref_internal(slice); + EXPECT_EQ(kRecvMessage, recv_message.grpc_message->JoinIntoString()); } TEST_F(BinderTransportTest, WireWriterRpcCallErrorPropagates) { diff --git a/test/core/transport/byte_stream_test.cc b/test/core/transport/byte_stream_test.cc deleted file mode 100644 index 3b3ff743484..00000000000 --- a/test/core/transport/byte_stream_test.cc +++ /dev/null @@ -1,254 +0,0 @@ -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include "src/core/lib/transport/byte_stream.h" - -#include - -#include -#include -#include - -#include "src/core/lib/gpr/useful.h" -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/slice/slice_internal.h" -#include "test/core/util/test_config.h" - -namespace grpc_core { -namespace { - -// -// SliceBufferByteStream tests -// - -void NotCalledClosure(void* /*arg*/, grpc_error_handle /*error*/) { - GPR_ASSERT(false); -} - -TEST(SliceBufferByteStream, Basic) { - ExecCtx exec_ctx; - // Create and populate slice buffer. - grpc_slice_buffer buffer; - grpc_slice_buffer_init(&buffer); - grpc_slice input[] = { - grpc_slice_from_static_string("foo"), - grpc_slice_from_static_string("bar"), - }; - for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { - grpc_slice_buffer_add(&buffer, input[i]); - } - // Create byte stream. - SliceBufferByteStream stream(&buffer, 0); - grpc_slice_buffer_destroy_internal(&buffer); - EXPECT_EQ(6U, stream.length()); - grpc_closure closure; - GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr, - grpc_schedule_on_exec_ctx); - // Read each slice. Note that Next() always returns synchronously. - for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { - ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); - grpc_slice output; - grpc_error_handle error = stream.Pull(&output); - EXPECT_TRUE(error == GRPC_ERROR_NONE); - EXPECT_TRUE(grpc_slice_eq(input[i], output)); - grpc_slice_unref_internal(output); - } - // Clean up. - stream.Orphan(); -} - -TEST(SliceBufferByteStream, Shutdown) { - ExecCtx exec_ctx; - // Create and populate slice buffer. - grpc_slice_buffer buffer; - grpc_slice_buffer_init(&buffer); - grpc_slice input[] = { - grpc_slice_from_static_string("foo"), - grpc_slice_from_static_string("bar"), - }; - for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { - grpc_slice_buffer_add(&buffer, input[i]); - } - // Create byte stream. - SliceBufferByteStream stream(&buffer, 0); - grpc_slice_buffer_destroy_internal(&buffer); - EXPECT_EQ(6U, stream.length()); - grpc_closure closure; - GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr, - grpc_schedule_on_exec_ctx); - // Read the first slice. - ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); - grpc_slice output; - grpc_error_handle error = stream.Pull(&output); - EXPECT_TRUE(error == GRPC_ERROR_NONE); - EXPECT_TRUE(grpc_slice_eq(input[0], output)); - grpc_slice_unref_internal(output); - // Now shutdown. - grpc_error_handle shutdown_error = - GRPC_ERROR_CREATE_FROM_STATIC_STRING("shutdown error"); - stream.Shutdown(GRPC_ERROR_REF(shutdown_error)); - // After shutdown, the next pull() should return the error. - ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); - error = stream.Pull(&output); - EXPECT_TRUE(error == shutdown_error); - GRPC_ERROR_UNREF(error); - GRPC_ERROR_UNREF(shutdown_error); - // Clean up. - stream.Orphan(); -} - -// -// CachingByteStream tests -// - -TEST(CachingByteStream, Basic) { - ExecCtx exec_ctx; - // Create and populate slice buffer byte stream. - grpc_slice_buffer buffer; - grpc_slice_buffer_init(&buffer); - grpc_slice input[] = { - grpc_slice_from_static_string("foo"), - grpc_slice_from_static_string("bar"), - }; - for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { - grpc_slice_buffer_add(&buffer, input[i]); - } - SliceBufferByteStream underlying_stream(&buffer, 0); - grpc_slice_buffer_destroy_internal(&buffer); - // Create cache and caching stream. - ByteStreamCache cache((OrphanablePtr(&underlying_stream))); - ByteStreamCache::CachingByteStream stream(&cache); - grpc_closure closure; - GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr, - grpc_schedule_on_exec_ctx); - // Read each slice. Note that next() always returns synchronously, - // because the underlying byte stream always does. - for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { - ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); - grpc_slice output; - grpc_error_handle error = stream.Pull(&output); - EXPECT_TRUE(error == GRPC_ERROR_NONE); - EXPECT_TRUE(grpc_slice_eq(input[i], output)); - grpc_slice_unref_internal(output); - } - // Clean up. - stream.Orphan(); - cache.Destroy(); -} - -TEST(CachingByteStream, Reset) { - ExecCtx exec_ctx; - // Create and populate slice buffer byte stream. - grpc_slice_buffer buffer; - grpc_slice_buffer_init(&buffer); - grpc_slice input[] = { - grpc_slice_from_static_string("foo"), - grpc_slice_from_static_string("bar"), - }; - for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { - grpc_slice_buffer_add(&buffer, input[i]); - } - SliceBufferByteStream underlying_stream(&buffer, 0); - grpc_slice_buffer_destroy_internal(&buffer); - // Create cache and caching stream. - ByteStreamCache cache((OrphanablePtr(&underlying_stream))); - ByteStreamCache::CachingByteStream stream(&cache); - grpc_closure closure; - GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr, - grpc_schedule_on_exec_ctx); - // Read one slice. - ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); - grpc_slice output; - grpc_error_handle error = stream.Pull(&output); - EXPECT_TRUE(error == GRPC_ERROR_NONE); - EXPECT_TRUE(grpc_slice_eq(input[0], output)); - grpc_slice_unref_internal(output); - // Reset the caching stream. The reads should start over from the - // first slice. - stream.Reset(); - for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { - ASSERT_TRUE(stream.Next(~(size_t)0, &closure)); - error = stream.Pull(&output); - EXPECT_TRUE(error == GRPC_ERROR_NONE); - EXPECT_TRUE(grpc_slice_eq(input[i], output)); - grpc_slice_unref_internal(output); - } - // Clean up. - stream.Orphan(); - cache.Destroy(); -} - -TEST(CachingByteStream, SharedCache) { - ExecCtx exec_ctx; - // Create and populate slice buffer byte stream. - grpc_slice_buffer buffer; - grpc_slice_buffer_init(&buffer); - grpc_slice input[] = { - grpc_slice_from_static_string("foo"), - grpc_slice_from_static_string("bar"), - }; - for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { - grpc_slice_buffer_add(&buffer, input[i]); - } - SliceBufferByteStream underlying_stream(&buffer, 0); - grpc_slice_buffer_destroy_internal(&buffer); - // Create cache and two caching streams. - ByteStreamCache cache((OrphanablePtr(&underlying_stream))); - ByteStreamCache::CachingByteStream stream1(&cache); - ByteStreamCache::CachingByteStream stream2(&cache); - grpc_closure closure; - GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr, - grpc_schedule_on_exec_ctx); - // Read one slice from stream1. - EXPECT_TRUE(stream1.Next(~(size_t)0, &closure)); - grpc_slice output; - grpc_error_handle error = stream1.Pull(&output); - EXPECT_TRUE(error == GRPC_ERROR_NONE); - EXPECT_TRUE(grpc_slice_eq(input[0], output)); - grpc_slice_unref_internal(output); - // Read all slices from stream2. - for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) { - EXPECT_TRUE(stream2.Next(~(size_t)0, &closure)); - error = stream2.Pull(&output); - EXPECT_TRUE(error == GRPC_ERROR_NONE); - EXPECT_TRUE(grpc_slice_eq(input[i], output)); - grpc_slice_unref_internal(output); - } - // Now read the second slice from stream1. - EXPECT_TRUE(stream1.Next(~(size_t)0, &closure)); - error = stream1.Pull(&output); - EXPECT_TRUE(error == GRPC_ERROR_NONE); - EXPECT_TRUE(grpc_slice_eq(input[1], output)); - grpc_slice_unref_internal(output); - // Clean up. - stream1.Orphan(); - stream2.Orphan(); - cache.Destroy(); -} - -} // namespace -} // namespace grpc_core - -int main(int argc, char** argv) { - grpc::testing::TestEnvironment env(&argc, argv); - ::testing::InitGoogleTest(&argc, argv); - grpc_init(); - int retval = RUN_ALL_TESTS(); - grpc_shutdown(); - return retval; -} diff --git a/test/core/transport/chttp2/BUILD b/test/core/transport/chttp2/BUILD index 7721baa8233..215fd619314 100644 --- a/test/core/transport/chttp2/BUILD +++ b/test/core/transport/chttp2/BUILD @@ -32,6 +32,22 @@ grpc_proto_fuzzer( ], ) +grpc_proto_fuzzer( + name = "flow_control_fuzzer", + srcs = ["flow_control_fuzzer.cc"], + corpus = "flow_control_fuzzer_corpus", + language = "C++", + proto = "flow_control_fuzzer.proto", + tags = ["no_windows"], + uses_event_engine = False, + uses_polling = False, + deps = [ + "//:chttp2_flow_control", + "//:gpr", + "//test/core/util:grpc_suppressions", + ], +) + grpc_cc_test( name = "alpn_test", srcs = ["alpn_test.cc"], @@ -87,17 +103,17 @@ grpc_cc_test( grpc_cc_test( name = "flow_control_test", - size = "large", srcs = ["flow_control_test.cc"], external_deps = [ "gtest", ], language = "C++", + uses_event_engine = False, + uses_polling = False, deps = [ - "//:gpr", - "//:grpc", - "//test/core/end2end:cq_verifier", - "//test/core/util:grpc_test_util", + "//:chttp2_flow_control", + "//:resource_quota", + "//test/core/util:grpc_suppressions", ], ) diff --git a/test/core/transport/chttp2/flow_control_fuzzer.cc b/test/core/transport/chttp2/flow_control_fuzzer.cc new file mode 100644 index 00000000000..43f92fc7e45 --- /dev/null +++ b/test/core/transport/chttp2/flow_control_fuzzer.cc @@ -0,0 +1,408 @@ +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include + +#include "src/core/ext/transport/chttp2/transport/flow_control.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/libfuzzer/libfuzzer_macro.h" +#include "test/core/transport/chttp2/flow_control_fuzzer.pb.h" + +bool squelch = true; + +extern gpr_timespec (*gpr_now_impl)(gpr_clock_type clock_type); + +namespace grpc_core { +namespace chttp2 { +namespace { + +constexpr uint64_t kMaxAdvanceTimeMillis = 24ull * 365 * 3600 * 1000; + +gpr_timespec g_now; +gpr_timespec now_impl(gpr_clock_type clock_type) { + GPR_ASSERT(clock_type != GPR_TIMESPAN); + gpr_timespec ts = g_now; + ts.clock_type = clock_type; + return ts; +} + +void InitGlobals() { + g_now = {1, 0, GPR_CLOCK_MONOTONIC}; + TestOnlySetProcessEpoch(g_now); + gpr_now_impl = now_impl; +} + +class FlowControlFuzzer { + public: + explicit FlowControlFuzzer(bool enable_bdp) { + ExecCtx exec_ctx; + tfc_ = absl::make_unique("fuzzer", enable_bdp, + &memory_owner_); + } + + ~FlowControlFuzzer() { + ExecCtx exec_ctx; + streams_.clear(); + tfc_.reset(); + memory_owner_.Release(allocated_memory_); + } + + void Perform(const flow_control_fuzzer::Action& action); + void AssertNoneStuck() const; + void AssertAnnouncedOverInitialWindowSizeCorrect() const; + + private: + struct StreamPayload { + uint32_t id; + uint64_t size; + }; + + struct SendToRemote { + bool bdp_ping = false; + absl::optional initial_window_size; + uint32_t transport_window_update; + std::vector stream_window_updates; + }; + + struct SendFromRemote { + bool bdp_pong = false; + absl::optional ack_initial_window_size; + std::vector stream_writes; + }; + + struct Stream { + explicit Stream(uint32_t id, TransportFlowControl* tfc) : id(id), fc(tfc) {} + uint32_t id; + StreamFlowControl fc; + int64_t queued_writes = 0; + int64_t window_delta = 0; + }; + + void PerformAction(FlowControlAction action, Stream* stream); + Stream* GetStream(uint32_t id) { + auto it = streams_.find(id); + if (it == streams_.end()) { + it = streams_.emplace(id, Stream{id, tfc_.get()}).first; + } + return &it->second; + } + + MemoryQuotaRefPtr memory_quota_ = MakeMemoryQuota("fuzzer"); + MemoryOwner memory_owner_ = memory_quota_->CreateMemoryOwner("owner"); + std::unique_ptr tfc_; + absl::optional queued_initial_window_size_; + absl::optional queued_send_max_frame_size_; + bool scheduled_write_ = false; + bool sending_initial_window_size_ = false; + std::deque send_to_remote_; + std::deque send_from_remote_; + uint32_t remote_initial_window_size_ = kDefaultWindow; + int64_t remote_transport_window_size_ = kDefaultWindow; + std::map streams_; + std::queue streams_to_update_; + uint64_t allocated_memory_ = 0; + Timestamp next_bdp_ping_ = Timestamp::ProcessEpoch(); +}; + +void FlowControlFuzzer::Perform(const flow_control_fuzzer::Action& action) { + ExecCtx exec_ctx; + bool sending_payload = false; + switch (action.action_case()) { + case flow_control_fuzzer::Action::ACTION_NOT_SET: + break; + case flow_control_fuzzer::Action::kSetMemoryQuota: { + memory_quota_->SetSize( + Clamp(action.set_memory_quota(), uint64_t(1), + uint64_t(std::numeric_limits::max()))); + } break; + case flow_control_fuzzer::Action::kStepTimeMs: { + g_now = gpr_time_add( + g_now, gpr_time_from_millis(Clamp(action.step_time_ms(), uint64_t(1), + kMaxAdvanceTimeMillis), + GPR_TIMESPAN)); + exec_ctx.InvalidateNow(); + if (exec_ctx.Now() >= next_bdp_ping_) { + scheduled_write_ = true; + } + } break; + case flow_control_fuzzer::Action::kPeriodicUpdate: { + PerformAction(tfc_->PeriodicUpdate(), nullptr); + } break; + case flow_control_fuzzer::Action::kPerformSendToRemote: { + scheduled_write_ = true; + } break; + case flow_control_fuzzer::Action::kPerformSendToRemoteWithPayload: { + scheduled_write_ = true; + sending_payload = true; + } break; + case flow_control_fuzzer::Action::kReadSendToRemote: { + if (send_to_remote_.empty()) break; + auto sent_to_remote = send_to_remote_.front(); + if (sent_to_remote.initial_window_size.has_value()) { + if (!squelch) { + fprintf(stderr, "Setting initial window size to %d\n", + sent_to_remote.initial_window_size.value()); + } + SendFromRemote send_from_remote; + send_from_remote.ack_initial_window_size = + sent_to_remote.initial_window_size; + for (const auto& id_stream : streams_) { + GPR_ASSERT(id_stream.second.window_delta + + *sent_to_remote.initial_window_size <= + (1u << 31) - 1); + } + remote_initial_window_size_ = *sent_to_remote.initial_window_size; + send_from_remote_.push_back(send_from_remote); + } + if (sent_to_remote.bdp_ping) { + SendFromRemote send_from_remote; + send_from_remote.bdp_pong = true; + send_from_remote_.push_back(send_from_remote); + } + for (auto stream_update : sent_to_remote.stream_window_updates) { + Stream* s = GetStream(stream_update.id); + if (!squelch) { + fprintf(stderr, + "[%" PRIu32 "]: increase window delta by %" PRIu64 + " from %" PRId64 "\n", + stream_update.id, stream_update.size, s->window_delta); + } + s->window_delta += stream_update.size; + GPR_ASSERT(s->window_delta <= chttp2::kMaxWindowDelta); + } + remote_transport_window_size_ += sent_to_remote.transport_window_update; + send_to_remote_.pop_front(); + } break; + case flow_control_fuzzer::Action::kReadSendFromRemote: { + if (send_from_remote_.empty()) break; + auto sent_from_remote = send_from_remote_.front(); + if (sent_from_remote.ack_initial_window_size.has_value()) { + if (!squelch) { + fprintf(stderr, "Received ACK for initial window size %d\n", + *sent_from_remote.ack_initial_window_size); + } + tfc_->SetAckedInitialWindow(*sent_from_remote.ack_initial_window_size); + sending_initial_window_size_ = false; + } + if (sent_from_remote.bdp_pong) { + next_bdp_ping_ = tfc_->bdp_estimator()->CompletePing(); + } + for (const auto& stream_write : sent_from_remote.stream_writes) { + Stream* stream = GetStream(stream_write.id); + if (!squelch) { + fprintf(stderr, "[%" PRIu32 "]: recv write of %" PRIu64 "\n", + stream_write.id, stream_write.size); + } + if (auto* bdp = tfc_->bdp_estimator()) { + bdp->AddIncomingBytes(stream_write.size); + } + StreamFlowControl::IncomingUpdateContext upd(&stream->fc); + GPR_ASSERT(upd.RecvData(stream_write.size).ok()); + PerformAction(upd.MakeAction(), stream); + } + send_from_remote_.pop_front(); + } break; + case flow_control_fuzzer::Action::kStreamWrite: { + Stream* s = GetStream(action.stream_write().id()); + s->queued_writes += action.stream_write().size(); + } break; + case flow_control_fuzzer::Action::kPerformSendFromRemote: { + SendFromRemote send; + for (auto& id_stream : streams_) { + auto send_amount = std::min( + {id_stream.second.queued_writes, remote_transport_window_size_, + remote_initial_window_size_ + id_stream.second.window_delta}); + if (send_amount <= 0) continue; + send.stream_writes.push_back({id_stream.first, uint64_t(send_amount)}); + id_stream.second.queued_writes -= send_amount; + id_stream.second.window_delta -= send_amount; + remote_transport_window_size_ -= send_amount; + } + send_from_remote_.push_back(send); + } break; + case flow_control_fuzzer::Action::kSetMinProgressSize: { + Stream* s = GetStream(action.set_min_progress_size().id()); + StreamFlowControl::IncomingUpdateContext upd(&s->fc); + upd.SetMinProgressSize(action.set_min_progress_size().size()); + PerformAction(upd.MakeAction(), s); + } break; + case flow_control_fuzzer::Action::kAllocateMemory: { + auto allocate = std::min( + size_t(action.allocate_memory()), + grpc_event_engine::experimental::MemoryRequest::max_allowed_size()); + allocated_memory_ += allocate; + memory_owner_.Reserve(allocate); + } break; + case flow_control_fuzzer::Action::kDeallocateMemory: { + auto deallocate = + std::min(uint64_t(action.deallocate_memory()), allocated_memory_); + allocated_memory_ -= deallocate; + memory_owner_.Release(deallocate); + } break; + case flow_control_fuzzer::Action::kSetPendingSize: { + Stream* s = GetStream(action.set_min_progress_size().id()); + StreamFlowControl::IncomingUpdateContext upd(&s->fc); + upd.SetPendingSize(action.set_pending_size().size()); + PerformAction(upd.MakeAction(), s); + } break; + } + if (scheduled_write_) { + SendToRemote send; + if (exec_ctx.Now() >= next_bdp_ping_) { + if (auto* bdp = tfc_->bdp_estimator()) { + bdp->SchedulePing(); + bdp->StartPing(); + next_bdp_ping_ = Timestamp::InfFuture(); + send.bdp_ping = true; + } + } + if (!sending_initial_window_size_ && + queued_initial_window_size_.has_value()) { + sending_initial_window_size_ = true; + send.initial_window_size = + absl::exchange(queued_initial_window_size_, absl::nullopt); + } + while (!streams_to_update_.empty()) { + auto* stream = GetStream(streams_to_update_.front()); + streams_to_update_.pop(); + send.stream_window_updates.push_back( + {stream->id, stream->fc.MaybeSendUpdate()}); + } + send.transport_window_update = tfc_->MaybeSendUpdate(sending_payload); + queued_send_max_frame_size_.reset(); + send_to_remote_.emplace_back(std::move(send)); + scheduled_write_ = false; + } +} + +void FlowControlFuzzer::PerformAction(FlowControlAction action, + Stream* stream) { + if (!squelch) { + fprintf(stderr, "[%" PRId64 "]: ACTION: %s\n", + stream == nullptr ? int64_t(-1) : int64_t(stream->id), + action.DebugString().c_str()); + } + + auto with_urgency = [this](FlowControlAction::Urgency urgency, + std::function f) { + switch (urgency) { + case FlowControlAction::Urgency::NO_ACTION_NEEDED: + break; + case FlowControlAction::Urgency::UPDATE_IMMEDIATELY: + scheduled_write_ = true; + ABSL_FALLTHROUGH_INTENDED; + case FlowControlAction::Urgency::QUEUE_UPDATE: + f(); + break; + } + }; + with_urgency(action.send_stream_update(), + [this, stream]() { streams_to_update_.push(stream->id); }); + with_urgency(action.send_transport_update(), []() {}); + with_urgency(action.send_initial_window_update(), [this, &action]() { + GPR_ASSERT(action.initial_window_size() >= chttp2::kMinInitialWindowSize); + GPR_ASSERT(action.initial_window_size() <= chttp2::kMaxInitialWindowSize); + queued_initial_window_size_ = action.initial_window_size(); + }); + with_urgency(action.send_max_frame_size_update(), [this, &action]() { + queued_send_max_frame_size_ = action.max_frame_size(); + }); +} + +void FlowControlFuzzer::AssertNoneStuck() const { + GPR_ASSERT(!scheduled_write_); + + // Reconcile all the values to get the view of the remote that is knowable to + // the flow control system. + std::map reconciled_stream_deltas; + int64_t reconciled_transport_window = remote_transport_window_size_; + int64_t reconciled_initial_window = remote_initial_window_size_; + for (const auto& id_stream : streams_) { + reconciled_stream_deltas[id_stream.first] = id_stream.second.window_delta; + } + + // Anything that's been sent from flow control -> remote needs to be added to + // the remote. + for (const auto& send_to_remote : send_to_remote_) { + if (send_to_remote.initial_window_size.has_value()) { + reconciled_initial_window = *send_to_remote.initial_window_size; + } + reconciled_transport_window += send_to_remote.transport_window_update; + for (const auto& stream_update : send_to_remote.stream_window_updates) { + reconciled_stream_deltas[stream_update.id] += stream_update.size; + } + } + + // Anything that's been sent from remote -> flow control needs to be wound + // back into the remote. + for (const auto& send_from_remote : send_from_remote_) { + for (const auto& stream_write : send_from_remote.stream_writes) { + reconciled_stream_deltas[stream_write.id] += stream_write.size; + reconciled_transport_window += stream_write.size; + } + } + + // Finally, if a stream has indicated it's willing to read, the reconciled + // remote *MUST* be in a state where it could send at least one byte. + for (const auto& id_stream : streams_) { + if (id_stream.second.fc.min_progress_size() == 0) continue; + int64_t stream_window = + reconciled_stream_deltas[id_stream.first] + reconciled_initial_window; + if (stream_window <= 0 || reconciled_transport_window <= 0) { + fprintf(stderr, + "FAILED: stream %d has stream_window=%" PRId64 + ", transport_window=%" PRId64 ", delta=%" PRId64 + ", init_window_size=%" PRId64 ", min_progress_size=%" PRId64 "\n", + id_stream.first, stream_window, reconciled_transport_window, + reconciled_stream_deltas[id_stream.first], + reconciled_initial_window, + int64_t(id_stream.second.fc.min_progress_size())); + abort(); + } + } +} + +void FlowControlFuzzer::AssertAnnouncedOverInitialWindowSizeCorrect() const { + uint64_t value_from_streams = 0; + + for (const auto& id_stream : streams_) { + const auto& stream = id_stream.second; + if (stream.fc.announced_window_delta() > 0) { + value_from_streams += stream.fc.announced_window_delta(); + } + } + + GPR_ASSERT(value_from_streams == + tfc_->announced_stream_total_over_incoming_window()); +} + +} // namespace +} // namespace chttp2 +} // namespace grpc_core + +DEFINE_PROTO_FUZZER(const flow_control_fuzzer::Msg& msg) { + grpc_core::chttp2::InitGlobals(); + grpc_core::chttp2::FlowControlFuzzer fuzzer(msg.enable_bdp()); + for (const auto& action : msg.actions()) { + if (!squelch) { + fprintf(stderr, "%s\n", action.DebugString().c_str()); + } + fuzzer.Perform(action); + fuzzer.AssertNoneStuck(); + fuzzer.AssertAnnouncedOverInitialWindowSizeCorrect(); + } +} diff --git a/test/core/transport/chttp2/flow_control_fuzzer.proto b/test/core/transport/chttp2/flow_control_fuzzer.proto new file mode 100644 index 00000000000..ede69c2b380 --- /dev/null +++ b/test/core/transport/chttp2/flow_control_fuzzer.proto @@ -0,0 +1,47 @@ +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package flow_control_fuzzer; + +message Empty {}; + +message StreamWrite { + uint32 id = 1; + uint32 size = 2; +} + +message Action { + oneof action { + uint64 set_memory_quota = 1; + uint64 step_time_ms = 2; + Empty periodic_update = 3; + Empty perform_send_to_remote = 4; + Empty read_send_to_remote = 5; + Empty read_send_from_remote = 6; + StreamWrite stream_write = 7; + Empty perform_send_from_remote = 8; + StreamWrite set_min_progress_size = 9; + uint32 allocate_memory = 10; + uint32 deallocate_memory = 11; + Empty perform_send_to_remote_with_payload = 12; + StreamWrite set_pending_size = 13; + } +} + +message Msg { + bool enable_bdp = 1; + repeated Action actions = 2; +} diff --git a/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-141c30df1c24a5c568185ea07136150bcf700431 b/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-141c30df1c24a5c568185ea07136150bcf700431 new file mode 100644 index 00000000000..0311bb46229 --- /dev/null +++ b/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-141c30df1c24a5c568185ea07136150bcf700431 @@ -0,0 +1,90 @@ +enable_bdp: true +actions { + perform_send_from_remote { + } +} +actions { + set_min_progress_size { + id: 11008 + size: 2883584 + } +} +actions { +} +actions { + perform_send_from_remote { + } +} +actions { + stream_write { + id: 11008 + size: 2883584 + } +} +actions { + read_send_to_remote { + } +} +actions { + perform_send_to_remote { + } +} +actions { + perform_send_to_remote { + } +} +actions { + read_send_from_remote { + } +} +actions { + read_send_from_remote { + } +} +actions { + read_send_from_remote { + } +} +actions { + perform_send_from_remote { + } +} +actions { + read_send_to_remote { + } +} +actions { + read_send_to_remote { + } +} +actions { + read_send_from_remote { + } +} +actions { + read_send_from_remote { + } +} +actions { + read_send_from_remote { + } +} +actions { +} +actions { + perform_send_from_remote { + } +} +actions { + read_send_from_remote { + } +} +actions { + set_memory_quota: 0 +} +actions { + read_send_from_remote { + } +} +actions { +} diff --git a/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-29cb974543cee194b588b3f6b1b565599312b872 b/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-29cb974543cee194b588b3f6b1b565599312b872 new file mode 100644 index 00000000000..709200a8b43 --- /dev/null +++ b/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-29cb974543cee194b588b3f6b1b565599312b872 @@ -0,0 +1,14 @@ +enable_bdp: true +actions { + set_min_progress_size { + size: 11 + } +} +actions { + periodic_update { + } +} +actions { + perform_send_from_remote { + } +} diff --git a/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-da55bfa50665c7fd287257814242aed32572dd71 b/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-da55bfa50665c7fd287257814242aed32572dd71 new file mode 100644 index 00000000000..33bea18e0b9 --- /dev/null +++ b/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-da55bfa50665c7fd287257814242aed32572dd71 @@ -0,0 +1,90 @@ +enable_bdp: true +actions { + perform_send_from_remote { + } +} +actions { + perform_send_from_remote { + } +} +actions { + perform_send_from_remote { + } +} +actions { + perform_send_from_remote { + } +} +actions { + stream_write { + id: 11008 + size: 2883584 + } +} +actions { + set_min_progress_size { + id: 11008 + size: 2883584 + } +} +actions { + perform_send_to_remote { + } +} +actions { + perform_send_to_remote { + } +} +actions { + read_send_to_remote { + } +} +actions { + read_send_from_remote { + } +} +actions { + read_send_from_remote { + } +} +actions { + perform_send_from_remote { + } +} +actions { + perform_send_from_remote { + } +} +actions { + perform_send_from_remote { + } +} +actions { +} +actions { + read_send_from_remote { + } +} +actions { + read_send_from_remote { + } +} +actions { +} +actions { + perform_send_from_remote { + } +} +actions { + read_send_from_remote { + } +} +actions { + set_memory_quota: 0 +} +actions { + read_send_from_remote { + } +} +actions { +} diff --git a/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-de683f8aa2e98bfd8e803dc1c89fd4ab16331bdc b/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-de683f8aa2e98bfd8e803dc1c89fd4ab16331bdc new file mode 100644 index 00000000000..2414c58cee9 --- /dev/null +++ b/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-de683f8aa2e98bfd8e803dc1c89fd4ab16331bdc @@ -0,0 +1,40 @@ +actions { + set_min_progress_size { + size: 33554432 + } +} +actions { + stream_write { + size: 33554432 + } +} +actions { + perform_send_from_remote { + } +} +actions { +} +actions { + read_send_from_remote { + } +} +actions { + read_send_to_remote { + } +} +actions { + perform_send_from_remote { + } +} +actions { + periodic_update { + } +} +actions { + read_send_from_remote { + } +} +actions { + perform_send_from_remote { + } +} diff --git a/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-f37d0c48f53d770642738680a1988b4c1d1f3425 b/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-f37d0c48f53d770642738680a1988b4c1d1f3425 new file mode 100644 index 00000000000..90bf66b374a --- /dev/null +++ b/test/core/transport/chttp2/flow_control_fuzzer_corpus/crash-f37d0c48f53d770642738680a1988b4c1d1f3425 @@ -0,0 +1,14 @@ +enable_bdp: true +actions { + allocate_memory: 1593844738 +} +actions { + perform_send_to_remote { + } +} +actions { + set_memory_quota: 0 +} +actions { + deallocate_memory: 1593844738 +} diff --git a/test/core/transport/chttp2/flow_control_fuzzer_corpus/empty b/test/core/transport/chttp2/flow_control_fuzzer_corpus/empty new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/core/transport/chttp2/flow_control_test.cc b/test/core/transport/chttp2/flow_control_test.cc index a4c22b2c652..a80ac3011da 100644 --- a/test/core/transport/chttp2/flow_control_test.cc +++ b/test/core/transport/chttp2/flow_control_test.cc @@ -1,380 +1,171 @@ -/* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "src/core/ext/transport/chttp2/transport/flow_control.h" -#include -#include - -#include -#include -#include - -#include +#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/resource_quota/resource_quota.h" -#include "src/core/ext/filters/client_channel/backup_poller.h" -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/gprpp/host_port.h" -#include "src/core/lib/surface/channel.h" -#include "test/core/end2end/cq_verifier.h" -#include "test/core/util/port.h" -#include "test/core/util/test_config.h" +namespace grpc_core { +namespace chttp2 { namespace { - -class TransportTargetWindowSizeMocker - : public grpc_core::chttp2::TestOnlyTransportTargetWindowEstimatesMocker { - public: - static constexpr uint32_t kLargeInitialWindowSize = 1u << 31; - static constexpr uint32_t kSmallInitialWindowSize = 0; - - double ComputeNextTargetInitialWindowSizeFromPeriodicUpdate( - double /* current_target */) override { - // Protecting access to variable window_size_ shared between client and - // server. - grpc_core::MutexLock lock(&mu_); - if (alternating_initial_window_sizes_) { - window_size_ = (window_size_ == kLargeInitialWindowSize) - ? kSmallInitialWindowSize - : kLargeInitialWindowSize; - } - return window_size_; - } - - // Alternates the initial window size targets. Computes a low values if it was - // previously high, or a high value if it was previously low. - void AlternateTargetInitialWindowSizes() { - grpc_core::MutexLock lock(&mu_); - alternating_initial_window_sizes_ = true; - } - - void Reset() { - // Protecting access to variable window_size_ shared between client and - // server. - grpc_core::MutexLock lock(&mu_); - alternating_initial_window_sizes_ = false; - window_size_ = kLargeInitialWindowSize; - } - - private: - grpc_core::Mutex mu_; - bool alternating_initial_window_sizes_ ABSL_GUARDED_BY(mu_) = false; - double window_size_ ABSL_GUARDED_BY(mu_) = kLargeInitialWindowSize; -}; - -TransportTargetWindowSizeMocker* g_target_initial_window_size_mocker; - -void* tag(intptr_t t) { return reinterpret_cast(t); } - -void VerifyChannelReady(grpc_channel* channel, grpc_completion_queue* cq) { - grpc_connectivity_state state = - grpc_channel_check_connectivity_state(channel, 1 /* try_to_connect */); - while (state != GRPC_CHANNEL_READY) { - grpc_channel_watch_connectivity_state( - channel, state, grpc_timeout_seconds_to_deadline(5), cq, nullptr); - grpc_completion_queue_next(cq, grpc_timeout_seconds_to_deadline(5), - nullptr); - state = grpc_channel_check_connectivity_state(channel, 0); - } +auto* g_memory_owner = new MemoryOwner( + ResourceQuota::Default()->memory_quota()->CreateMemoryOwner("test")); } -void VerifyChannelConnected(grpc_channel* channel, grpc_completion_queue* cq) { - // Verify channel is connected. Use a ping to make sure that clients - // tries sending/receiving bytes if the channel is connected. - grpc_channel_ping(channel, cq, reinterpret_cast(2000), nullptr); - grpc_event ev = grpc_completion_queue_next( - cq, grpc_timeout_seconds_to_deadline(5), nullptr); - GPR_ASSERT(ev.type == GRPC_OP_COMPLETE); - GPR_ASSERT(ev.tag == reinterpret_cast(2000)); - GPR_ASSERT(ev.success == 1); - GPR_ASSERT(grpc_channel_check_connectivity_state(channel, 0) == - GRPC_CHANNEL_READY); +TEST(FlowControl, NoOp) { + ExecCtx exec_ctx; + TransportFlowControl tfc("test", true, g_memory_owner); + StreamFlowControl sfc(&tfc); + // Check initial values are per http2 spec + EXPECT_EQ(tfc.acked_init_window(), 65535); + EXPECT_EQ(tfc.remote_window(), 65535); + EXPECT_EQ(tfc.target_frame_size(), 16384); + EXPECT_EQ(sfc.remote_window_delta(), 0); + EXPECT_EQ(sfc.min_progress_size(), 0); + EXPECT_EQ(sfc.announced_window_delta(), 0); } -// Shuts down and destroys the server. -void ServerShutdownAndDestroy(grpc_server* server, grpc_completion_queue* cq) { - // Shutdown and destroy server - grpc_server_shutdown_and_notify(server, cq, reinterpret_cast(1000)); - while (grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME), - nullptr) - .tag != reinterpret_cast(1000)) { +TEST(FlowControl, SendData) { + ExecCtx exec_ctx; + TransportFlowControl tfc("test", true, g_memory_owner); + StreamFlowControl sfc(&tfc); + { + StreamFlowControl::OutgoingUpdateContext sfc_upd(&sfc); + sfc_upd.SentData(1024); } - grpc_server_destroy(server); + EXPECT_EQ(sfc.remote_window_delta(), -1024); + EXPECT_EQ(tfc.remote_window(), 65535 - 1024); } -grpc_slice LargeSlice(void) { - grpc_slice slice = grpc_slice_malloc(10000000); // ~10MB - memset(GRPC_SLICE_START_PTR(slice), 'x', GRPC_SLICE_LENGTH(slice)); - return slice; +TEST(FlowControl, InitialTransportUpdate) { + ExecCtx exec_ctx; + TransportFlowControl tfc("test", true, g_memory_owner); + EXPECT_EQ(TransportFlowControl::IncomingUpdateContext(&tfc).MakeAction(), + FlowControlAction()); } -void PerformCallWithLargePayload(grpc_channel* channel, grpc_server* server, - grpc_completion_queue* cq) { - grpc_slice request_payload_slice = LargeSlice(); - grpc_slice response_payload_slice = LargeSlice(); - grpc_call* c; - grpc_call* s; - grpc_byte_buffer* request_payload = - grpc_raw_byte_buffer_create(&request_payload_slice, 1); - grpc_byte_buffer* response_payload = - grpc_raw_byte_buffer_create(&response_payload_slice, 1); - cq_verifier* cqv = cq_verifier_create(cq); - grpc_op ops[6]; - grpc_op* op; - grpc_metadata_array initial_metadata_recv; - grpc_metadata_array trailing_metadata_recv; - grpc_metadata_array request_metadata_recv; - grpc_byte_buffer* request_payload_recv = nullptr; - grpc_byte_buffer* response_payload_recv = nullptr; - grpc_call_details call_details; - grpc_status_code status; - grpc_call_error error; - grpc_slice details; - int was_cancelled = 2; - - gpr_timespec deadline = grpc_timeout_seconds_to_deadline(30); - c = grpc_channel_create_call(channel, nullptr, GRPC_PROPAGATE_DEFAULTS, cq, - grpc_slice_from_static_string("/foo"), nullptr, - deadline, nullptr); - GPR_ASSERT(c); - - grpc_metadata_array_init(&initial_metadata_recv); - grpc_metadata_array_init(&trailing_metadata_recv); - grpc_metadata_array_init(&request_metadata_recv); - grpc_call_details_init(&call_details); - - memset(ops, 0, sizeof(ops)); - op = ops; - op->op = GRPC_OP_SEND_INITIAL_METADATA; - op->data.send_initial_metadata.count = 0; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_SEND_MESSAGE; - op->data.send_message.send_message = request_payload; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_RECV_INITIAL_METADATA; - op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_RECV_MESSAGE; - op->data.recv_message.recv_message = &response_payload_recv; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; - op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; - op->data.recv_status_on_client.status = &status; - op->data.recv_status_on_client.status_details = &details; - op->flags = 0; - op->reserved = nullptr; - op++; - error = grpc_call_start_batch(c, ops, static_cast(op - ops), tag(1), - nullptr); - GPR_ASSERT(GRPC_CALL_OK == error); - - error = grpc_server_request_call(server, &s, &call_details, - &request_metadata_recv, cq, cq, tag(101)); - GPR_ASSERT(GRPC_CALL_OK == error); - CQ_EXPECT_COMPLETION(cqv, tag(101), 1); - cq_verify(cqv); - - memset(ops, 0, sizeof(ops)); - op = ops; - op->op = GRPC_OP_SEND_INITIAL_METADATA; - op->data.send_initial_metadata.count = 0; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_RECV_MESSAGE; - op->data.recv_message.recv_message = &request_payload_recv; - op->flags = 0; - op->reserved = nullptr; - op++; - error = grpc_call_start_batch(s, ops, static_cast(op - ops), tag(102), - nullptr); - GPR_ASSERT(GRPC_CALL_OK == error); - - CQ_EXPECT_COMPLETION(cqv, tag(102), 1); - cq_verify(cqv); - - memset(ops, 0, sizeof(ops)); - op = ops; - op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; - op->data.recv_close_on_server.cancelled = &was_cancelled; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_SEND_MESSAGE; - op->data.send_message.send_message = response_payload; - op->flags = 0; - op->reserved = nullptr; - op++; - op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; - op->data.send_status_from_server.trailing_metadata_count = 0; - op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED; - grpc_slice status_details = grpc_slice_from_static_string("xyz"); - op->data.send_status_from_server.status_details = &status_details; - op->flags = 0; - op->reserved = nullptr; - op++; - error = grpc_call_start_batch(s, ops, static_cast(op - ops), tag(103), - nullptr); - GPR_ASSERT(GRPC_CALL_OK == error); - - CQ_EXPECT_COMPLETION(cqv, tag(103), 1); - CQ_EXPECT_COMPLETION(cqv, tag(1), 1); - cq_verify(cqv); - - GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); - GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); - GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 0); - - grpc_slice_unref(details); - grpc_metadata_array_destroy(&initial_metadata_recv); - grpc_metadata_array_destroy(&trailing_metadata_recv); - grpc_metadata_array_destroy(&request_metadata_recv); - grpc_call_details_destroy(&call_details); - - grpc_call_unref(c); - grpc_call_unref(s); - - cq_verifier_destroy(cqv); +TEST(FlowControl, InitialStreamUpdate) { + ExecCtx exec_ctx; + TransportFlowControl tfc("test", true, g_memory_owner); + StreamFlowControl sfc(&tfc); + EXPECT_EQ(StreamFlowControl::IncomingUpdateContext(&sfc).MakeAction(), + FlowControlAction()); +} - grpc_byte_buffer_destroy(request_payload); - grpc_byte_buffer_destroy(response_payload); - grpc_byte_buffer_destroy(request_payload_recv); - grpc_byte_buffer_destroy(response_payload_recv); - grpc_slice_unref(request_payload_slice); - grpc_slice_unref(response_payload_slice); +TEST(FlowControl, RecvData) { + ExecCtx exec_ctx; + TransportFlowControl tfc("test", true, g_memory_owner); + StreamFlowControl sfc(&tfc); + StreamFlowControl::IncomingUpdateContext sfc_upd(&sfc); + EXPECT_EQ(absl::OkStatus(), sfc_upd.RecvData(1024)); + sfc_upd.MakeAction(); + EXPECT_EQ(tfc.announced_window(), 65535 - 1024); + EXPECT_EQ(sfc.announced_window_delta(), -1024); } -class FlowControlTest : public ::testing::Test { - protected: - void SetUp() override { - cq_ = grpc_completion_queue_create_for_next(nullptr); - // create the server - std::string server_address = - grpc_core::JoinHostPort("localhost", grpc_pick_unused_port_or_die()); - grpc_arg server_args[] = { - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_HTTP2_MAX_PING_STRIKES), 0), - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), -1), - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH), -1)}; - grpc_channel_args server_channel_args = {GPR_ARRAY_SIZE(server_args), - server_args}; - server_ = grpc_server_create(&server_channel_args, nullptr); - grpc_server_register_completion_queue(server_, cq_, nullptr); - grpc_server_credentials* server_creds = - grpc_insecure_server_credentials_create(); - GPR_ASSERT(grpc_server_add_http2_port(server_, server_address.c_str(), - server_creds)); - grpc_server_credentials_release(server_creds); - grpc_server_start(server_); - // create the channel (bdp pings are enabled by default) - grpc_arg client_args[] = { - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA), 0), - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS), 1), - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), -1), - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH), -1)}; - grpc_channel_args client_channel_args = {GPR_ARRAY_SIZE(client_args), - client_args}; - grpc_channel_credentials* creds = grpc_insecure_credentials_create(); - channel_ = grpc_channel_create(server_address.c_str(), creds, - &client_channel_args); - grpc_channel_credentials_release(creds); - VerifyChannelReady(channel_, cq_); - g_target_initial_window_size_mocker->Reset(); +TEST(FlowControl, TrackMinProgressSize) { + ExecCtx exec_ctx; + TransportFlowControl tfc("test", true, g_memory_owner); + StreamFlowControl sfc(&tfc); + { + StreamFlowControl::IncomingUpdateContext sfc_upd(&sfc); + sfc_upd.SetMinProgressSize(5); + sfc_upd.MakeAction(); } - - void TearDown() override { - // shutdown and destroy the client and server - grpc_channel_destroy(channel_); - ServerShutdownAndDestroy(server_, cq_); - grpc_completion_queue_shutdown(cq_); - while (grpc_completion_queue_next(cq_, gpr_inf_future(GPR_CLOCK_REALTIME), - nullptr) - .type != GRPC_QUEUE_SHUTDOWN) { - } - grpc_completion_queue_destroy(cq_); + EXPECT_EQ(sfc.min_progress_size(), 5); + { + StreamFlowControl::IncomingUpdateContext sfc_upd(&sfc); + sfc_upd.SetMinProgressSize(10); + sfc_upd.MakeAction(); } + EXPECT_EQ(sfc.min_progress_size(), 10); + { + StreamFlowControl::IncomingUpdateContext sfc_upd(&sfc); + EXPECT_EQ(absl::OkStatus(), sfc_upd.RecvData(5)); + sfc_upd.MakeAction(); + } + EXPECT_EQ(sfc.min_progress_size(), 5); + { + StreamFlowControl::IncomingUpdateContext sfc_upd(&sfc); + EXPECT_EQ(absl::OkStatus(), sfc_upd.RecvData(5)); + sfc_upd.MakeAction(); + } + EXPECT_EQ(sfc.min_progress_size(), 0); + { + StreamFlowControl::IncomingUpdateContext sfc_upd(&sfc); + EXPECT_EQ(absl::OkStatus(), sfc_upd.RecvData(5)); + sfc_upd.MakeAction(); + } + EXPECT_EQ(sfc.min_progress_size(), 0); +} - grpc_server* server_ = nullptr; - grpc_channel* channel_ = nullptr; - grpc_completion_queue* cq_ = nullptr; -}; - -TEST_F(FlowControlTest, - TestLargeWindowSizeUpdatesDoNotCauseIllegalFlowControlWindows) { - for (int i = 0; i < 10; ++i) { - PerformCallWithLargePayload(channel_, server_, cq_); - VerifyChannelConnected(channel_, cq_); +TEST(FlowControl, NoUpdateWithoutReader) { + ExecCtx exec_ctx; + TransportFlowControl tfc("test", true, g_memory_owner); + StreamFlowControl sfc(&tfc); + for (int i = 0; i < 65535; i++) { + StreamFlowControl::IncomingUpdateContext sfc_upd(&sfc); + EXPECT_EQ(sfc_upd.RecvData(1), absl::OkStatus()); + EXPECT_EQ(sfc_upd.MakeAction().send_stream_update(), + FlowControlAction::Urgency::NO_ACTION_NEEDED); } + // Empty window needing 1 byte to progress should trigger an immediate read. + { + StreamFlowControl::IncomingUpdateContext sfc_upd(&sfc); + sfc_upd.SetMinProgressSize(1); + EXPECT_EQ(sfc.min_progress_size(), 1); + EXPECT_EQ(sfc_upd.MakeAction().send_stream_update(), + FlowControlAction::Urgency::UPDATE_IMMEDIATELY); + } + EXPECT_GT(tfc.MaybeSendUpdate(false), 0); + EXPECT_GT(sfc.MaybeSendUpdate(), 0); } -TEST_F(FlowControlTest, TestWindowSizeUpdatesDoNotCauseStalledStreams) { - g_target_initial_window_size_mocker->AlternateTargetInitialWindowSizes(); - for (int i = 0; i < 100; ++i) { - PerformCallWithLargePayload(channel_, server_, cq_); - VerifyChannelConnected(channel_, cq_); +TEST(FlowControl, GradualReadsUpdate) { + ExecCtx exec_ctx; + TransportFlowControl tfc("test", true, g_memory_owner); + StreamFlowControl sfc(&tfc); + int immediate_updates = 0; + int queued_updates = 0; + for (int i = 0; i < 65535; i++) { + StreamFlowControl::IncomingUpdateContext sfc_upd(&sfc); + EXPECT_EQ(sfc_upd.RecvData(1), absl::OkStatus()); + sfc_upd.SetPendingSize(0); + switch (sfc_upd.MakeAction().send_stream_update()) { + case FlowControlAction::Urgency::UPDATE_IMMEDIATELY: + immediate_updates++; + break; + case FlowControlAction::Urgency::QUEUE_UPDATE: + queued_updates++; + break; + case FlowControlAction::Urgency::NO_ACTION_NEEDED: + break; + } } + EXPECT_GT(immediate_updates, 0); + EXPECT_GT(queued_updates, 0); + EXPECT_EQ(immediate_updates + queued_updates, 65535); } -} // namespace +} // namespace chttp2 +} // namespace grpc_core int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - // Make sure that we will have an active poller on all client-side fd's that - // are capable of sending and receiving even in the case that we don't have an - // active RPC operation on the fd. - GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 1); - grpc_core::chttp2::g_test_only_transport_flow_control_window_check = true; - g_target_initial_window_size_mocker = new TransportTargetWindowSizeMocker(); - grpc_core::chttp2::g_test_only_transport_target_window_estimates_mocker = - g_target_initial_window_size_mocker; - grpc::testing::TestEnvironment env(&argc, argv); - grpc_init(); - auto result = RUN_ALL_TESTS(); - grpc_shutdown(); - return result; + return RUN_ALL_TESTS(); } diff --git a/test/core/transport/chttp2/remove_stream_from_stalled_lists_test.cc b/test/core/transport/chttp2/remove_stream_from_stalled_lists_test.cc index 7731c1eb0d2..ea381c3a472 100644 --- a/test/core/transport/chttp2/remove_stream_from_stalled_lists_test.cc +++ b/test/core/transport/chttp2/remove_stream_from_stalled_lists_test.cc @@ -232,10 +232,7 @@ class TestServer { static void HandleOneRpc(grpc_call* call, grpc_completion_queue* call_cq) { // Send a large enough payload to get us stalled on outgoing flow control - std::string send_payload = ""; - for (int i = 0; i < 4 * 1e6; i++) { - send_payload += "a"; - } + std::string send_payload(4 * 1024 * 1024, 'a'); grpc_slice request_payload_slice = grpc_slice_from_copied_string(send_payload.c_str()); grpc_byte_buffer* request_payload = diff --git a/test/core/util/grpc_fuzzer.bzl b/test/core/util/grpc_fuzzer.bzl index 157c3df63ac..80589815957 100644 --- a/test/core/util/grpc_fuzzer.bzl +++ b/test/core/util/grpc_fuzzer.bzl @@ -46,7 +46,7 @@ def grpc_fuzzer(name, corpus, srcs = [], tags = [], deps = [], data = [], size = ], size = size, args = select({ - "//:grpc_build_fuzzers": [CORPUS_DIR, "-runs=5000", "-max_total_time=300"], + "//:grpc_build_fuzzers": [CORPUS_DIR, "-runs=20000", "-max_total_time=300"], "//conditions:default": ["--directory=" + CORPUS_DIR], }), **kwargs @@ -94,7 +94,7 @@ def grpc_proto_fuzzer(name, corpus, proto, proto_deps = [], srcs = [], tags = [] ], size = size, args = select({ - "//:grpc_build_fuzzers": [CORPUS_DIR, "-runs=5000", "-max_total_time=300"], + "//:grpc_build_fuzzers": [CORPUS_DIR, "-runs=20000", "-max_total_time=300"], "//conditions:default": ["--directory=" + CORPUS_DIR], }), **kwargs diff --git a/test/cpp/end2end/client_interceptors_end2end_test.cc b/test/cpp/end2end/client_interceptors_end2end_test.cc index b736336ac22..cad71c5d3b5 100644 --- a/test/cpp/end2end/client_interceptors_end2end_test.cc +++ b/test/cpp/end2end/client_interceptors_end2end_test.cc @@ -1262,6 +1262,6 @@ int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); int ret = RUN_ALL_TESTS(); // Make sure that gRPC shuts down cleanly - GPR_ASSERT(grpc_wait_until_shutdown(1)); + GPR_ASSERT(grpc_wait_until_shutdown(10)); return ret; } diff --git a/test/cpp/interop/client_helper.h b/test/cpp/interop/client_helper.h index f361ca767a0..3b7ec7a540c 100644 --- a/test/cpp/interop/client_helper.h +++ b/test/cpp/interop/client_helper.h @@ -27,7 +27,7 @@ #include #include "src/core/lib/surface/call_test_only.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/transport/transport.h" namespace grpc { namespace testing { diff --git a/test/cpp/interop/http2_client.cc b/test/cpp/interop/http2_client.cc index d890d6a3ac1..dc1a5581433 100644 --- a/test/cpp/interop/http2_client.cc +++ b/test/cpp/interop/http2_client.cc @@ -29,7 +29,6 @@ #include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/useful.h" -#include "src/core/lib/transport/byte_stream.h" #include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/util/create_test_channel.h" diff --git a/test/cpp/interop/server_helper.cc b/test/cpp/interop/server_helper.cc index 5f1dcd5c16c..b6f9faaff12 100644 --- a/test/cpp/interop/server_helper.cc +++ b/test/cpp/interop/server_helper.cc @@ -26,7 +26,7 @@ #include #include "src/core/lib/surface/call_test_only.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/transport/transport.h" #include "test/cpp/util/test_credentials_provider.h" ABSL_DECLARE_FLAG(bool, use_alts); diff --git a/test/cpp/interop/xds_interop_server.cc b/test/cpp/interop/xds_interop_server.cc index 9d278c67cac..1b3724bd47f 100644 --- a/test/cpp/interop/xds_interop_server.cc +++ b/test/cpp/interop/xds_interop_server.cc @@ -35,7 +35,7 @@ #include "src/core/lib/gpr/string.h" #include "src/core/lib/iomgr/gethostname.h" -#include "src/core/lib/transport/byte_stream.h" +#include "src/core/lib/transport/transport.h" #include "src/proto/grpc/testing/empty.pb.h" #include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" diff --git a/test/cpp/microbenchmarks/bm_chttp2_transport.cc b/test/cpp/microbenchmarks/bm_chttp2_transport.cc index 15dbb9d8aec..415fd2c5238 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_transport.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_transport.cc @@ -412,268 +412,6 @@ static void BM_TransportEmptyOp(benchmark::State& state) { } BENCHMARK(BM_TransportEmptyOp); -static void BM_TransportStreamSend(benchmark::State& state) { - TrackCounters track_counters; - grpc_core::ExecCtx exec_ctx; - Fixture f(grpc::ChannelArguments(), true); - auto* s = new Stream(&f); - s->Init(state); - grpc_transport_stream_op_batch op; - grpc_transport_stream_op_batch_payload op_payload(nullptr); - auto reset_op = [&]() { - op = {}; - op.payload = &op_payload; - }; - // Create the send_message payload slice. - // Note: We use grpc_slice_malloc_large() instead of grpc_slice_malloc() - // to force the slice to be refcounted, so that it remains alive when it - // is unreffed after each send_message op. - grpc_slice send_slice = grpc_slice_malloc_large(state.range(0)); - memset(GRPC_SLICE_START_PTR(send_slice), 0, GRPC_SLICE_LENGTH(send_slice)); - grpc_core::ManualConstructor send_stream; - auto arena = grpc_core::MakeScopedArena(1024, g_memory_allocator); - grpc_metadata_batch b(arena.get()); - RepresentativeClientInitialMetadata::Prepare(&b); - - gpr_event* bm_done = new gpr_event; - gpr_event_init(bm_done); - - std::unique_ptr c = - MakeTestClosure([&](grpc_error_handle /*error*/) { - if (!state.KeepRunning()) { - gpr_event_set(bm_done, reinterpret_cast(1)); - return; - } - grpc_slice_buffer send_buffer; - grpc_slice_buffer_init(&send_buffer); - grpc_slice_buffer_add(&send_buffer, grpc_slice_ref(send_slice)); - send_stream.Init(&send_buffer, 0); - grpc_slice_buffer_destroy(&send_buffer); - // force outgoing window to be yuge - s->chttp2_stream()->flow_control->TestOnlyForceHugeWindow(); - f.chttp2_transport()->flow_control->TestOnlyForceHugeWindow(); - reset_op(); - op.on_complete = c.get(); - op.send_message = true; - op.payload->send_message.send_message.reset(send_stream.get()); - s->Op(&op); - }); - - reset_op(); - op.send_initial_metadata = true; - op.payload->send_initial_metadata.send_initial_metadata = &b; - op.on_complete = c.get(); - s->Op(&op); - - f.FlushExecCtx(); - gpr_event_wait(bm_done, gpr_inf_future(GPR_CLOCK_REALTIME)); - done_events.emplace_back(bm_done); - - reset_op(); - op.cancel_stream = true; - op.payload->cancel_stream.cancel_error = GRPC_ERROR_CANCELLED; - gpr_event* stream_cancel_done = new gpr_event; - gpr_event_init(stream_cancel_done); - std::unique_ptr stream_cancel_closure = - MakeTestClosure([&](grpc_error_handle error) { - GPR_ASSERT(GRPC_ERROR_IS_NONE(error)); - gpr_event_set(stream_cancel_done, reinterpret_cast(1)); - }); - op.on_complete = stream_cancel_closure.get(); - s->Op(&op); - f.FlushExecCtx(); - gpr_event_wait(stream_cancel_done, gpr_inf_future(GPR_CLOCK_REALTIME)); - done_events.emplace_back(stream_cancel_done); - s->DestroyThen( - MakeOnceClosure([s](grpc_error_handle /*error*/) { delete s; })); - f.FlushExecCtx(); - track_counters.Finish(state); - grpc_slice_unref(send_slice); -} -BENCHMARK(BM_TransportStreamSend)->Range(0, 128 * 1024 * 1024); - -#define SLICE_FROM_BUFFER(s) grpc_slice_from_static_buffer(s, sizeof(s) - 1) - -static grpc_slice CreateIncomingDataSlice(size_t length, size_t frame_size) { - std::queue unframed; - - unframed.push(static_cast(0)); - unframed.push(static_cast(length >> 24)); - unframed.push(static_cast(length >> 16)); - unframed.push(static_cast(length >> 8)); - unframed.push(static_cast(length)); - for (size_t i = 0; i < length; i++) { - unframed.push('a'); - } - - std::vector framed; - while (unframed.size() > frame_size) { - // frame size - framed.push_back(static_cast(frame_size >> 16)); - framed.push_back(static_cast(frame_size >> 8)); - framed.push_back(static_cast(frame_size)); - // data frame - framed.push_back(0); - // no flags - framed.push_back(0); - // stream id - framed.push_back(0); - framed.push_back(0); - framed.push_back(0); - framed.push_back(1); - // frame data - for (size_t i = 0; i < frame_size; i++) { - framed.push_back(unframed.front()); - unframed.pop(); - } - } - - // frame size - framed.push_back(static_cast(unframed.size() >> 16)); - framed.push_back(static_cast(unframed.size() >> 8)); - framed.push_back(static_cast(unframed.size())); - // data frame - framed.push_back(0); - // no flags - framed.push_back(0); - // stream id - framed.push_back(0); - framed.push_back(0); - framed.push_back(0); - framed.push_back(1); - while (!unframed.empty()) { - framed.push_back(unframed.front()); - unframed.pop(); - } - - return grpc_slice_from_copied_buffer(framed.data(), framed.size()); -} - -static void BM_TransportStreamRecv(benchmark::State& state) { - TrackCounters track_counters; - grpc_core::ExecCtx exec_ctx; - Fixture f(grpc::ChannelArguments(), true); - auto* s = new Stream(&f); - s->Init(state); - grpc_transport_stream_op_batch_payload op_payload(nullptr); - grpc_transport_stream_op_batch op; - grpc_core::OrphanablePtr recv_stream; - grpc_slice incoming_data = CreateIncomingDataSlice(state.range(0), 16384); - - auto reset_op = [&]() { - op = {}; - op.payload = &op_payload; - }; - - auto arena = grpc_core::MakeScopedArena(1024, g_memory_allocator); - grpc_metadata_batch b(arena.get()); - RepresentativeClientInitialMetadata::Prepare(&b); - - std::unique_ptr do_nothing = - MakeTestClosure([](grpc_error_handle /*error*/) {}); - - uint32_t received; - - std::unique_ptr drain_start; - std::unique_ptr drain; - std::unique_ptr drain_continue; - grpc_slice recv_slice; - - std::unique_ptr c = - MakeTestClosure([&](grpc_error_handle /*error*/) { - if (!state.KeepRunning()) return; - // force outgoing window to be yuge - s->chttp2_stream()->flow_control->TestOnlyForceHugeWindow(); - f.chttp2_transport()->flow_control->TestOnlyForceHugeWindow(); - received = 0; - reset_op(); - op.on_complete = do_nothing.get(); - op.recv_message = true; - op.payload->recv_message.recv_message = &recv_stream; - op.payload->recv_message.call_failed_before_recv_message = nullptr; - op.payload->recv_message.recv_message_ready = drain_start.get(); - s->Op(&op); - f.PushInput(grpc_slice_ref(incoming_data)); - }); - - drain_start = MakeTestClosure([&](grpc_error_handle /*error*/) { - if (recv_stream == nullptr) { - GPR_ASSERT(!state.KeepRunning()); - return; - } - grpc_core::Closure::Run(DEBUG_LOCATION, drain.get(), GRPC_ERROR_NONE); - }); - - drain = MakeTestClosure([&](grpc_error_handle /*error*/) { - do { - if (received == recv_stream->length()) { - recv_stream.reset(); - grpc_core::ExecCtx::Run(DEBUG_LOCATION, c.get(), GRPC_ERROR_NONE); - return; - } - } while (recv_stream->Next(recv_stream->length() - received, - drain_continue.get()) && - GRPC_ERROR_NONE == recv_stream->Pull(&recv_slice) && - (received += GRPC_SLICE_LENGTH(recv_slice), - grpc_slice_unref_internal(recv_slice), true)); - }); - - drain_continue = MakeTestClosure([&](grpc_error_handle /*error*/) { - GPR_ASSERT(GRPC_LOG_IF_ERROR("Pull", recv_stream->Pull(&recv_slice))); - received += GRPC_SLICE_LENGTH(recv_slice); - grpc_slice_unref_internal(recv_slice); - grpc_core::Closure::Run(DEBUG_LOCATION, drain.get(), GRPC_ERROR_NONE); - }); - - reset_op(); - auto b_recv = absl::make_unique(arena.get()); - op.send_initial_metadata = true; - op.payload->send_initial_metadata.send_initial_metadata = &b; - op.recv_initial_metadata = true; - op.payload->recv_initial_metadata.recv_initial_metadata = b_recv.get(); - op.payload->recv_initial_metadata.recv_initial_metadata_ready = - do_nothing.get(); - op.on_complete = c.get(); - s->Op(&op); - f.PushInput(SLICE_FROM_BUFFER( - "\x00\x00\x00\x04\x00\x00\x00\x00\x00" - // Generated using: - // tools/codegen/core/gen_header_frame.py < - // test/cpp/microbenchmarks/representative_server_initial_metadata.headers - "\x00\x00X\x01\x04\x00\x00\x00\x01" - "\x10\x07:status\x03" - "200" - "\x10\x0c" - "content-type\x10" - "application/grpc" - "\x10\x14grpc-accept-encoding\x15identity,deflate,gzip")); - - f.FlushExecCtx(); - reset_op(); - op.cancel_stream = true; - op.payload->cancel_stream.cancel_error = GRPC_ERROR_CANCELLED; - gpr_event* stream_cancel_done = new gpr_event; - gpr_event_init(stream_cancel_done); - std::unique_ptr stream_cancel_closure = - MakeTestClosure([&](grpc_error_handle error) { - GPR_ASSERT(GRPC_ERROR_IS_NONE(error)); - gpr_event_set(stream_cancel_done, reinterpret_cast(1)); - }); - op.on_complete = stream_cancel_closure.get(); - s->Op(&op); - f.FlushExecCtx(); - gpr_event_wait(stream_cancel_done, gpr_inf_future(GPR_CLOCK_REALTIME)); - done_events.emplace_back(stream_cancel_done); - s->DestroyThen(MakeOnceClosure([s, &b_recv](grpc_error_handle /*error*/) { - b_recv.reset(); - delete s; - })); - f.FlushExecCtx(); - track_counters.Finish(state); - grpc_slice_unref(incoming_data); -} -BENCHMARK(BM_TransportStreamRecv)->Range(0, 128 * 1024 * 1024); - // Some distros have RunSpecifiedBenchmarks under the benchmark namespace, // and others do not. This allows us to support both modes. namespace benchmark { diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 0ebb80e0d82..2300a3372e5 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2414,8 +2414,6 @@ src/core/lib/surface/validate_metadata.h \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ src/core/lib/transport/bdp_estimator.h \ -src/core/lib/transport/byte_stream.cc \ -src/core/lib/transport/byte_stream.h \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/connectivity_state.h \ src/core/lib/transport/error_utils.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 4e74e620a46..98727ce0989 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -2210,8 +2210,6 @@ src/core/lib/surface/version.cc \ src/core/lib/transport/README.md \ src/core/lib/transport/bdp_estimator.cc \ src/core/lib/transport/bdp_estimator.h \ -src/core/lib/transport/byte_stream.cc \ -src/core/lib/transport/byte_stream.h \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/connectivity_state.h \ src/core/lib/transport/error_utils.cc \ diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 5b0cf7e7d74..e65974fdf0b 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -1467,30 +1467,6 @@ ], "uses_polling": false }, - { - "args": [], - "benchmark": false, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "gtest": false, - "language": "c", - "name": "manual_constructor_test", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "uses_polling": false - }, { "args": [], "benchmark": false, @@ -3065,30 +3041,6 @@ ], "uses_polling": false }, - { - "args": [], - "benchmark": false, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "gtest": true, - "language": "c++", - "name": "byte_stream_test", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "uses_polling": false - }, { "args": [], "benchmark": false, @@ -4259,7 +4211,7 @@ "posix", "windows" ], - "uses_polling": true + "uses_polling": false }, { "args": [],