Revert "[transport] Remove ByteStream (#29637)" (#29890)

This reverts commit aacf0e252b.
pull/29891/head
Craig Tiller 3 years ago committed by GitHub
parent 32f1766cf5
commit d53986657f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 66
      BUILD
  2. 134
      CMakeLists.txt
  3. 2
      Makefile
  4. 108
      build_autogenerated.yaml
  5. 1
      config.m4
  6. 1
      config.w32
  7. 5
      doc/environment_variables.md
  8. 2
      gRPC-C++.podspec
  9. 3
      gRPC-Core.podspec
  10. 2
      grpc.gemspec
  11. 2
      grpc.gyp
  12. 10
      include/grpc/event_engine/slice_buffer.h
  13. 2
      package.xml
  14. 2
      src/core/ext/filters/client_channel/client_channel.cc
  15. 4
      src/core/ext/filters/client_channel/client_channel.h
  16. 46
      src/core/ext/filters/client_channel/retry_filter.cc
  17. 1
      src/core/ext/filters/client_channel/subchannel.cc
  18. 101
      src/core/ext/filters/client_channel/subchannel_stream_client.cc
  19. 14
      src/core/ext/filters/client_channel/subchannel_stream_client.h
  20. 235
      src/core/ext/filters/http/message_compress/message_compress_filter.cc
  21. 128
      src/core/ext/filters/http/message_compress/message_decompress_filter.cc
  22. 15
      src/core/ext/filters/message_size/message_size_filter.cc
  23. 1
      src/core/ext/filters/server_config_selector/server_config_selector_filter.cc
  24. 3
      src/core/ext/transport/binder/transport/binder_stream.h
  25. 34
      src/core/ext/transport/binder/transport/binder_transport.cc
  26. 548
      src/core/ext/transport/chttp2/transport/chttp2_transport.cc
  27. 370
      src/core/ext/transport/chttp2/transport/flow_control.cc
  28. 333
      src/core/ext/transport/chttp2/transport/flow_control.h
  29. 272
      src/core/ext/transport/chttp2/transport/frame_data.cc
  30. 46
      src/core/ext/transport/chttp2/transport/frame_data.h
  31. 9
      src/core/ext/transport/chttp2/transport/frame_settings.cc
  32. 11
      src/core/ext/transport/chttp2/transport/frame_window_update.cc
  33. 119
      src/core/ext/transport/chttp2/transport/internal.h
  34. 54
      src/core/ext/transport/chttp2/transport/parsing.cc
  35. 4
      src/core/ext/transport/chttp2/transport/stream_lists.cc
  36. 36
      src/core/ext/transport/chttp2/transport/writing.cc
  37. 72
      src/core/ext/transport/cronet/transport/cronet_transport.cc
  38. 92
      src/core/ext/transport/inproc/inproc_transport.cc
  39. 6
      src/core/lib/channel/call_tracer.h
  40. 67
      src/core/lib/gprpp/manual_constructor.h
  41. 1
      src/core/lib/iomgr/tcp_posix.cc
  42. 31
      src/core/lib/slice/slice_buffer.cc
  43. 43
      src/core/lib/slice/slice_buffer.h
  44. 125
      src/core/lib/surface/call.cc
  45. 165
      src/core/lib/transport/byte_stream.cc
  46. 170
      src/core/lib/transport/byte_stream.h
  47. 3
      src/core/lib/transport/transport.cc
  48. 31
      src/core/lib/transport/transport.h
  49. 9
      src/core/lib/transport/transport_op_string.cc
  50. 20
      src/cpp/common/channel_filter.h
  51. 6
      src/cpp/ext/filters/census/client_filter.cc
  52. 6
      src/cpp/ext/filters/census/open_census_call_tracer.h
  53. 3
      src/cpp/ext/filters/census/server_filter.cc
  54. 6
      src/cpp/ext/filters/census/server_filter.h
  55. 1
      src/python/grpcio/grpc_core_dependencies.py
  56. 60
      src/python/grpcio_tests/tests/unit/_compression_test.py
  57. 2
      test/core/bad_client/tests/duplicate_header.cc
  58. 118
      test/core/end2end/fixtures/h2_full_no_retry.cc
  59. 70
      test/core/end2end/generate_tests.bzl
  60. 2
      test/core/end2end/tests/request_with_flags.cc
  61. 12
      test/core/gprpp/BUILD
  62. 100
      test/core/gprpp/manual_constructor_test.cc
  63. 16
      test/core/transport/BUILD
  64. 38
      test/core/transport/binder/binder_transport_test.cc
  65. 254
      test/core/transport/byte_stream_test.cc
  66. 18
      test/core/transport/chttp2/BUILD
  67. 379
      test/core/transport/chttp2/flow_control_end2end_test.cc
  68. 441
      test/core/transport/chttp2/flow_control_test.cc
  69. 5
      test/core/transport/chttp2/remove_stream_from_stalled_lists_test.cc
  70. 2
      test/cpp/interop/client_helper.h
  71. 1
      test/cpp/interop/http2_client.cc
  72. 2
      test/cpp/interop/server_helper.cc
  73. 2
      test/cpp/interop/xds_interop_server.cc
  74. 57
      test/cpp/microbenchmarks/bm_chttp2_transport.cc
  75. 2
      tools/doxygen/Doxyfile.c++.internal
  76. 2
      tools/doxygen/Doxyfile.core.internal
  77. 72
      tools/run_tests/generated/tests.json

66
BUILD

@ -2384,23 +2384,6 @@ grpc_cc_library(
], ],
) )
grpc_cc_library(
name = "bdp_estimator",
srcs = [
"src/core/lib/transport/bdp_estimator.cc",
],
hdrs = ["src/core/lib/transport/bdp_estimator.h"],
tags = ["grpc-autodeps"],
deps = [
"exec_ctx",
"gpr_base",
"gpr_codegen",
"gpr_platform",
"grpc_trace",
"time",
],
)
grpc_cc_library( grpc_cc_library(
name = "percent_encoding", name = "percent_encoding",
srcs = [ srcs = [
@ -2523,6 +2506,8 @@ grpc_cc_library(
"src/core/lib/surface/server.cc", "src/core/lib/surface/server.cc",
"src/core/lib/surface/validate_metadata.cc", "src/core/lib/surface/validate_metadata.cc",
"src/core/lib/surface/version.cc", "src/core/lib/surface/version.cc",
"src/core/lib/transport/bdp_estimator.cc",
"src/core/lib/transport/byte_stream.cc",
"src/core/lib/transport/connectivity_state.cc", "src/core/lib/transport/connectivity_state.cc",
"src/core/lib/transport/error_utils.cc", "src/core/lib/transport/error_utils.cc",
"src/core/lib/transport/metadata_batch.cc", "src/core/lib/transport/metadata_batch.cc",
@ -2618,6 +2603,8 @@ grpc_cc_library(
"src/core/lib/surface/lame_client.h", "src/core/lib/surface/lame_client.h",
"src/core/lib/surface/server.h", "src/core/lib/surface/server.h",
"src/core/lib/surface/validate_metadata.h", "src/core/lib/surface/validate_metadata.h",
"src/core/lib/transport/bdp_estimator.h",
"src/core/lib/transport/byte_stream.h",
"src/core/lib/transport/connectivity_state.h", "src/core/lib/transport/connectivity_state.h",
"src/core/lib/transport/metadata_batch.h", "src/core/lib/transport/metadata_batch.h",
"src/core/lib/transport/parsed_metadata.h", "src/core/lib/transport/parsed_metadata.h",
@ -3072,7 +3059,6 @@ grpc_cc_library(
"absl/types:variant", "absl/types:variant",
"absl/status", "absl/status",
"absl/status:statusor", "absl/status:statusor",
"absl/utility",
"upb_lib", "upb_lib",
], ],
language = "c++", language = "c++",
@ -3084,7 +3070,6 @@ grpc_cc_library(
"channel_stack_type", "channel_stack_type",
"chunked_vector", "chunked_vector",
"config", "config",
"construct_destruct",
"debug_location", "debug_location",
"default_event_engine_factory_hdrs", "default_event_engine_factory_hdrs",
"dual_ref_counted", "dual_ref_counted",
@ -3117,7 +3102,6 @@ grpc_cc_library(
"server_address", "server_address",
"service_config_parser", "service_config_parser",
"slice", "slice",
"slice_buffer",
"slice_refcount", "slice_refcount",
"sockaddr_utils", "sockaddr_utils",
"time", "time",
@ -3350,8 +3334,8 @@ grpc_cc_library(
"grpc_public_hdrs", "grpc_public_hdrs",
"grpc_service_config", "grpc_service_config",
"json", "json",
"orphanable",
"service_config_parser", "service_config_parser",
"slice_buffer",
], ],
) )
@ -3479,7 +3463,6 @@ grpc_cc_library(
"promise", "promise",
"seq", "seq",
"slice", "slice",
"slice_buffer",
"transport_fwd", "transport_fwd",
], ],
) )
@ -5718,33 +5701,6 @@ grpc_cc_library(
], ],
) )
grpc_cc_library(
name = "chttp2_flow_control",
srcs = [
"src/core/ext/transport/chttp2/transport/flow_control.cc",
],
hdrs = [
"src/core/ext/transport/chttp2/transport/flow_control.h",
],
external_deps = [
"absl/status",
"absl/strings",
"absl/strings:str_format",
],
tags = ["grpc-autodeps"],
deps = [
"bdp_estimator",
"exec_ctx",
"gpr_base",
"gpr_platform",
"grpc_trace",
"memory_quota",
"pid_controller",
"time",
"useful",
],
)
grpc_cc_library( grpc_cc_library(
name = "grpc_transport_chttp2", name = "grpc_transport_chttp2",
srcs = [ srcs = [
@ -5752,6 +5708,7 @@ grpc_cc_library(
"src/core/ext/transport/chttp2/transport/bin_encoder.cc", "src/core/ext/transport/chttp2/transport/bin_encoder.cc",
"src/core/ext/transport/chttp2/transport/chttp2_transport.cc", "src/core/ext/transport/chttp2/transport/chttp2_transport.cc",
"src/core/ext/transport/chttp2/transport/context_list.cc", "src/core/ext/transport/chttp2/transport/context_list.cc",
"src/core/ext/transport/chttp2/transport/flow_control.cc",
"src/core/ext/transport/chttp2/transport/frame_data.cc", "src/core/ext/transport/chttp2/transport/frame_data.cc",
"src/core/ext/transport/chttp2/transport/frame_goaway.cc", "src/core/ext/transport/chttp2/transport/frame_goaway.cc",
"src/core/ext/transport/chttp2/transport/frame_ping.cc", "src/core/ext/transport/chttp2/transport/frame_ping.cc",
@ -5774,6 +5731,7 @@ grpc_cc_library(
"src/core/ext/transport/chttp2/transport/bin_encoder.h", "src/core/ext/transport/chttp2/transport/bin_encoder.h",
"src/core/ext/transport/chttp2/transport/chttp2_transport.h", "src/core/ext/transport/chttp2/transport/chttp2_transport.h",
"src/core/ext/transport/chttp2/transport/context_list.h", "src/core/ext/transport/chttp2/transport/context_list.h",
"src/core/ext/transport/chttp2/transport/flow_control.h",
"src/core/ext/transport/chttp2/transport/frame.h", "src/core/ext/transport/chttp2/transport/frame.h",
"src/core/ext/transport/chttp2/transport/frame_data.h", "src/core/ext/transport/chttp2/transport/frame_data.h",
"src/core/ext/transport/chttp2/transport/frame_goaway.h", "src/core/ext/transport/chttp2/transport/frame_goaway.h",
@ -5806,9 +5764,7 @@ grpc_cc_library(
visibility = ["@grpc:grpclb"], visibility = ["@grpc:grpclb"],
deps = [ deps = [
"arena", "arena",
"bdp_estimator",
"bitset", "bitset",
"chttp2_flow_control",
"chunked_vector", "chunked_vector",
"debug_location", "debug_location",
"gpr_base", "gpr_base",
@ -5826,13 +5782,11 @@ grpc_cc_library(
"memory_quota", "memory_quota",
"orphanable", "orphanable",
"pid_controller", "pid_controller",
"poll",
"ref_counted", "ref_counted",
"ref_counted_ptr", "ref_counted_ptr",
"resource_quota", "resource_quota",
"resource_quota_trace", "resource_quota_trace",
"slice", "slice",
"slice_buffer",
"slice_refcount", "slice_refcount",
"time", "time",
"uri_parser", "uri_parser",
@ -5959,10 +5913,8 @@ grpc_cc_library(
"absl/status:statusor", "absl/status:statusor",
"absl/strings", "absl/strings",
"absl/types:optional", "absl/types:optional",
"absl/utility",
], ],
language = "c++", language = "c++",
tags = ["grpc-autodeps"],
deps = [ deps = [
"arena", "arena",
"channel_args_preconditioning", "channel_args_preconditioning",
@ -5972,12 +5924,11 @@ grpc_cc_library(
"gpr_base", "gpr_base",
"grpc_base", "grpc_base",
"grpc_codegen", "grpc_codegen",
"grpc_public_hdrs",
"grpc_trace", "grpc_trace",
"iomgr_fwd", "iomgr_fwd",
"orphanable",
"ref_counted_ptr", "ref_counted_ptr",
"slice", "slice",
"slice_buffer",
"time", "time",
"transport_fwd", "transport_fwd",
"useful", "useful",
@ -6562,7 +6513,6 @@ grpc_cc_library(
"grpc++_base", "grpc++_base",
"grpc_base", "grpc_base",
"slice", "slice",
"slice_buffer",
"slice_refcount", "slice_refcount",
], ],
) )

134
CMakeLists.txt generated

@ -851,6 +851,7 @@ if(gRPC_BUILD_TESTS)
add_dependencies(buildtests_c jwt_verifier_test) add_dependencies(buildtests_c jwt_verifier_test)
add_dependencies(buildtests_c lame_client_test) add_dependencies(buildtests_c lame_client_test)
add_dependencies(buildtests_c load_file_test) add_dependencies(buildtests_c load_file_test)
add_dependencies(buildtests_c manual_constructor_test)
if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX)
add_dependencies(buildtests_c memory_quota_stress_test) add_dependencies(buildtests_c memory_quota_stress_test)
endif() endif()
@ -953,6 +954,7 @@ if(gRPC_BUILD_TESTS)
add_dependencies(buildtests_cxx binder_transport_test) add_dependencies(buildtests_cxx binder_transport_test)
add_dependencies(buildtests_cxx bitset_test) add_dependencies(buildtests_cxx bitset_test)
add_dependencies(buildtests_cxx byte_buffer_test) add_dependencies(buildtests_cxx byte_buffer_test)
add_dependencies(buildtests_cxx byte_stream_test)
add_dependencies(buildtests_cxx call_finalization_test) add_dependencies(buildtests_cxx call_finalization_test)
add_dependencies(buildtests_cxx call_push_pull_test) add_dependencies(buildtests_cxx call_push_pull_test)
add_dependencies(buildtests_cxx cancel_ares_query_test) add_dependencies(buildtests_cxx cancel_ares_query_test)
@ -1014,7 +1016,6 @@ if(gRPC_BUILD_TESTS)
add_dependencies(buildtests_cxx file_watcher_certificate_provider_factory_test) add_dependencies(buildtests_cxx file_watcher_certificate_provider_factory_test)
add_dependencies(buildtests_cxx filter_end2end_test) add_dependencies(buildtests_cxx filter_end2end_test)
add_dependencies(buildtests_cxx flaky_network_test) add_dependencies(buildtests_cxx flaky_network_test)
add_dependencies(buildtests_cxx flow_control_end2end_test)
add_dependencies(buildtests_cxx flow_control_test) add_dependencies(buildtests_cxx flow_control_test)
add_dependencies(buildtests_cxx for_each_test) add_dependencies(buildtests_cxx for_each_test)
add_dependencies(buildtests_cxx generic_end2end_test) add_dependencies(buildtests_cxx generic_end2end_test)
@ -2242,6 +2243,7 @@ add_library(grpc
src/core/lib/surface/validate_metadata.cc src/core/lib/surface/validate_metadata.cc
src/core/lib/surface/version.cc src/core/lib/surface/version.cc
src/core/lib/transport/bdp_estimator.cc src/core/lib/transport/bdp_estimator.cc
src/core/lib/transport/byte_stream.cc
src/core/lib/transport/connectivity_state.cc src/core/lib/transport/connectivity_state.cc
src/core/lib/transport/error_utils.cc src/core/lib/transport/error_utils.cc
src/core/lib/transport/handshaker.cc src/core/lib/transport/handshaker.cc
@ -2829,6 +2831,7 @@ add_library(grpc_unsecure
src/core/lib/surface/validate_metadata.cc src/core/lib/surface/validate_metadata.cc
src/core/lib/surface/version.cc src/core/lib/surface/version.cc
src/core/lib/transport/bdp_estimator.cc src/core/lib/transport/bdp_estimator.cc
src/core/lib/transport/byte_stream.cc
src/core/lib/transport/connectivity_state.cc src/core/lib/transport/connectivity_state.cc
src/core/lib/transport/error_utils.cc src/core/lib/transport/error_utils.cc
src/core/lib/transport/handshaker.cc src/core/lib/transport/handshaker.cc
@ -6003,6 +6006,33 @@ target_link_libraries(load_file_test
) )
endif()
if(gRPC_BUILD_TESTS)
add_executable(manual_constructor_test
test/core/gprpp/manual_constructor_test.cc
)
target_include_directories(manual_constructor_test
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/include
${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
${_gRPC_RE2_INCLUDE_DIR}
${_gRPC_SSL_INCLUDE_DIR}
${_gRPC_UPB_GENERATED_DIR}
${_gRPC_UPB_GRPC_GENERATED_DIR}
${_gRPC_UPB_INCLUDE_DIR}
${_gRPC_XXHASH_INCLUDE_DIR}
${_gRPC_ZLIB_INCLUDE_DIR}
)
target_link_libraries(manual_constructor_test
${_gRPC_ALLTARGETS_LIBRARIES}
grpc_test_util
)
endif() endif()
if(gRPC_BUILD_TESTS) if(gRPC_BUILD_TESTS)
if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX)
@ -8325,6 +8355,41 @@ target_link_libraries(byte_buffer_test
) )
endif()
if(gRPC_BUILD_TESTS)
add_executable(byte_stream_test
test/core/transport/byte_stream_test.cc
third_party/googletest/googletest/src/gtest-all.cc
third_party/googletest/googlemock/src/gmock-all.cc
)
target_include_directories(byte_stream_test
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/include
${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
${_gRPC_RE2_INCLUDE_DIR}
${_gRPC_SSL_INCLUDE_DIR}
${_gRPC_UPB_GENERATED_DIR}
${_gRPC_UPB_GRPC_GENERATED_DIR}
${_gRPC_UPB_INCLUDE_DIR}
${_gRPC_XXHASH_INCLUDE_DIR}
${_gRPC_ZLIB_INCLUDE_DIR}
third_party/googletest/googletest/include
third_party/googletest/googletest
third_party/googletest/googlemock/include
third_party/googletest/googlemock
${_gRPC_PROTO_GENS_DIR}
)
target_link_libraries(byte_stream_test
${_gRPC_PROTOBUF_LIBRARIES}
${_gRPC_ALLTARGETS_LIBRARIES}
grpc_test_util
)
endif() endif()
if(gRPC_BUILD_TESTS) if(gRPC_BUILD_TESTS)
@ -10666,69 +10731,11 @@ target_link_libraries(flaky_network_test
) )
endif()
if(gRPC_BUILD_TESTS)
add_executable(flow_control_end2end_test
test/core/end2end/cq_verifier.cc
test/core/transport/chttp2/flow_control_end2end_test.cc
third_party/googletest/googletest/src/gtest-all.cc
third_party/googletest/googlemock/src/gmock-all.cc
)
target_include_directories(flow_control_end2end_test
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/include
${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
${_gRPC_RE2_INCLUDE_DIR}
${_gRPC_SSL_INCLUDE_DIR}
${_gRPC_UPB_GENERATED_DIR}
${_gRPC_UPB_GRPC_GENERATED_DIR}
${_gRPC_UPB_INCLUDE_DIR}
${_gRPC_XXHASH_INCLUDE_DIR}
${_gRPC_ZLIB_INCLUDE_DIR}
third_party/googletest/googletest/include
third_party/googletest/googletest
third_party/googletest/googlemock/include
third_party/googletest/googlemock
${_gRPC_PROTO_GENS_DIR}
)
target_link_libraries(flow_control_end2end_test
${_gRPC_PROTOBUF_LIBRARIES}
${_gRPC_ALLTARGETS_LIBRARIES}
grpc_test_util
)
endif() endif()
if(gRPC_BUILD_TESTS) if(gRPC_BUILD_TESTS)
add_executable(flow_control_test add_executable(flow_control_test
src/core/ext/transport/chttp2/transport/flow_control.cc test/core/end2end/cq_verifier.cc
src/core/ext/upb-generated/google/protobuf/any.upb.c
src/core/ext/upb-generated/google/rpc/status.upb.c
src/core/lib/debug/trace.cc
src/core/lib/event_engine/memory_allocator.cc
src/core/lib/gprpp/status_helper.cc
src/core/lib/gprpp/time.cc
src/core/lib/iomgr/combiner.cc
src/core/lib/iomgr/error.cc
src/core/lib/iomgr/exec_ctx.cc
src/core/lib/iomgr/executor.cc
src/core/lib/iomgr/iomgr_internal.cc
src/core/lib/promise/activity.cc
src/core/lib/resource_quota/memory_quota.cc
src/core/lib/resource_quota/resource_quota.cc
src/core/lib/resource_quota/thread_quota.cc
src/core/lib/resource_quota/trace.cc
src/core/lib/slice/percent_encoding.cc
src/core/lib/slice/slice.cc
src/core/lib/slice/slice_refcount.cc
src/core/lib/slice/slice_string_helpers.cc
src/core/lib/transport/bdp_estimator.cc
src/core/lib/transport/pid_controller.cc
test/core/transport/chttp2/flow_control_test.cc test/core/transport/chttp2/flow_control_test.cc
third_party/googletest/googletest/src/gtest-all.cc third_party/googletest/googletest/src/gtest-all.cc
third_party/googletest/googlemock/src/gmock-all.cc third_party/googletest/googlemock/src/gmock-all.cc
@ -10756,12 +10763,7 @@ target_include_directories(flow_control_test
target_link_libraries(flow_control_test target_link_libraries(flow_control_test
${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_PROTOBUF_LIBRARIES}
${_gRPC_ALLTARGETS_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES}
absl::type_traits grpc_test_util
absl::statusor
absl::variant
absl::utility
gpr
upb
) )

2
Makefile generated

@ -1635,6 +1635,7 @@ LIBGRPC_SRC = \
src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/validate_metadata.cc \
src/core/lib/surface/version.cc \ src/core/lib/surface/version.cc \
src/core/lib/transport/bdp_estimator.cc \ src/core/lib/transport/bdp_estimator.cc \
src/core/lib/transport/byte_stream.cc \
src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/connectivity_state.cc \
src/core/lib/transport/error_utils.cc \ src/core/lib/transport/error_utils.cc \
src/core/lib/transport/handshaker.cc \ src/core/lib/transport/handshaker.cc \
@ -2062,6 +2063,7 @@ LIBGRPC_UNSECURE_SRC = \
src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/validate_metadata.cc \
src/core/lib/surface/version.cc \ src/core/lib/surface/version.cc \
src/core/lib/transport/bdp_estimator.cc \ src/core/lib/transport/bdp_estimator.cc \
src/core/lib/transport/byte_stream.cc \
src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/connectivity_state.cc \
src/core/lib/transport/error_utils.cc \ src/core/lib/transport/error_utils.cc \
src/core/lib/transport/handshaker.cc \ src/core/lib/transport/handshaker.cc \

@ -946,6 +946,7 @@ libs:
- src/core/lib/surface/server.h - src/core/lib/surface/server.h
- src/core/lib/surface/validate_metadata.h - src/core/lib/surface/validate_metadata.h
- src/core/lib/transport/bdp_estimator.h - src/core/lib/transport/bdp_estimator.h
- src/core/lib/transport/byte_stream.h
- src/core/lib/transport/connectivity_state.h - src/core/lib/transport/connectivity_state.h
- src/core/lib/transport/error_utils.h - src/core/lib/transport/error_utils.h
- src/core/lib/transport/handshaker.h - src/core/lib/transport/handshaker.h
@ -1613,6 +1614,7 @@ libs:
- src/core/lib/surface/validate_metadata.cc - src/core/lib/surface/validate_metadata.cc
- src/core/lib/surface/version.cc - src/core/lib/surface/version.cc
- src/core/lib/transport/bdp_estimator.cc - src/core/lib/transport/bdp_estimator.cc
- src/core/lib/transport/byte_stream.cc
- src/core/lib/transport/connectivity_state.cc - src/core/lib/transport/connectivity_state.cc
- src/core/lib/transport/error_utils.cc - src/core/lib/transport/error_utils.cc
- src/core/lib/transport/handshaker.cc - src/core/lib/transport/handshaker.cc
@ -2121,6 +2123,7 @@ libs:
- src/core/lib/surface/server.h - src/core/lib/surface/server.h
- src/core/lib/surface/validate_metadata.h - src/core/lib/surface/validate_metadata.h
- src/core/lib/transport/bdp_estimator.h - src/core/lib/transport/bdp_estimator.h
- src/core/lib/transport/byte_stream.h
- src/core/lib/transport/connectivity_state.h - src/core/lib/transport/connectivity_state.h
- src/core/lib/transport/error_utils.h - src/core/lib/transport/error_utils.h
- src/core/lib/transport/handshaker.h - src/core/lib/transport/handshaker.h
@ -2431,6 +2434,7 @@ libs:
- src/core/lib/surface/validate_metadata.cc - src/core/lib/surface/validate_metadata.cc
- src/core/lib/surface/version.cc - src/core/lib/surface/version.cc
- src/core/lib/transport/bdp_estimator.cc - src/core/lib/transport/bdp_estimator.cc
- src/core/lib/transport/byte_stream.cc
- src/core/lib/transport/connectivity_state.cc - src/core/lib/transport/connectivity_state.cc
- src/core/lib/transport/error_utils.cc - src/core/lib/transport/error_utils.cc
- src/core/lib/transport/handshaker.cc - src/core/lib/transport/handshaker.cc
@ -3807,6 +3811,15 @@ targets:
deps: deps:
- grpc_test_util - grpc_test_util
uses_polling: false uses_polling: false
- name: manual_constructor_test
build: test
language: c
headers: []
src:
- test/core/gprpp/manual_constructor_test.cc
deps:
- grpc_test_util
uses_polling: false
- name: memory_quota_stress_test - name: memory_quota_stress_test
build: test build: test
language: c language: c
@ -4881,6 +4894,16 @@ targets:
deps: deps:
- grpc++_test_util - grpc++_test_util
uses_polling: false uses_polling: false
- name: byte_stream_test
gtest: true
build: test
language: c++
headers: []
src:
- test/core/transport/byte_stream_test.cc
deps:
- grpc_test_util
uses_polling: false
- name: call_finalization_test - name: call_finalization_test
gtest: true gtest: true
build: test build: test
@ -5859,7 +5882,7 @@ targets:
- test/cpp/end2end/test_service_impl.cc - test/cpp/end2end/test_service_impl.cc
deps: deps:
- grpc++_test_util - grpc++_test_util
- name: flow_control_end2end_test - name: flow_control_test
gtest: true gtest: true
build: test build: test
language: c++ language: c++
@ -5867,90 +5890,9 @@ targets:
- test/core/end2end/cq_verifier.h - test/core/end2end/cq_verifier.h
src: src:
- test/core/end2end/cq_verifier.cc - test/core/end2end/cq_verifier.cc
- test/core/transport/chttp2/flow_control_end2end_test.cc
deps:
- grpc_test_util
- name: flow_control_test
gtest: true
build: test
language: c++
headers:
- src/core/ext/transport/chttp2/transport/flow_control.h
- src/core/ext/upb-generated/google/protobuf/any.upb.h
- src/core/ext/upb-generated/google/rpc/status.upb.h
- src/core/lib/debug/trace.h
- src/core/lib/gprpp/atomic_utils.h
- src/core/lib/gprpp/bitset.h
- src/core/lib/gprpp/cpp_impl_of.h
- src/core/lib/gprpp/orphanable.h
- src/core/lib/gprpp/ref_counted.h
- src/core/lib/gprpp/ref_counted_ptr.h
- src/core/lib/gprpp/status_helper.h
- src/core/lib/gprpp/time.h
- src/core/lib/iomgr/closure.h
- src/core/lib/iomgr/combiner.h
- src/core/lib/iomgr/error.h
- src/core/lib/iomgr/error_internal.h
- src/core/lib/iomgr/exec_ctx.h
- src/core/lib/iomgr/executor.h
- src/core/lib/iomgr/iomgr_internal.h
- src/core/lib/promise/activity.h
- src/core/lib/promise/context.h
- src/core/lib/promise/detail/basic_seq.h
- src/core/lib/promise/detail/promise_factory.h
- src/core/lib/promise/detail/promise_like.h
- src/core/lib/promise/detail/status.h
- src/core/lib/promise/detail/switch.h
- src/core/lib/promise/exec_ctx_wakeup_scheduler.h
- src/core/lib/promise/loop.h
- src/core/lib/promise/map.h
- src/core/lib/promise/poll.h
- src/core/lib/promise/race.h
- src/core/lib/promise/seq.h
- src/core/lib/resource_quota/memory_quota.h
- src/core/lib/resource_quota/resource_quota.h
- src/core/lib/resource_quota/thread_quota.h
- src/core/lib/resource_quota/trace.h
- src/core/lib/slice/percent_encoding.h
- src/core/lib/slice/slice.h
- src/core/lib/slice/slice_internal.h
- src/core/lib/slice/slice_refcount.h
- src/core/lib/slice/slice_refcount_base.h
- src/core/lib/slice/slice_string_helpers.h
- src/core/lib/transport/bdp_estimator.h
- src/core/lib/transport/pid_controller.h
src:
- src/core/ext/transport/chttp2/transport/flow_control.cc
- src/core/ext/upb-generated/google/protobuf/any.upb.c
- src/core/ext/upb-generated/google/rpc/status.upb.c
- src/core/lib/debug/trace.cc
- src/core/lib/event_engine/memory_allocator.cc
- src/core/lib/gprpp/status_helper.cc
- src/core/lib/gprpp/time.cc
- src/core/lib/iomgr/combiner.cc
- src/core/lib/iomgr/error.cc
- src/core/lib/iomgr/exec_ctx.cc
- src/core/lib/iomgr/executor.cc
- src/core/lib/iomgr/iomgr_internal.cc
- src/core/lib/promise/activity.cc
- src/core/lib/resource_quota/memory_quota.cc
- src/core/lib/resource_quota/resource_quota.cc
- src/core/lib/resource_quota/thread_quota.cc
- src/core/lib/resource_quota/trace.cc
- src/core/lib/slice/percent_encoding.cc
- src/core/lib/slice/slice.cc
- src/core/lib/slice/slice_refcount.cc
- src/core/lib/slice/slice_string_helpers.cc
- src/core/lib/transport/bdp_estimator.cc
- src/core/lib/transport/pid_controller.cc
- test/core/transport/chttp2/flow_control_test.cc - test/core/transport/chttp2/flow_control_test.cc
deps: deps:
- absl/meta:type_traits - grpc_test_util
- absl/status:statusor
- absl/types:variant
- absl/utility:utility
- gpr
- upb
- name: for_each_test - name: for_each_test
gtest: true gtest: true
build: test build: test

1
config.m4 generated

@ -703,6 +703,7 @@ if test "$PHP_GRPC" != "no"; then
src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/validate_metadata.cc \
src/core/lib/surface/version.cc \ src/core/lib/surface/version.cc \
src/core/lib/transport/bdp_estimator.cc \ src/core/lib/transport/bdp_estimator.cc \
src/core/lib/transport/byte_stream.cc \
src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/connectivity_state.cc \
src/core/lib/transport/error_utils.cc \ src/core/lib/transport/error_utils.cc \
src/core/lib/transport/handshaker.cc \ src/core/lib/transport/handshaker.cc \

1
config.w32 generated

@ -669,6 +669,7 @@ if (PHP_GRPC != "no") {
"src\\core\\lib\\surface\\validate_metadata.cc " + "src\\core\\lib\\surface\\validate_metadata.cc " +
"src\\core\\lib\\surface\\version.cc " + "src\\core\\lib\\surface\\version.cc " +
"src\\core\\lib\\transport\\bdp_estimator.cc " + "src\\core\\lib\\transport\\bdp_estimator.cc " +
"src\\core\\lib\\transport\\byte_stream.cc " +
"src\\core\\lib\\transport\\connectivity_state.cc " + "src\\core\\lib\\transport\\connectivity_state.cc " +
"src\\core\\lib\\transport\\error_utils.cc " + "src\\core\\lib\\transport\\error_utils.cc " +
"src\\core\\lib\\transport\\handshaker.cc " + "src\\core\\lib\\transport\\handshaker.cc " +

@ -167,6 +167,11 @@ some configuration as environment variables that can be set.
channels (mostly due to idleness), so that the next RPC on this channel won't channels (mostly due to idleness), so that the next RPC on this channel won't
fail. Set to 0 to turn off the backup polls. fail. Set to 0 to turn off the backup polls.
* GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL
if set, flow control will be effectively disabled. Max out all values and
assume the remote peer does the same. Thus we can ignore any flow control
bookkeeping, error checking, and decision making
* grpc_cfstream * grpc_cfstream
set to 1 to turn on CFStream experiment. With this experiment gRPC uses CFStream API to make TCP set to 1 to turn on CFStream experiment. With this experiment gRPC uses CFStream API to make TCP
connections. The option is only available on iOS platform and when macro GRPC_CFSTREAM is defined. connections. The option is only available on iOS platform and when macro GRPC_CFSTREAM is defined.

2
gRPC-C++.podspec generated

@ -904,6 +904,7 @@ Pod::Spec.new do |s|
'src/core/lib/surface/server.h', 'src/core/lib/surface/server.h',
'src/core/lib/surface/validate_metadata.h', 'src/core/lib/surface/validate_metadata.h',
'src/core/lib/transport/bdp_estimator.h', 'src/core/lib/transport/bdp_estimator.h',
'src/core/lib/transport/byte_stream.h',
'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/connectivity_state.h',
'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/error_utils.h',
'src/core/lib/transport/handshaker.h', 'src/core/lib/transport/handshaker.h',
@ -1724,6 +1725,7 @@ Pod::Spec.new do |s|
'src/core/lib/surface/server.h', 'src/core/lib/surface/server.h',
'src/core/lib/surface/validate_metadata.h', 'src/core/lib/surface/validate_metadata.h',
'src/core/lib/transport/bdp_estimator.h', 'src/core/lib/transport/bdp_estimator.h',
'src/core/lib/transport/byte_stream.h',
'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/connectivity_state.h',
'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/error_utils.h',
'src/core/lib/transport/handshaker.h', 'src/core/lib/transport/handshaker.h',

3
gRPC-Core.podspec generated

@ -1507,6 +1507,8 @@ Pod::Spec.new do |s|
'src/core/lib/surface/version.cc', 'src/core/lib/surface/version.cc',
'src/core/lib/transport/bdp_estimator.cc', 'src/core/lib/transport/bdp_estimator.cc',
'src/core/lib/transport/bdp_estimator.h', 'src/core/lib/transport/bdp_estimator.h',
'src/core/lib/transport/byte_stream.cc',
'src/core/lib/transport/byte_stream.h',
'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/connectivity_state.cc',
'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/connectivity_state.h',
'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/error_utils.cc',
@ -2325,6 +2327,7 @@ Pod::Spec.new do |s|
'src/core/lib/surface/server.h', 'src/core/lib/surface/server.h',
'src/core/lib/surface/validate_metadata.h', 'src/core/lib/surface/validate_metadata.h',
'src/core/lib/transport/bdp_estimator.h', 'src/core/lib/transport/bdp_estimator.h',
'src/core/lib/transport/byte_stream.h',
'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/connectivity_state.h',
'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/error_utils.h',
'src/core/lib/transport/handshaker.h', 'src/core/lib/transport/handshaker.h',

2
grpc.gemspec generated

@ -1420,6 +1420,8 @@ Gem::Specification.new do |s|
s.files += %w( src/core/lib/surface/version.cc ) s.files += %w( src/core/lib/surface/version.cc )
s.files += %w( src/core/lib/transport/bdp_estimator.cc ) s.files += %w( src/core/lib/transport/bdp_estimator.cc )
s.files += %w( src/core/lib/transport/bdp_estimator.h ) s.files += %w( src/core/lib/transport/bdp_estimator.h )
s.files += %w( src/core/lib/transport/byte_stream.cc )
s.files += %w( src/core/lib/transport/byte_stream.h )
s.files += %w( src/core/lib/transport/connectivity_state.cc ) s.files += %w( src/core/lib/transport/connectivity_state.cc )
s.files += %w( src/core/lib/transport/connectivity_state.h ) s.files += %w( src/core/lib/transport/connectivity_state.h )
s.files += %w( src/core/lib/transport/error_utils.cc ) s.files += %w( src/core/lib/transport/error_utils.cc )

2
grpc.gyp generated

@ -992,6 +992,7 @@
'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/validate_metadata.cc',
'src/core/lib/surface/version.cc', 'src/core/lib/surface/version.cc',
'src/core/lib/transport/bdp_estimator.cc', 'src/core/lib/transport/bdp_estimator.cc',
'src/core/lib/transport/byte_stream.cc',
'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/connectivity_state.cc',
'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/error_utils.cc',
'src/core/lib/transport/handshaker.cc', 'src/core/lib/transport/handshaker.cc',
@ -1411,6 +1412,7 @@
'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/validate_metadata.cc',
'src/core/lib/surface/version.cc', 'src/core/lib/surface/version.cc',
'src/core/lib/transport/bdp_estimator.cc', 'src/core/lib/transport/bdp_estimator.cc',
'src/core/lib/transport/byte_stream.cc',
'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/connectivity_state.cc',
'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/error_utils.cc',
'src/core/lib/transport/handshaker.cc', 'src/core/lib/transport/handshaker.cc',

@ -54,19 +54,13 @@ class SliceBuffer {
SliceBuffer(const SliceBuffer& other) = delete; SliceBuffer(const SliceBuffer& other) = delete;
SliceBuffer(SliceBuffer&& other) noexcept SliceBuffer(SliceBuffer&& other) noexcept
: slice_buffer_(other.slice_buffer_) { : slice_buffer_(other.slice_buffer_) {
grpc_slice_buffer_init(&slice_buffer_); grpc_slice_buffer_reset_and_unref(&slice_buffer_);
grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_); grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_);
} }
/// Upon destruction, the underlying raw slice buffer is cleaned out and all /// Upon destruction, the underlying raw slice buffer is cleaned out and all
/// slices are unreffed. /// slices are unreffed.
~SliceBuffer() { grpc_slice_buffer_destroy(&slice_buffer_); } ~SliceBuffer() { grpc_slice_buffer_destroy(&slice_buffer_); }
SliceBuffer& operator=(const SliceBuffer&) = delete;
SliceBuffer& operator=(SliceBuffer&& other) noexcept {
grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_);
return *this;
}
/// Appends a new slice into the SliceBuffer and makes an attempt to merge /// Appends a new slice into the SliceBuffer and makes an attempt to merge
/// this slice with the last slice in the SliceBuffer. /// this slice with the last slice in the SliceBuffer.
void Append(Slice slice); void Append(Slice slice);
@ -105,7 +99,7 @@ class SliceBuffer {
size_t Length() { return slice_buffer_.length; } size_t Length() { return slice_buffer_.length; }
/// Return a pointer to the back raw grpc_slice_buffer /// Return a pointer to the back raw grpc_slice_buffer
grpc_slice_buffer* c_slice_buffer() { return &slice_buffer_; } grpc_slice_buffer* RawSliceBuffer() { return &slice_buffer_; }
private: private:
/// The backing raw slice buffer. /// The backing raw slice buffer.

2
package.xml generated

@ -1402,6 +1402,8 @@
<file baseinstalldir="/" name="src/core/lib/surface/version.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/surface/version.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/bdp_estimator.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/transport/bdp_estimator.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/bdp_estimator.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/transport/bdp_estimator.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/byte_stream.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/byte_stream.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/connectivity_state.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/transport/connectivity_state.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/connectivity_state.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/transport/connectivity_state.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/transport/error_utils.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/transport/error_utils.cc" role="src" />

@ -2956,7 +2956,7 @@ void ClientChannel::LoadBalancedCall::RecvMessageReady(
gpr_log(GPR_INFO, "chand=%p lb_call=%p: got recv_message_ready: error=%s", gpr_log(GPR_INFO, "chand=%p lb_call=%p: got recv_message_ready: error=%s",
self->chand_, self, grpc_error_std_string(error).c_str()); self->chand_, self, grpc_error_std_string(error).c_str());
} }
if (self->recv_message_->has_value()) { if (*self->recv_message_ != nullptr) {
self->call_attempt_tracer_->RecordReceivedMessage(**self->recv_message_); self->call_attempt_tracer_->RecordReceivedMessage(**self->recv_message_);
} }
Closure::Run(DEBUG_LOCATION, self->original_recv_message_ready_, Closure::Run(DEBUG_LOCATION, self->original_recv_message_ready_,

@ -68,8 +68,8 @@
#include "src/core/lib/service_config/service_config_call_data.h" #include "src/core/lib/service_config/service_config_call_data.h"
#include "src/core/lib/service_config/service_config_parser.h" #include "src/core/lib/service_config/service_config_parser.h"
#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice.h"
#include "src/core/lib/slice/slice_buffer.h"
#include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/channel.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/connectivity_state.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -534,7 +534,7 @@ class ClientChannel::LoadBalancedCall
grpc_closure* original_recv_initial_metadata_ready_ = nullptr; grpc_closure* original_recv_initial_metadata_ready_ = nullptr;
// For intercepting recv_message_ready. // For intercepting recv_message_ready.
absl::optional<SliceBuffer>* recv_message_ = nullptr; OrphanablePtr<ByteStream>* recv_message_ = nullptr;
grpc_closure recv_message_ready_; grpc_closure recv_message_ready_;
grpc_closure* original_recv_message_ready_ = nullptr; grpc_closure* original_recv_message_ready_ = nullptr;

@ -33,7 +33,6 @@
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "absl/strings/strip.h" #include "absl/strings/strip.h"
#include "absl/types/optional.h" #include "absl/types/optional.h"
#include "absl/utility/utility.h"
#include <grpc/impl/codegen/grpc_types.h> #include <grpc/impl/codegen/grpc_types.h>
#include <grpc/slice.h> #include <grpc/slice.h>
@ -52,8 +51,8 @@
#include "src/core/lib/channel/status_util.h" #include "src/core/lib/channel/status_util.h"
#include "src/core/lib/debug/trace.h" #include "src/core/lib/debug/trace.h"
#include "src/core/lib/gpr/useful.h" #include "src/core/lib/gpr/useful.h"
#include "src/core/lib/gprpp/construct_destruct.h"
#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/manual_constructor.h"
#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted.h"
#include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/ref_counted_ptr.h"
@ -67,8 +66,8 @@
#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/arena.h"
#include "src/core/lib/service_config/service_config.h" #include "src/core/lib/service_config/service_config.h"
#include "src/core/lib/service_config/service_config_call_data.h" #include "src/core/lib/service_config/service_config_call_data.h"
#include "src/core/lib/slice/slice_buffer.h"
#include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/slice/slice_refcount.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/error_utils.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -457,6 +456,9 @@ class RetryFilter::CallData {
grpc_transport_stream_op_batch_payload batch_payload_; grpc_transport_stream_op_batch_payload batch_payload_;
// For send_initial_metadata. // For send_initial_metadata.
grpc_metadata_batch send_initial_metadata_{calld_->arena_}; grpc_metadata_batch send_initial_metadata_{calld_->arena_};
// For send_message.
// TODO(roth): Restructure this to eliminate use of ManualConstructor.
ManualConstructor<ByteStreamCache::CachingByteStream> send_message_;
// For send_trailing_metadata. // For send_trailing_metadata.
grpc_metadata_batch send_trailing_metadata_{calld_->arena_}; grpc_metadata_batch send_trailing_metadata_{calld_->arena_};
// For intercepting recv_initial_metadata. // For intercepting recv_initial_metadata.
@ -465,8 +467,7 @@ class RetryFilter::CallData {
bool trailing_metadata_available_ = false; bool trailing_metadata_available_ = false;
// For intercepting recv_message. // For intercepting recv_message.
grpc_closure recv_message_ready_; grpc_closure recv_message_ready_;
absl::optional<SliceBuffer> recv_message_; OrphanablePtr<ByteStream> recv_message_;
uint32_t recv_message_flags_;
// For intercepting recv_trailing_metadata. // For intercepting recv_trailing_metadata.
grpc_metadata_batch recv_trailing_metadata_{calld_->arena_}; grpc_metadata_batch recv_trailing_metadata_{calld_->arena_};
grpc_transport_stream_stats collect_stats_; grpc_transport_stream_stats collect_stats_;
@ -627,11 +628,11 @@ class RetryFilter::CallData {
// Note: We inline the cache for the first 3 send_message ops and use // Note: We inline the cache for the first 3 send_message ops and use
// dynamic allocation after that. This number was essentially picked // dynamic allocation after that. This number was essentially picked
// at random; it could be changed in the future to tune performance. // at random; it could be changed in the future to tune performance.
struct CachedSendMessage { // TODO(roth): As part of implementing hedging, we may need some
SliceBuffer* slices; // synchronization here, since ByteStreamCache does not provide any
uint32_t flags; // synchronization, so it's not safe to have multiple
}; // CachingByteStreams read from the same ByteStreamCache concurrently.
absl::InlinedVector<CachedSendMessage, 3> send_messages_; absl::InlinedVector<ByteStreamCache*, 3> send_messages_;
// send_trailing_metadata // send_trailing_metadata
bool seen_send_trailing_metadata_ = false; bool seen_send_trailing_metadata_ = false;
grpc_metadata_batch send_trailing_metadata_{arena_}; grpc_metadata_batch send_trailing_metadata_{arena_};
@ -1510,8 +1511,6 @@ void RetryFilter::CallData::CallAttempt::BatchData::
// Return payload. // Return payload.
*pending->batch->payload->recv_message.recv_message = *pending->batch->payload->recv_message.recv_message =
std::move(call_attempt_->recv_message_); std::move(call_attempt_->recv_message_);
*pending->batch->payload->recv_message.flags =
call_attempt_->recv_message_flags_;
// Update bookkeeping. // Update bookkeeping.
// Note: Need to do this before invoking the callback, since invoking // Note: Need to do this before invoking the callback, since invoking
// the callback will result in yielding the call combiner. // the callback will result in yielding the call combiner.
@ -1556,7 +1555,7 @@ void RetryFilter::CallData::CallAttempt::BatchData::RecvMessageReady(
// the recv_trailing_metadata_ready callback, then defer propagating this // the recv_trailing_metadata_ready callback, then defer propagating this
// callback back to the surface. We can evaluate whether to retry when // callback back to the surface. We can evaluate whether to retry when
// recv_trailing_metadata comes back. // recv_trailing_metadata comes back.
if (GPR_UNLIKELY((!call_attempt->recv_message_.has_value() || if (GPR_UNLIKELY((call_attempt->recv_message_ == nullptr ||
error != GRPC_ERROR_NONE) && error != GRPC_ERROR_NONE) &&
!call_attempt->completed_recv_trailing_metadata_)) { !call_attempt->completed_recv_trailing_metadata_)) {
if (GRPC_TRACE_FLAG_ENABLED(grpc_retry_trace)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_retry_trace)) {
@ -2030,12 +2029,13 @@ void RetryFilter::CallData::CallAttempt::BatchData::
calld->chand_, calld, call_attempt_.get(), calld->chand_, calld, call_attempt_.get(),
call_attempt_->started_send_message_count_); call_attempt_->started_send_message_count_);
} }
CachedSendMessage cache = ByteStreamCache* cache =
calld->send_messages_[call_attempt_->started_send_message_count_]; calld->send_messages_[call_attempt_->started_send_message_count_];
++call_attempt_->started_send_message_count_; ++call_attempt_->started_send_message_count_;
call_attempt_->send_message_.Init(cache);
batch_.send_message = true; batch_.send_message = true;
batch_.payload->send_message.send_message = cache.slices; batch_.payload->send_message.send_message.reset(
batch_.payload->send_message.flags = cache.flags; call_attempt_->send_message_.get());
} }
void RetryFilter::CallData::CallAttempt::BatchData:: void RetryFilter::CallData::CallAttempt::BatchData::
@ -2072,7 +2072,6 @@ void RetryFilter::CallData::CallAttempt::BatchData::
++call_attempt_->started_recv_message_count_; ++call_attempt_->started_recv_message_count_;
batch_.recv_message = true; batch_.recv_message = true;
batch_.payload->recv_message.recv_message = &call_attempt_->recv_message_; batch_.payload->recv_message.recv_message = &call_attempt_->recv_message_;
batch_.payload->recv_message.flags = &call_attempt_->recv_message_flags_;
batch_.payload->recv_message.call_failed_before_recv_message = nullptr; batch_.payload->recv_message.call_failed_before_recv_message = nullptr;
GRPC_CLOSURE_INIT(&call_attempt_->recv_message_ready_, RecvMessageReady, this, GRPC_CLOSURE_INIT(&call_attempt_->recv_message_ready_, RecvMessageReady, this,
grpc_schedule_on_exec_ctx); grpc_schedule_on_exec_ctx);
@ -2373,9 +2372,9 @@ void RetryFilter::CallData::MaybeCacheSendOpsForBatch(PendingBatch* pending) {
} }
// Set up cache for send_message ops. // Set up cache for send_message ops.
if (batch->send_message) { if (batch->send_message) {
SliceBuffer* cache = arena_->New<SliceBuffer>(std::move( ByteStreamCache* cache = arena_->New<ByteStreamCache>(
*absl::exchange(batch->payload->send_message.send_message, nullptr))); std::move(batch->payload->send_message.send_message));
send_messages_.push_back({cache, batch->payload->send_message.flags}); send_messages_.push_back(cache);
} }
// Save metadata batch for send_trailing_metadata ops. // Save metadata batch for send_trailing_metadata ops.
if (batch->send_trailing_metadata) { if (batch->send_trailing_metadata) {
@ -2395,13 +2394,14 @@ void RetryFilter::CallData::FreeCachedSendInitialMetadata() {
} }
void RetryFilter::CallData::FreeCachedSendMessage(size_t idx) { void RetryFilter::CallData::FreeCachedSendMessage(size_t idx) {
if (send_messages_[idx].slices != nullptr) { if (send_messages_[idx] != nullptr) {
if (GRPC_TRACE_FLAG_ENABLED(grpc_retry_trace)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_retry_trace)) {
gpr_log(GPR_INFO, gpr_log(GPR_INFO,
"chand=%p calld=%p: destroying send_messages[%" PRIuPTR "]", "chand=%p calld=%p: destroying send_messages[%" PRIuPTR "]",
chand_, this, idx); chand_, this, idx);
} }
Destruct(absl::exchange(send_messages_[idx].slices, nullptr)); send_messages_[idx]->Destroy();
send_messages_[idx] = nullptr;
} }
} }
@ -2465,7 +2465,7 @@ RetryFilter::CallData::PendingBatch* RetryFilter::CallData::PendingBatchesAdd(
if (batch->send_message) { if (batch->send_message) {
pending_send_message_ = true; pending_send_message_ = true;
bytes_buffered_for_retry_ += bytes_buffered_for_retry_ +=
batch->payload->send_message.send_message->Length(); batch->payload->send_message.send_message->length();
} }
if (batch->send_trailing_metadata) { if (batch->send_trailing_metadata) {
pending_send_trailing_metadata_ = true; pending_send_trailing_metadata_ = true;

@ -25,6 +25,7 @@
#include <cstring> #include <cstring>
#include <memory> #include <memory>
#include <new> #include <new>
#include <type_traits>
#include <utility> #include <utility>
#include "absl/status/statusor.h" #include "absl/status/statusor.h"

@ -20,11 +20,15 @@
#include <inttypes.h> #include <inttypes.h>
#include <stdio.h> #include <stdio.h>
#include <string.h>
#include <cstdint>
#include <string> #include <string>
#include <utility> #include <utility>
#include <grpc/slice_buffer.h>
#include <grpc/status.h> #include <grpc/status.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h> #include <grpc/support/log.h>
#include "src/core/lib/gpr/time_precise.h" #include "src/core/lib/gpr/time_precise.h"
@ -34,6 +38,7 @@
#include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/exec_ctx.h"
#include "src/core/lib/resource_quota/api.h" #include "src/core/lib/resource_quota/api.h"
#include "src/core/lib/resource_quota/resource_quota.h" #include "src/core/lib/resource_quota/resource_quota.h"
#include "src/core/lib/slice/slice_internal.h"
#include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/error_utils.h"
#define SUBCHANNEL_STREAM_INITIAL_CONNECT_BACKOFF_SECONDS 1 #define SUBCHANNEL_STREAM_INITIAL_CONNECT_BACKOFF_SECONDS 1
@ -247,9 +252,14 @@ void SubchannelStreamClient::CallState::StartCallLocked() {
payload_.send_initial_metadata.peer_string = nullptr; payload_.send_initial_metadata.peer_string = nullptr;
batch_.send_initial_metadata = true; batch_.send_initial_metadata = true;
// Add send_message op. // Add send_message op.
send_message_.Append(Slice( grpc_slice request_slice =
subchannel_stream_client_->event_handler_->EncodeSendMessageLocked())); subchannel_stream_client_->event_handler_->EncodeSendMessageLocked();
payload_.send_message.send_message = &send_message_; grpc_slice_buffer slice_buffer;
grpc_slice_buffer_init(&slice_buffer);
grpc_slice_buffer_add(&slice_buffer, request_slice);
send_message_.emplace(&slice_buffer, 0);
grpc_slice_buffer_destroy_internal(&slice_buffer);
payload_.send_message.send_message.reset(&*send_message_);
batch_.send_message = true; batch_.send_message = true;
// Add send_trailing_metadata op. // Add send_trailing_metadata op.
payload_.send_trailing_metadata.send_trailing_metadata = payload_.send_trailing_metadata.send_trailing_metadata =
@ -364,18 +374,42 @@ void SubchannelStreamClient::CallState::RecvInitialMetadataReady(
self->call_->Unref(DEBUG_LOCATION, "recv_initial_metadata_ready"); self->call_->Unref(DEBUG_LOCATION, "recv_initial_metadata_ready");
} }
void SubchannelStreamClient::CallState::RecvMessageReady() { void SubchannelStreamClient::CallState::DoneReadingRecvMessage(
if (!recv_message_.has_value()) { grpc_error_handle error) {
recv_message_.reset();
if (error != GRPC_ERROR_NONE) {
GRPC_ERROR_UNREF(error);
Cancel();
grpc_slice_buffer_destroy_internal(&recv_message_buffer_);
call_->Unref(DEBUG_LOCATION, "recv_message_ready"); call_->Unref(DEBUG_LOCATION, "recv_message_ready");
return; return;
} }
// Concatenate the slices to form a single string.
std::unique_ptr<uint8_t> recv_message_deleter;
uint8_t* recv_message;
if (recv_message_buffer_.count == 1) {
recv_message = GRPC_SLICE_START_PTR(recv_message_buffer_.slices[0]);
} else {
recv_message =
static_cast<uint8_t*>(gpr_malloc(recv_message_buffer_.length));
recv_message_deleter.reset(recv_message);
size_t offset = 0;
for (size_t i = 0; i < recv_message_buffer_.count; ++i) {
memcpy(recv_message + offset,
GRPC_SLICE_START_PTR(recv_message_buffer_.slices[i]),
GRPC_SLICE_LENGTH(recv_message_buffer_.slices[i]));
offset += GRPC_SLICE_LENGTH(recv_message_buffer_.slices[i]);
}
}
// Report payload. // Report payload.
{ {
MutexLock lock(&subchannel_stream_client_->mu_); MutexLock lock(&subchannel_stream_client_->mu_);
if (subchannel_stream_client_->event_handler_ != nullptr) { if (subchannel_stream_client_->event_handler_ != nullptr) {
absl::string_view serialized_message(
reinterpret_cast<char*>(recv_message), recv_message_buffer_.length);
absl::Status status = absl::Status status =
subchannel_stream_client_->event_handler_->RecvMessageReadyLocked( subchannel_stream_client_->event_handler_->RecvMessageReadyLocked(
subchannel_stream_client_.get(), recv_message_->JoinIntoString()); subchannel_stream_client_.get(), serialized_message);
if (!status.ok()) { if (!status.ok()) {
if (GPR_UNLIKELY(subchannel_stream_client_->tracer_ != nullptr)) { if (GPR_UNLIKELY(subchannel_stream_client_->tracer_ != nullptr)) {
gpr_log(GPR_INFO, gpr_log(GPR_INFO,
@ -390,7 +424,7 @@ void SubchannelStreamClient::CallState::RecvMessageReady() {
} }
} }
seen_response_.store(true, std::memory_order_release); seen_response_.store(true, std::memory_order_release);
recv_message_.reset(); grpc_slice_buffer_destroy_internal(&recv_message_buffer_);
// Start another recv_message batch. // Start another recv_message batch.
// This re-uses the ref we're holding. // This re-uses the ref we're holding.
// Note: Can't just reuse batch_ here, since we don't know that all // Note: Can't just reuse batch_ here, since we don't know that all
@ -404,11 +438,62 @@ void SubchannelStreamClient::CallState::RecvMessageReady() {
StartBatch(&recv_message_batch_); StartBatch(&recv_message_batch_);
} }
grpc_error_handle
SubchannelStreamClient::CallState::PullSliceFromRecvMessage() {
grpc_slice slice;
grpc_error_handle error = recv_message_->Pull(&slice);
if (error == GRPC_ERROR_NONE) {
grpc_slice_buffer_add(&recv_message_buffer_, slice);
}
return error;
}
void SubchannelStreamClient::CallState::ContinueReadingRecvMessage() {
while (recv_message_->Next(SIZE_MAX, &recv_message_ready_)) {
grpc_error_handle error = PullSliceFromRecvMessage();
if (error != GRPC_ERROR_NONE) {
DoneReadingRecvMessage(error);
return;
}
if (recv_message_buffer_.length == recv_message_->length()) {
DoneReadingRecvMessage(GRPC_ERROR_NONE);
break;
}
}
}
void SubchannelStreamClient::CallState::OnByteStreamNext(
void* arg, grpc_error_handle error) {
auto* self = static_cast<SubchannelStreamClient::CallState*>(arg);
if (error != GRPC_ERROR_NONE) {
self->DoneReadingRecvMessage(GRPC_ERROR_REF(error));
return;
}
error = self->PullSliceFromRecvMessage();
if (error != GRPC_ERROR_NONE) {
self->DoneReadingRecvMessage(error);
return;
}
if (self->recv_message_buffer_.length == self->recv_message_->length()) {
self->DoneReadingRecvMessage(GRPC_ERROR_NONE);
} else {
self->ContinueReadingRecvMessage();
}
}
void SubchannelStreamClient::CallState::RecvMessageReady( void SubchannelStreamClient::CallState::RecvMessageReady(
void* arg, grpc_error_handle /*error*/) { void* arg, grpc_error_handle /*error*/) {
auto* self = static_cast<SubchannelStreamClient::CallState*>(arg); auto* self = static_cast<SubchannelStreamClient::CallState*>(arg);
GRPC_CALL_COMBINER_STOP(&self->call_combiner_, "recv_message_ready"); GRPC_CALL_COMBINER_STOP(&self->call_combiner_, "recv_message_ready");
self->RecvMessageReady(); if (self->recv_message_ == nullptr) {
self->call_->Unref(DEBUG_LOCATION, "recv_message_ready");
return;
}
grpc_slice_buffer_init(&self->recv_message_buffer_);
GRPC_CLOSURE_INIT(&self->recv_message_ready_, OnByteStreamNext, self,
grpc_schedule_on_exec_ctx);
self->ContinueReadingRecvMessage();
// Ref will continue to be held until we finish draining the byte stream.
} }
void SubchannelStreamClient::CallState::RecvTrailingMetadataReady( void SubchannelStreamClient::CallState::RecvTrailingMetadataReady(

@ -46,7 +46,7 @@
#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/arena.h"
#include "src/core/lib/resource_quota/memory_quota.h" #include "src/core/lib/resource_quota/memory_quota.h"
#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -132,8 +132,6 @@ class SubchannelStreamClient
void CallEndedLocked(bool retry) void CallEndedLocked(bool retry)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(&subchannel_stream_client_->mu_); ABSL_EXCLUSIVE_LOCKS_REQUIRED(&subchannel_stream_client_->mu_);
void RecvMessageReady();
static void OnComplete(void* arg, grpc_error_handle error); static void OnComplete(void* arg, grpc_error_handle error);
static void RecvInitialMetadataReady(void* arg, grpc_error_handle error); static void RecvInitialMetadataReady(void* arg, grpc_error_handle error);
static void RecvMessageReady(void* arg, grpc_error_handle error); static void RecvMessageReady(void* arg, grpc_error_handle error);
@ -141,6 +139,11 @@ class SubchannelStreamClient
static void StartCancel(void* arg, grpc_error_handle error); static void StartCancel(void* arg, grpc_error_handle error);
static void OnCancelComplete(void* arg, grpc_error_handle error); static void OnCancelComplete(void* arg, grpc_error_handle error);
static void OnByteStreamNext(void* arg, grpc_error_handle error);
void ContinueReadingRecvMessage();
grpc_error_handle PullSliceFromRecvMessage();
void DoneReadingRecvMessage(grpc_error_handle error);
static void AfterCallStackDestruction(void* arg, grpc_error_handle error); static void AfterCallStackDestruction(void* arg, grpc_error_handle error);
RefCountedPtr<SubchannelStreamClient> subchannel_stream_client_; RefCountedPtr<SubchannelStreamClient> subchannel_stream_client_;
@ -166,7 +169,7 @@ class SubchannelStreamClient
grpc_metadata_batch send_initial_metadata_; grpc_metadata_batch send_initial_metadata_;
// send_message // send_message
SliceBuffer send_message_; absl::optional<SliceBufferByteStream> send_message_;
// send_trailing_metadata // send_trailing_metadata
grpc_metadata_batch send_trailing_metadata_; grpc_metadata_batch send_trailing_metadata_;
@ -176,8 +179,9 @@ class SubchannelStreamClient
grpc_closure recv_initial_metadata_ready_; grpc_closure recv_initial_metadata_ready_;
// recv_message // recv_message
absl::optional<SliceBuffer> recv_message_; OrphanablePtr<ByteStream> recv_message_;
grpc_closure recv_message_ready_; grpc_closure recv_message_ready_;
grpc_slice_buffer recv_message_buffer_;
std::atomic<bool> seen_response_{false}; std::atomic<bool> seen_response_{false};
// True if the cancel_stream batch has been started. // True if the cancel_stream batch has been started.

@ -23,26 +23,32 @@
#include <inttypes.h> #include <inttypes.h>
#include <stdlib.h> #include <stdlib.h>
#include <memory>
#include <new> #include <new>
#include <type_traits>
#include "absl/meta/type_traits.h" #include "absl/meta/type_traits.h"
#include "absl/types/optional.h" #include "absl/types/optional.h"
#include "absl/utility/utility.h"
#include <grpc/compression.h> #include <grpc/compression.h>
#include <grpc/impl/codegen/compression_types.h> #include <grpc/impl/codegen/compression_types.h>
#include <grpc/impl/codegen/grpc_types.h> #include <grpc/impl/codegen/grpc_types.h>
#include <grpc/slice.h>
#include <grpc/slice_buffer.h>
#include <grpc/support/log.h> #include <grpc/support/log.h>
#include "src/core/lib/compression/compression_internal.h" #include "src/core/lib/compression/compression_internal.h"
#include "src/core/lib/compression/message_compress.h" #include "src/core/lib/compression/message_compress.h"
#include "src/core/lib/debug/trace.h" #include "src/core/lib/debug/trace.h"
#include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/call_combiner.h"
#include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/closure.h"
#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/error.h"
#include "src/core/lib/profiling/timers.h" #include "src/core/lib/profiling/timers.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_internal.h"
#include "src/core/lib/surface/call.h" #include "src/core/lib/surface/call.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -100,40 +106,66 @@ class CallData {
channeld->default_compression_algorithm()))) { channeld->default_compression_algorithm()))) {
compression_algorithm_ = channeld->default_compression_algorithm(); compression_algorithm_ = channeld->default_compression_algorithm();
} }
GRPC_CLOSURE_INIT(&forward_send_message_batch_in_call_combiner_, GRPC_CLOSURE_INIT(&start_send_message_batch_in_call_combiner_,
ForwardSendMessageBatch, elem, grpc_schedule_on_exec_ctx); StartSendMessageBatch, elem, grpc_schedule_on_exec_ctx);
} }
~CallData() { GRPC_ERROR_UNREF(cancel_error_); } ~CallData() {
if (state_initialized_) {
grpc_slice_buffer_destroy_internal(&slices_);
}
GRPC_ERROR_UNREF(cancel_error_);
}
void CompressStartTransportStreamOpBatch( void CompressStartTransportStreamOpBatch(
grpc_call_element* elem, grpc_transport_stream_op_batch* batch); grpc_call_element* elem, grpc_transport_stream_op_batch* batch);
private: private:
bool SkipMessageCompression(); bool SkipMessageCompression();
void FinishSendMessage(grpc_call_element* elem); void InitializeState(grpc_call_element* elem);
void ProcessSendInitialMetadata(grpc_call_element* elem, void ProcessSendInitialMetadata(grpc_call_element* elem,
grpc_metadata_batch* initial_metadata); grpc_metadata_batch* initial_metadata);
// Methods for processing a send_message batch // Methods for processing a send_message batch
static void StartSendMessageBatch(void* elem_arg, grpc_error_handle unused);
static void OnSendMessageNextDone(void* elem_arg, grpc_error_handle error);
grpc_error_handle PullSliceFromSendMessage();
void ContinueReadingSendMessage(grpc_call_element* elem);
void FinishSendMessage(grpc_call_element* elem);
void SendMessageBatchContinue(grpc_call_element* elem);
static void FailSendMessageBatchInCallCombiner(void* calld_arg, static void FailSendMessageBatchInCallCombiner(void* calld_arg,
grpc_error_handle error); grpc_error_handle error);
static void ForwardSendMessageBatch(void* elem_arg, grpc_error_handle unused);
static void SendMessageOnComplete(void* calld_arg, grpc_error_handle error);
grpc_core::CallCombiner* call_combiner_; grpc_core::CallCombiner* call_combiner_;
grpc_compression_algorithm compression_algorithm_ = GRPC_COMPRESS_NONE; grpc_compression_algorithm compression_algorithm_ = GRPC_COMPRESS_NONE;
grpc_error_handle cancel_error_ = GRPC_ERROR_NONE; grpc_error_handle cancel_error_ = GRPC_ERROR_NONE;
grpc_transport_stream_op_batch* send_message_batch_ = nullptr; grpc_transport_stream_op_batch* send_message_batch_ = nullptr;
bool seen_initial_metadata_ = false; bool seen_initial_metadata_ = false;
grpc_closure forward_send_message_batch_in_call_combiner_; /* Set to true, if the fields below are initialized. */
bool state_initialized_ = false;
grpc_closure start_send_message_batch_in_call_combiner_;
/* The fields below are only initialized when we compress the payload.
* Keep them at the bottom of the struct, so they don't pollute the
* cache-lines. */
grpc_slice_buffer slices_; /**< Buffers up input slices to be compressed */
// Allocate space for the replacement stream
std::aligned_storage<sizeof(grpc_core::SliceBufferByteStream),
alignof(grpc_core::SliceBufferByteStream)>::type
replacement_stream_;
grpc_closure* original_send_message_on_complete_ = nullptr;
grpc_closure send_message_on_complete_;
grpc_closure on_send_message_next_done_;
}; };
// Returns true if we should skip message compression for the current message. // Returns true if we should skip message compression for the current message.
bool CallData::SkipMessageCompression() { bool CallData::SkipMessageCompression() {
// If the flags of this message indicate that it shouldn't be compressed, we // If the flags of this message indicate that it shouldn't be compressed, we
// skip message compression. // skip message compression.
uint32_t flags = send_message_batch_->payload->send_message.flags; uint32_t flags =
send_message_batch_->payload->send_message.send_message->flags();
if (flags & (GRPC_WRITE_NO_COMPRESS | GRPC_WRITE_INTERNAL_COMPRESS)) { if (flags & (GRPC_WRITE_NO_COMPRESS | GRPC_WRITE_INTERNAL_COMPRESS)) {
return true; return true;
} }
@ -142,6 +174,16 @@ bool CallData::SkipMessageCompression() {
return compression_algorithm_ == GRPC_COMPRESS_NONE; return compression_algorithm_ == GRPC_COMPRESS_NONE;
} }
void CallData::InitializeState(grpc_call_element* elem) {
GPR_DEBUG_ASSERT(!state_initialized_);
state_initialized_ = true;
grpc_slice_buffer_init(&slices_);
GRPC_CLOSURE_INIT(&send_message_on_complete_, SendMessageOnComplete, this,
grpc_schedule_on_exec_ctx);
GRPC_CLOSURE_INIT(&on_send_message_next_done_, OnSendMessageNextDone, elem,
grpc_schedule_on_exec_ctx);
}
void CallData::ProcessSendInitialMetadata( void CallData::ProcessSendInitialMetadata(
grpc_call_element* elem, grpc_metadata_batch* initial_metadata) { grpc_call_element* elem, grpc_metadata_batch* initial_metadata) {
ChannelData* channeld = static_cast<ChannelData*>(elem->channel_data); ChannelData* channeld = static_cast<ChannelData*>(elem->channel_data);
@ -154,6 +196,7 @@ void CallData::ProcessSendInitialMetadata(
break; break;
case GRPC_COMPRESS_DEFLATE: case GRPC_COMPRESS_DEFLATE:
case GRPC_COMPRESS_GZIP: case GRPC_COMPRESS_GZIP:
InitializeState(elem);
initial_metadata->Set(grpc_core::GrpcEncodingMetadata(), initial_metadata->Set(grpc_core::GrpcEncodingMetadata(),
compression_algorithm_); compression_algorithm_);
break; break;
@ -165,46 +208,68 @@ void CallData::ProcessSendInitialMetadata(
channeld->enabled_compression_algorithms()); channeld->enabled_compression_algorithms());
} }
void CallData::SendMessageOnComplete(void* calld_arg, grpc_error_handle error) {
CallData* calld = static_cast<CallData*>(calld_arg);
grpc_slice_buffer_reset_and_unref_internal(&calld->slices_);
grpc_core::Closure::Run(DEBUG_LOCATION,
calld->original_send_message_on_complete_,
GRPC_ERROR_REF(error));
}
void CallData::SendMessageBatchContinue(grpc_call_element* elem) {
// Note: The call to grpc_call_next_op() results in yielding the
// call combiner, so we need to clear send_message_batch_ before we do that.
grpc_transport_stream_op_batch* send_message_batch = send_message_batch_;
send_message_batch_ = nullptr;
grpc_call_next_op(elem, send_message_batch);
}
void CallData::FinishSendMessage(grpc_call_element* elem) { void CallData::FinishSendMessage(grpc_call_element* elem) {
GPR_DEBUG_ASSERT(compression_algorithm_ != GRPC_COMPRESS_NONE);
// Compress the data if appropriate. // Compress the data if appropriate.
if (!SkipMessageCompression()) { grpc_slice_buffer tmp;
grpc_core::SliceBuffer tmp; grpc_slice_buffer_init(&tmp);
uint32_t& send_flags = send_message_batch_->payload->send_message.flags; uint32_t send_flags =
grpc_core::SliceBuffer* payload = send_message_batch_->payload->send_message.send_message->flags();
send_message_batch_->payload->send_message.send_message; bool did_compress = grpc_msg_compress(compression_algorithm_, &slices_, &tmp);
bool did_compress = if (did_compress) {
grpc_msg_compress(compression_algorithm_, payload->c_slice_buffer(), if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) {
tmp.c_slice_buffer()); const char* algo_name;
if (did_compress) { const size_t before_size = slices_.length;
if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { const size_t after_size = tmp.length;
const char* algo_name; const float savings_ratio = 1.0f - static_cast<float>(after_size) /
const size_t before_size = payload->Length(); static_cast<float>(before_size);
const size_t after_size = tmp.Length(); GPR_ASSERT(
const float savings_ratio = 1.0f - static_cast<float>(after_size) / grpc_compression_algorithm_name(compression_algorithm_, &algo_name));
static_cast<float>(before_size); gpr_log(GPR_INFO,
GPR_ASSERT(grpc_compression_algorithm_name(compression_algorithm_, "Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR
&algo_name)); " bytes (%.2f%% savings)",
gpr_log(GPR_INFO, algo_name, before_size, after_size, 100 * savings_ratio);
"Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR }
" bytes (%.2f%% savings)", grpc_slice_buffer_swap(&slices_, &tmp);
algo_name, before_size, after_size, 100 * savings_ratio); send_flags |= GRPC_WRITE_INTERNAL_COMPRESS;
} } else {
tmp.Swap(payload); if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) {
send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; const char* algo_name;
} else { GPR_ASSERT(
if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { grpc_compression_algorithm_name(compression_algorithm_, &algo_name));
const char* algo_name; gpr_log(GPR_INFO,
GPR_ASSERT(grpc_compression_algorithm_name(compression_algorithm_, "Algorithm '%s' enabled but decided not to compress. Input size: "
&algo_name)); "%" PRIuPTR,
gpr_log( algo_name, slices_.length);
GPR_INFO,
"Algorithm '%s' enabled but decided not to compress. Input size: "
"%" PRIuPTR,
algo_name, payload->Length());
}
} }
} }
grpc_call_next_op(elem, absl::exchange(send_message_batch_, nullptr)); grpc_slice_buffer_destroy_internal(&tmp);
// Swap out the original byte stream with our new one and send the
// batch down.
new (&replacement_stream_)
grpc_core::SliceBufferByteStream(&slices_, send_flags);
send_message_batch_->payload->send_message.send_message.reset(
reinterpret_cast<grpc_core::SliceBufferByteStream*>(
&replacement_stream_));
original_send_message_on_complete_ = send_message_batch_->on_complete;
send_message_batch_->on_complete = &send_message_on_complete_;
SendMessageBatchContinue(elem);
} }
void CallData::FailSendMessageBatchInCallCombiner(void* calld_arg, void CallData::FailSendMessageBatchInCallCombiner(void* calld_arg,
@ -218,11 +283,78 @@ void CallData::FailSendMessageBatchInCallCombiner(void* calld_arg,
} }
} }
void CallData::ForwardSendMessageBatch(void* elem_arg, // Pulls a slice from the send_message byte stream and adds it to slices_.
grpc_error_handle /*unused*/) { grpc_error_handle CallData::PullSliceFromSendMessage() {
grpc_slice incoming_slice;
grpc_error_handle error =
send_message_batch_->payload->send_message.send_message->Pull(
&incoming_slice);
if (error == GRPC_ERROR_NONE) {
grpc_slice_buffer_add(&slices_, incoming_slice);
}
return error;
}
// Reads as many slices as possible from the send_message byte stream.
// If all data has been read, invokes FinishSendMessage(). Otherwise,
// an async call to ByteStream::Next() has been started, which will
// eventually result in calling OnSendMessageNextDone().
void CallData::ContinueReadingSendMessage(grpc_call_element* elem) {
if (slices_.length ==
send_message_batch_->payload->send_message.send_message->length()) {
FinishSendMessage(elem);
return;
}
while (send_message_batch_->payload->send_message.send_message->Next(
~static_cast<size_t>(0), &on_send_message_next_done_)) {
grpc_error_handle error = PullSliceFromSendMessage();
if (error != GRPC_ERROR_NONE) {
// Closure callback; does not take ownership of error.
FailSendMessageBatchInCallCombiner(this, error);
GRPC_ERROR_UNREF(error);
return;
}
if (slices_.length ==
send_message_batch_->payload->send_message.send_message->length()) {
FinishSendMessage(elem);
break;
}
}
}
// Async callback for ByteStream::Next().
void CallData::OnSendMessageNextDone(void* elem_arg, grpc_error_handle error) {
grpc_call_element* elem = static_cast<grpc_call_element*>(elem_arg); grpc_call_element* elem = static_cast<grpc_call_element*>(elem_arg);
CallData* calld = static_cast<CallData*>(elem->call_data); CallData* calld = static_cast<CallData*>(elem->call_data);
calld->FinishSendMessage(elem); if (error != GRPC_ERROR_NONE) {
// Closure callback; does not take ownership of error.
FailSendMessageBatchInCallCombiner(calld, error);
return;
}
error = calld->PullSliceFromSendMessage();
if (error != GRPC_ERROR_NONE) {
// Closure callback; does not take ownership of error.
FailSendMessageBatchInCallCombiner(calld, error);
GRPC_ERROR_UNREF(error);
return;
}
if (calld->slices_.length == calld->send_message_batch_->payload->send_message
.send_message->length()) {
calld->FinishSendMessage(elem);
} else {
calld->ContinueReadingSendMessage(elem);
}
}
void CallData::StartSendMessageBatch(void* elem_arg,
grpc_error_handle /*unused*/) {
grpc_call_element* elem = static_cast<grpc_call_element*>(elem_arg);
CallData* calld = static_cast<CallData*>(elem->call_data);
if (calld->SkipMessageCompression()) {
calld->SendMessageBatchContinue(elem);
} else {
calld->ContinueReadingSendMessage(elem);
}
} }
void CallData::CompressStartTransportStreamOpBatch( void CallData::CompressStartTransportStreamOpBatch(
@ -239,6 +371,9 @@ void CallData::CompressStartTransportStreamOpBatch(
GRPC_CLOSURE_CREATE(FailSendMessageBatchInCallCombiner, this, GRPC_CLOSURE_CREATE(FailSendMessageBatchInCallCombiner, this,
grpc_schedule_on_exec_ctx), grpc_schedule_on_exec_ctx),
GRPC_ERROR_REF(cancel_error_), "failing send_message op"); GRPC_ERROR_REF(cancel_error_), "failing send_message op");
} else {
send_message_batch_->payload->send_message.send_message->Shutdown(
GRPC_ERROR_REF(cancel_error_));
} }
} }
} else if (cancel_error_ != GRPC_ERROR_NONE) { } else if (cancel_error_ != GRPC_ERROR_NONE) {
@ -259,7 +394,7 @@ void CallData::CompressStartTransportStreamOpBatch(
// the call stack) will release the call combiner for each batch it sees. // the call stack) will release the call combiner for each batch it sees.
if (send_message_batch_ != nullptr) { if (send_message_batch_ != nullptr) {
GRPC_CALL_COMBINER_START( GRPC_CALL_COMBINER_START(
call_combiner_, &forward_send_message_batch_in_call_combiner_, call_combiner_, &start_send_message_batch_in_call_combiner_,
GRPC_ERROR_NONE, "starting send_message after send_initial_metadata"); GRPC_ERROR_NONE, "starting send_message after send_initial_metadata");
} }
} }
@ -275,7 +410,7 @@ void CallData::CompressStartTransportStreamOpBatch(
call_combiner_, "send_message batch pending send_initial_metadata"); call_combiner_, "send_message batch pending send_initial_metadata");
return; return;
} }
FinishSendMessage(elem); StartSendMessageBatch(elem, GRPC_ERROR_NONE);
} else { } else {
// Pass control down the stack. // Pass control down the stack.
grpc_call_next_op(elem, batch); grpc_call_next_op(elem, batch);

@ -23,13 +23,17 @@
#include <stdint.h> #include <stdint.h>
#include <string.h> #include <string.h>
#include <memory>
#include <new> #include <new>
#include <type_traits>
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "absl/types/optional.h" #include "absl/types/optional.h"
#include <grpc/impl/codegen/compression_types.h> #include <grpc/impl/codegen/compression_types.h>
#include <grpc/slice.h>
#include <grpc/slice_buffer.h>
#include <grpc/status.h> #include <grpc/status.h>
#include <grpc/support/log.h> #include <grpc/support/log.h>
@ -37,11 +41,13 @@
#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_args.h"
#include "src/core/lib/compression/message_compress.h" #include "src/core/lib/compression/message_compress.h"
#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/call_combiner.h"
#include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/closure.h"
#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/error.h"
#include "src/core/lib/profiling/timers.h" #include "src/core/lib/profiling/timers.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_internal.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -76,6 +82,9 @@ class CallData {
OnRecvInitialMetadataReady, this, OnRecvInitialMetadataReady, this,
grpc_schedule_on_exec_ctx); grpc_schedule_on_exec_ctx);
// Initialize state for recv_message_ready callback // Initialize state for recv_message_ready callback
grpc_slice_buffer_init(&recv_slices_);
GRPC_CLOSURE_INIT(&on_recv_message_next_done_, OnRecvMessageNextDone, this,
grpc_schedule_on_exec_ctx);
GRPC_CLOSURE_INIT(&on_recv_message_ready_, OnRecvMessageReady, this, GRPC_CLOSURE_INIT(&on_recv_message_ready_, OnRecvMessageReady, this,
grpc_schedule_on_exec_ctx); grpc_schedule_on_exec_ctx);
// Initialize state for recv_trailing_metadata_ready callback // Initialize state for recv_trailing_metadata_ready callback
@ -92,6 +101,8 @@ class CallData {
} }
} }
~CallData() { grpc_slice_buffer_destroy_internal(&recv_slices_); }
void DecompressStartTransportStreamOpBatch( void DecompressStartTransportStreamOpBatch(
grpc_call_element* elem, grpc_transport_stream_op_batch* batch); grpc_call_element* elem, grpc_transport_stream_op_batch* batch);
@ -101,6 +112,10 @@ class CallData {
// Methods for processing a receive message event // Methods for processing a receive message event
void MaybeResumeOnRecvMessageReady(); void MaybeResumeOnRecvMessageReady();
static void OnRecvMessageReady(void* arg, grpc_error_handle error); static void OnRecvMessageReady(void* arg, grpc_error_handle error);
static void OnRecvMessageNextDone(void* arg, grpc_error_handle error);
grpc_error_handle PullSliceFromRecvMessage();
void ContinueReadingRecvMessage();
void FinishRecvMessage();
void ContinueRecvMessageReadyCallback(grpc_error_handle error); void ContinueRecvMessageReadyCallback(grpc_error_handle error);
// Methods for processing a recv_trailing_metadata event // Methods for processing a recv_trailing_metadata event
@ -118,10 +133,17 @@ class CallData {
bool seen_recv_message_ready_ = false; bool seen_recv_message_ready_ = false;
int max_recv_message_length_; int max_recv_message_length_;
grpc_compression_algorithm algorithm_ = GRPC_COMPRESS_NONE; grpc_compression_algorithm algorithm_ = GRPC_COMPRESS_NONE;
absl::optional<SliceBuffer>* recv_message_ = nullptr;
uint32_t* recv_message_flags_ = nullptr;
grpc_closure on_recv_message_ready_; grpc_closure on_recv_message_ready_;
grpc_closure* original_recv_message_ready_ = nullptr; grpc_closure* original_recv_message_ready_ = nullptr;
grpc_closure on_recv_message_next_done_;
OrphanablePtr<ByteStream>* recv_message_ = nullptr;
// recv_slices_ holds the slices read from the original recv_message stream.
// It is initialized during construction and reset when a new stream is
// created using it.
grpc_slice_buffer recv_slices_;
std::aligned_storage<sizeof(SliceBufferByteStream),
alignof(SliceBufferByteStream)>::type
recv_replacement_stream_;
// Fields for handling recv_trailing_metadata_ready callback // Fields for handling recv_trailing_metadata_ready callback
bool seen_recv_trailing_metadata_ready_ = false; bool seen_recv_trailing_metadata_ready_ = false;
grpc_closure on_recv_trailing_metadata_ready_; grpc_closure on_recv_trailing_metadata_ready_;
@ -165,46 +187,101 @@ void CallData::OnRecvMessageReady(void* arg, grpc_error_handle error) {
if (calld->algorithm_ != GRPC_COMPRESS_NONE) { if (calld->algorithm_ != GRPC_COMPRESS_NONE) {
// recv_message can be NULL if trailing metadata is received instead of // recv_message can be NULL if trailing metadata is received instead of
// message, or it's possible that the message was not compressed. // message, or it's possible that the message was not compressed.
if (!calld->recv_message_->has_value() || if (*calld->recv_message_ == nullptr ||
(*calld->recv_message_)->Length() == 0 || (*calld->recv_message_)->length() == 0 ||
((*calld->recv_message_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) == 0)) { ((*calld->recv_message_)->flags() & GRPC_WRITE_INTERNAL_COMPRESS) ==
0) {
return calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_NONE); return calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_NONE);
} }
if (calld->max_recv_message_length_ >= 0 && if (calld->max_recv_message_length_ >= 0 &&
(*calld->recv_message_)->Length() > (*calld->recv_message_)->length() >
static_cast<uint32_t>(calld->max_recv_message_length_)) { static_cast<uint32_t>(calld->max_recv_message_length_)) {
GPR_DEBUG_ASSERT(calld->error_ == GRPC_ERROR_NONE); GPR_DEBUG_ASSERT(calld->error_ == GRPC_ERROR_NONE);
calld->error_ = grpc_error_set_int( calld->error_ = grpc_error_set_int(
GRPC_ERROR_CREATE_FROM_CPP_STRING( GRPC_ERROR_CREATE_FROM_CPP_STRING(
absl::StrFormat("Received message larger than max (%u vs. %d)", absl::StrFormat("Received message larger than max (%u vs. %d)",
(*calld->recv_message_)->Length(), (*calld->recv_message_)->length(),
calld->max_recv_message_length_)), calld->max_recv_message_length_)),
GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED); GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED);
return calld->ContinueRecvMessageReadyCallback( return calld->ContinueRecvMessageReadyCallback(
GRPC_ERROR_REF(calld->error_)); GRPC_ERROR_REF(calld->error_));
} }
SliceBuffer decompressed_slices; grpc_slice_buffer_destroy_internal(&calld->recv_slices_);
if (grpc_msg_decompress(calld->algorithm_, grpc_slice_buffer_init(&calld->recv_slices_);
(*calld->recv_message_)->c_slice_buffer(), return calld->ContinueReadingRecvMessage();
decompressed_slices.c_slice_buffer()) == 0) {
GPR_DEBUG_ASSERT(calld->error_ == GRPC_ERROR_NONE);
calld->error_ = GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat(
"Unexpected error decompressing data for algorithm with "
"enum value ",
calld->algorithm_));
} else {
*calld->recv_message_flags_ =
(*calld->recv_message_flags_ & (~GRPC_WRITE_INTERNAL_COMPRESS)) |
GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED;
(*calld->recv_message_)->Swap(&decompressed_slices);
}
return calld->ContinueRecvMessageReadyCallback(
GRPC_ERROR_REF(calld->error_));
} }
} }
calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_REF(error)); calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_REF(error));
} }
void CallData::ContinueReadingRecvMessage() {
while ((*recv_message_)
->Next((*recv_message_)->length() - recv_slices_.length,
&on_recv_message_next_done_)) {
grpc_error_handle error = PullSliceFromRecvMessage();
if (error != GRPC_ERROR_NONE) {
return ContinueRecvMessageReadyCallback(error);
}
// We have read the entire message.
if (recv_slices_.length == (*recv_message_)->length()) {
return FinishRecvMessage();
}
}
}
grpc_error_handle CallData::PullSliceFromRecvMessage() {
grpc_slice incoming_slice;
grpc_error_handle error = (*recv_message_)->Pull(&incoming_slice);
if (error == GRPC_ERROR_NONE) {
grpc_slice_buffer_add(&recv_slices_, incoming_slice);
}
return error;
}
void CallData::OnRecvMessageNextDone(void* arg, grpc_error_handle error) {
CallData* calld = static_cast<CallData*>(arg);
if (error != GRPC_ERROR_NONE) {
return calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_REF(error));
}
error = calld->PullSliceFromRecvMessage();
if (error != GRPC_ERROR_NONE) {
return calld->ContinueRecvMessageReadyCallback(error);
}
if (calld->recv_slices_.length == (*calld->recv_message_)->length()) {
calld->FinishRecvMessage();
} else {
calld->ContinueReadingRecvMessage();
}
}
void CallData::FinishRecvMessage() {
grpc_slice_buffer decompressed_slices;
grpc_slice_buffer_init(&decompressed_slices);
if (grpc_msg_decompress(algorithm_, &recv_slices_, &decompressed_slices) ==
0) {
GPR_DEBUG_ASSERT(error_ == GRPC_ERROR_NONE);
error_ = GRPC_ERROR_CREATE_FROM_CPP_STRING(
absl::StrCat("Unexpected error decompressing data for algorithm with "
"enum value ",
algorithm_));
grpc_slice_buffer_destroy_internal(&decompressed_slices);
} else {
uint32_t recv_flags =
((*recv_message_)->flags() & (~GRPC_WRITE_INTERNAL_COMPRESS)) |
GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED;
// Swap out the original receive byte stream with our new one and send the
// batch down.
// Initializing recv_replacement_stream_ with decompressed_slices removes
// all the slices from decompressed_slices leaving it empty.
new (&recv_replacement_stream_)
SliceBufferByteStream(&decompressed_slices, recv_flags);
recv_message_->reset(
reinterpret_cast<SliceBufferByteStream*>(&recv_replacement_stream_));
recv_message_ = nullptr;
}
ContinueRecvMessageReadyCallback(GRPC_ERROR_REF(error_));
}
void CallData::ContinueRecvMessageReadyCallback(grpc_error_handle error) { void CallData::ContinueRecvMessageReadyCallback(grpc_error_handle error) {
MaybeResumeOnRecvTrailingMetadataReady(); MaybeResumeOnRecvTrailingMetadataReady();
// The surface will clean up the receiving stream if there is an error. // The surface will clean up the receiving stream if there is an error.
@ -256,7 +333,6 @@ void CallData::DecompressStartTransportStreamOpBatch(
// Handle recv_message // Handle recv_message
if (batch->recv_message) { if (batch->recv_message) {
recv_message_ = batch->payload->recv_message.recv_message; recv_message_ = batch->payload->recv_message.recv_message;
recv_message_flags_ = batch->payload->recv_message.flags;
original_recv_message_ready_ = original_recv_message_ready_ =
batch->payload->recv_message.recv_message_ready; batch->payload->recv_message.recv_message_ready;
batch->payload->recv_message.recv_message_ready = &on_recv_message_ready_; batch->payload->recv_message.recv_message_ready = &on_recv_message_ready_;

@ -39,12 +39,13 @@
#include "src/core/lib/config/core_configuration.h" #include "src/core/lib/config/core_configuration.h"
#include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/string.h"
#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/call_combiner.h"
#include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/closure.h"
#include "src/core/lib/service_config/service_config_call_data.h" #include "src/core/lib/service_config/service_config_call_data.h"
#include "src/core/lib/slice/slice_buffer.h"
#include "src/core/lib/surface/channel_init.h" #include "src/core/lib/surface/channel_init.h"
#include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/surface/channel_stack_type.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
static void recv_message_ready(void* user_data, grpc_error_handle error); static void recv_message_ready(void* user_data, grpc_error_handle error);
@ -193,7 +194,7 @@ struct call_data {
// The error caused by a message that is too large, or GRPC_ERROR_NONE // The error caused by a message that is too large, or GRPC_ERROR_NONE
grpc_error_handle error = GRPC_ERROR_NONE; grpc_error_handle error = GRPC_ERROR_NONE;
// Used by recv_message_ready. // Used by recv_message_ready.
absl::optional<grpc_core::SliceBuffer>* recv_message = nullptr; grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message = nullptr;
// Original recv_message_ready callback, invoked after our own. // Original recv_message_ready callback, invoked after our own.
grpc_closure* next_recv_message_ready = nullptr; grpc_closure* next_recv_message_ready = nullptr;
// Original recv_trailing_metadata callback, invoked after our own. // Original recv_trailing_metadata callback, invoked after our own.
@ -209,13 +210,13 @@ struct call_data {
static void recv_message_ready(void* user_data, grpc_error_handle error) { static void recv_message_ready(void* user_data, grpc_error_handle error) {
grpc_call_element* elem = static_cast<grpc_call_element*>(user_data); grpc_call_element* elem = static_cast<grpc_call_element*>(user_data);
call_data* calld = static_cast<call_data*>(elem->call_data); call_data* calld = static_cast<call_data*>(elem->call_data);
if (calld->recv_message->has_value() && calld->limits.max_recv_size >= 0 && if (*calld->recv_message != nullptr && calld->limits.max_recv_size >= 0 &&
(*calld->recv_message)->Length() > (*calld->recv_message)->length() >
static_cast<size_t>(calld->limits.max_recv_size)) { static_cast<size_t>(calld->limits.max_recv_size)) {
grpc_error_handle new_error = grpc_error_set_int( grpc_error_handle new_error = grpc_error_set_int(
GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat(
"Received message larger than max (%u vs. %d)", "Received message larger than max (%u vs. %d)",
(*calld->recv_message)->Length(), calld->limits.max_recv_size)), (*calld->recv_message)->length(), calld->limits.max_recv_size)),
GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED); GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED);
error = grpc_error_add_child(GRPC_ERROR_REF(error), new_error); error = grpc_error_add_child(GRPC_ERROR_REF(error), new_error);
GRPC_ERROR_UNREF(calld->error); GRPC_ERROR_UNREF(calld->error);
@ -268,13 +269,13 @@ static void message_size_start_transport_stream_op_batch(
call_data* calld = static_cast<call_data*>(elem->call_data); call_data* calld = static_cast<call_data*>(elem->call_data);
// Check max send message size. // Check max send message size.
if (op->send_message && calld->limits.max_send_size >= 0 && if (op->send_message && calld->limits.max_send_size >= 0 &&
op->payload->send_message.send_message->Length() > op->payload->send_message.send_message->length() >
static_cast<size_t>(calld->limits.max_send_size)) { static_cast<size_t>(calld->limits.max_send_size)) {
grpc_transport_stream_op_batch_finish_with_failure( grpc_transport_stream_op_batch_finish_with_failure(
op, op,
grpc_error_set_int(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat( grpc_error_set_int(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat(
"Sent message larger than max (%u vs. %d)", "Sent message larger than max (%u vs. %d)",
op->payload->send_message.send_message->Length(), op->payload->send_message.send_message->length(),
calld->limits.max_send_size)), calld->limits.max_send_size)),
GRPC_ERROR_INT_GRPC_STATUS, GRPC_ERROR_INT_GRPC_STATUS,
GRPC_STATUS_RESOURCE_EXHAUSTED), GRPC_STATUS_RESOURCE_EXHAUSTED),

@ -18,6 +18,7 @@
#include <functional> #include <functional>
#include <memory> #include <memory>
#include <type_traits>
#include <utility> #include <utility>
#include "absl/base/thread_annotations.h" #include "absl/base/thread_annotations.h"

@ -80,6 +80,7 @@ struct grpc_binder_stream {
grpc_binder_transport* t; grpc_binder_transport* t;
grpc_stream_refcount* refcount; grpc_stream_refcount* refcount;
grpc_core::Arena* arena; grpc_core::Arena* arena;
grpc_core::ManualConstructor<grpc_core::SliceBufferByteStream> sbs;
int tx_code; int tx_code;
const bool is_client; const bool is_client;
bool is_closed; bool is_closed;
@ -105,7 +106,7 @@ struct grpc_binder_stream {
grpc_metadata_batch* recv_initial_metadata; grpc_metadata_batch* recv_initial_metadata;
grpc_closure* recv_initial_metadata_ready = nullptr; grpc_closure* recv_initial_metadata_ready = nullptr;
bool* trailing_metadata_available = nullptr; bool* trailing_metadata_available = nullptr;
absl::optional<grpc_core::SliceBuffer>* recv_message; grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message;
grpc_closure* recv_message_ready = nullptr; grpc_closure* recv_message_ready = nullptr;
bool* call_failed_before_recv_message = nullptr; bool* call_failed_before_recv_message = nullptr;
grpc_metadata_batch* recv_trailing_metadata; grpc_metadata_batch* recv_trailing_metadata;

@ -37,6 +37,7 @@
#include "src/core/ext/transport/binder/wire_format/wire_writer.h" #include "src/core/ext/transport/binder/wire_format/wire_writer.h"
#include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/exec_ctx.h"
#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_internal.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/error_utils.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -256,9 +257,12 @@ static void recv_message_locked(void* arg, grpc_error_handle /*error*/) {
return absl_status_to_grpc_error(args->message.status()); return absl_status_to_grpc_error(args->message.status());
} }
} }
grpc_core::SliceBuffer buf; grpc_slice_buffer buf;
buf.Append(grpc_core::Slice(grpc_slice_from_cpp_string(*args->message))); grpc_slice_buffer_init(&buf);
*gbs->recv_message = std::move(buf); grpc_slice_buffer_add(&buf, grpc_slice_from_cpp_string(*args->message));
gbs->sbs.Init(&buf, 0);
gbs->recv_message->reset(gbs->sbs.get());
return GRPC_ERROR_NONE; return GRPC_ERROR_NONE;
}(); }();
@ -407,7 +411,7 @@ static void perform_stream_op_locked(void* stream_op,
if (gbs->is_closed) { if (gbs->is_closed) {
if (op->send_message) { if (op->send_message) {
// Reset the send_message payload to prevent memory leaks. // Reset the send_message payload to prevent memory leaks.
op->payload->send_message.send_message->Clear(); op->payload->send_message.send_message.reset();
} }
if (op->recv_initial_metadata) { if (op->recv_initial_metadata) {
grpc_core::ExecCtx::Run( grpc_core::ExecCtx::Run(
@ -448,7 +452,27 @@ static void perform_stream_op_locked(void* stream_op,
} }
if (op->send_message) { if (op->send_message) {
gpr_log(GPR_INFO, "send_message"); gpr_log(GPR_INFO, "send_message");
tx.SetData(op->payload->send_message.send_message->JoinIntoString()); size_t remaining = op->payload->send_message.send_message->length();
std::string message_data;
while (remaining > 0) {
grpc_slice message_slice;
// TODO(waynetu): Temporarily assume that the message is ready.
GPR_ASSERT(
op->payload->send_message.send_message->Next(SIZE_MAX, nullptr));
grpc_error_handle error =
op->payload->send_message.send_message->Pull(&message_slice);
// TODO(waynetu): Cancel the stream if error is not GRPC_ERROR_NONE.
GPR_ASSERT(error == GRPC_ERROR_NONE);
uint8_t* p = GRPC_SLICE_START_PTR(message_slice);
size_t len = GRPC_SLICE_LENGTH(message_slice);
remaining -= len;
message_data += std::string(reinterpret_cast<char*>(p), len);
grpc_slice_unref_internal(message_slice);
}
tx.SetData(message_data);
// TODO(b/192369787): Are we supposed to reset here to avoid
// use-after-free issue in call.cc?
op->payload->send_message.send_message.reset();
} }
if (op->send_trailing_metadata) { if (op->send_trailing_metadata) {

@ -37,7 +37,6 @@
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "absl/types/optional.h" #include "absl/types/optional.h"
#include "absl/types/variant.h"
#include <grpc/impl/codegen/connectivity_state.h> #include <grpc/impl/codegen/connectivity_state.h>
#include <grpc/slice_buffer.h> #include <grpc/slice_buffer.h>
@ -62,6 +61,9 @@
#include "src/core/lib/gpr/useful.h" #include "src/core/lib/gpr/useful.h"
#include "src/core/lib/gprpp/bitset.h" #include "src/core/lib/gprpp/bitset.h"
#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/global_config_env.h"
#include "src/core/lib/gprpp/manual_constructor.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted.h"
#include "src/core/lib/gprpp/time.h" #include "src/core/lib/gprpp/time.h"
#include "src/core/lib/http/parser.h" #include "src/core/lib/http/parser.h"
@ -72,17 +74,16 @@
#include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset.h"
#include "src/core/lib/iomgr/timer.h" #include "src/core/lib/iomgr/timer.h"
#include "src/core/lib/profiling/timers.h" #include "src/core/lib/profiling/timers.h"
#include "src/core/lib/promise/poll.h"
#include "src/core/lib/resource_quota/api.h" #include "src/core/lib/resource_quota/api.h"
#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/arena.h"
#include "src/core/lib/resource_quota/memory_quota.h" #include "src/core/lib/resource_quota/memory_quota.h"
#include "src/core/lib/resource_quota/resource_quota.h" #include "src/core/lib/resource_quota/resource_quota.h"
#include "src/core/lib/resource_quota/trace.h" #include "src/core/lib/resource_quota/trace.h"
#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice.h"
#include "src/core/lib/slice/slice_buffer.h"
#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_internal.h"
#include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/slice/slice_refcount.h"
#include "src/core/lib/transport/bdp_estimator.h" #include "src/core/lib/transport/bdp_estimator.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/connectivity_state.h"
#include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/error_utils.h"
#include "src/core/lib/transport/http2_errors.h" #include "src/core/lib/transport/http2_errors.h"
@ -91,6 +92,12 @@
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
#include "src/core/lib/transport/transport_impl.h" #include "src/core/lib/transport/transport_impl.h"
GPR_GLOBAL_CONFIG_DEFINE_BOOL(
grpc_experimental_disable_flow_control, false,
"If set, flow control will be effectively disabled. Max out all values and "
"assume the remote peer does the same. Thus we can ignore any flow control "
"bookkeeping, error checking, and decision making");
#define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024) #define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024)
#define MAX_WINDOW 0x7fffffffu #define MAX_WINDOW 0x7fffffffu
#define MAX_WRITE_BUFFER_SIZE (64 * 1024 * 1024) #define MAX_WRITE_BUFFER_SIZE (64 * 1024 * 1024)
@ -143,6 +150,8 @@ static void read_action(void* t, grpc_error_handle error);
static void read_action_locked(void* t, grpc_error_handle error); static void read_action_locked(void* t, grpc_error_handle error);
static void continue_read_action_locked(grpc_chttp2_transport* t); static void continue_read_action_locked(grpc_chttp2_transport* t);
static void complete_fetch(void* gs, grpc_error_handle error);
static void complete_fetch_locked(void* gs, grpc_error_handle error);
// Set a transport level setting, and push it to our peer // Set a transport level setting, and push it to our peer
static void queue_setting_update(grpc_chttp2_transport* t, static void queue_setting_update(grpc_chttp2_transport* t,
grpc_chttp2_setting_id id, uint32_t value); grpc_chttp2_setting_id id, uint32_t value);
@ -192,6 +201,8 @@ static void finish_keepalive_ping_locked(void* arg, grpc_error_handle error);
static void keepalive_watchdog_fired(void* arg, grpc_error_handle error); static void keepalive_watchdog_fired(void* arg, grpc_error_handle error);
static void keepalive_watchdog_fired_locked(void* arg, grpc_error_handle error); static void keepalive_watchdog_fired_locked(void* arg, grpc_error_handle error);
static void reset_byte_stream(void* arg, grpc_error_handle error);
namespace grpc_core { namespace grpc_core {
namespace { namespace {
@ -268,6 +279,8 @@ grpc_chttp2_transport::~grpc_chttp2_transport() {
write_cb_pool = next; write_cb_pool = next;
} }
flow_control.Destroy();
GRPC_ERROR_UNREF(closed_with_error); GRPC_ERROR_UNREF(closed_with_error);
gpr_free(ping_acks); gpr_free(ping_acks);
if (grpc_core::test_only_destruct_callback != nullptr) { if (grpc_core::test_only_destruct_callback != nullptr) {
@ -277,9 +290,11 @@ grpc_chttp2_transport::~grpc_chttp2_transport() {
static const grpc_transport_vtable* get_vtable(void); static const grpc_transport_vtable* get_vtable(void);
static void read_channel_args(grpc_chttp2_transport* t, // Returns whether bdp is enabled
static bool read_channel_args(grpc_chttp2_transport* t,
const grpc_channel_args* channel_args, const grpc_channel_args* channel_args,
bool is_client) { bool is_client) {
bool enable_bdp = true;
bool channelz_enabled = GRPC_ENABLE_CHANNELZ_DEFAULT; bool channelz_enabled = GRPC_ENABLE_CHANNELZ_DEFAULT;
size_t i; size_t i;
int j; int j;
@ -329,6 +344,9 @@ static void read_channel_args(grpc_chttp2_transport* t,
GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE)) { GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE)) {
t->write_buffer_size = static_cast<uint32_t>(grpc_channel_arg_get_integer( t->write_buffer_size = static_cast<uint32_t>(grpc_channel_arg_get_integer(
&channel_args->args[i], {0, 0, MAX_WRITE_BUFFER_SIZE})); &channel_args->args[i], {0, 0, MAX_WRITE_BUFFER_SIZE}));
} else if (0 ==
strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_BDP_PROBE)) {
enable_bdp = grpc_channel_arg_get_bool(&channel_args->args[i], true);
} else if (0 == } else if (0 ==
strcmp(channel_args->args[i].key, GRPC_ARG_KEEPALIVE_TIME_MS)) { strcmp(channel_args->args[i].key, GRPC_ARG_KEEPALIVE_TIME_MS)) {
const int value = grpc_channel_arg_get_integer( const int value = grpc_channel_arg_get_integer(
@ -420,6 +438,7 @@ static void read_channel_args(grpc_chttp2_transport* t,
grpc_core::channelz::SocketNode::Security::GetFromChannelArgs( grpc_core::channelz::SocketNode::Security::GetFromChannelArgs(
channel_args)); channel_args));
} }
return enable_bdp;
} }
static void init_transport_keepalive_settings(grpc_chttp2_transport* t) { static void init_transport_keepalive_settings(grpc_chttp2_transport* t) {
@ -490,10 +509,6 @@ grpc_chttp2_transport::grpc_chttp2_transport(
GRPC_CHANNEL_READY), GRPC_CHANNEL_READY),
is_client(is_client), is_client(is_client),
next_stream_id(is_client ? 1 : 2), next_stream_id(is_client ? 1 : 2),
flow_control(peer_string.c_str(),
grpc_channel_args_find_bool(channel_args,
GRPC_ARG_HTTP2_BDP_PROBE, true),
&memory_owner),
deframe_state(is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0) { deframe_state(is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0) {
GPR_ASSERT(strlen(GRPC_CHTTP2_CLIENT_CONNECT_STRING) == GPR_ASSERT(strlen(GRPC_CHTTP2_CLIENT_CONNECT_STRING) ==
GRPC_CHTTP2_CLIENT_CONNECT_STRLEN); GRPC_CHTTP2_CLIENT_CONNECT_STRLEN);
@ -535,8 +550,19 @@ grpc_chttp2_transport::grpc_chttp2_transport(
configure_transport_ping_policy(this); configure_transport_ping_policy(this);
init_transport_keepalive_settings(this); init_transport_keepalive_settings(this);
if (channel_args != nullptr) { bool enable_bdp = true;
read_channel_args(this, channel_args, is_client); if (channel_args) {
enable_bdp = read_channel_args(this, channel_args, is_client);
}
static const bool kEnableFlowControl =
!GPR_GLOBAL_CONFIG_GET(grpc_experimental_disable_flow_control);
if (kEnableFlowControl) {
flow_control.Init<grpc_core::chttp2::TransportFlowControl>(this,
enable_bdp);
} else {
flow_control.Init<grpc_core::chttp2::TransportFlowControlDisabled>(this);
enable_bdp = false;
} }
// No pings allowed before receiving a header or data frame. // No pings allowed before receiving a header or data frame.
@ -549,9 +575,9 @@ grpc_chttp2_transport::grpc_chttp2_transport(
init_keepalive_pings_if_enabled(this); init_keepalive_pings_if_enabled(this);
if (flow_control.bdp_probe()) { if (enable_bdp) {
bdp_ping_blocked = true; bdp_ping_blocked = true;
grpc_chttp2_act_on_flowctl_action(flow_control.PeriodicUpdate(), this, grpc_chttp2_act_on_flowctl_action(flow_control->PeriodicUpdate(), this,
nullptr); nullptr);
} }
@ -676,17 +702,26 @@ grpc_chttp2_stream::grpc_chttp2_stream(grpc_chttp2_transport* t,
refcount(refcount), refcount(refcount),
reffer(this), reffer(this),
initial_metadata_buffer(arena), initial_metadata_buffer(arena),
trailing_metadata_buffer(arena), trailing_metadata_buffer(arena) {
flow_control(&t->flow_control) {
if (server_data) { if (server_data) {
id = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(server_data)); id = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(server_data));
*t->accepting_stream = this; *t->accepting_stream = this;
grpc_chttp2_stream_map_add(&t->stream_map, id, this); grpc_chttp2_stream_map_add(&t->stream_map, id, this);
post_destructive_reclaimer(t); post_destructive_reclaimer(t);
} }
if (t->flow_control->flow_control_enabled()) {
flow_control.Init<grpc_core::chttp2::StreamFlowControl>(
static_cast<grpc_core::chttp2::TransportFlowControl*>(
t->flow_control.get()),
this);
} else {
flow_control.Init<grpc_core::chttp2::StreamFlowControlDisabled>();
}
grpc_slice_buffer_init(&frame_storage); grpc_slice_buffer_init(&frame_storage);
grpc_slice_buffer_init(&unprocessed_incoming_frames_buffer);
grpc_slice_buffer_init(&flow_controlled_buffer); grpc_slice_buffer_init(&flow_controlled_buffer);
GRPC_CLOSURE_INIT(&reset_byte_stream, ::reset_byte_stream, this, nullptr);
} }
grpc_chttp2_stream::~grpc_chttp2_stream() { grpc_chttp2_stream::~grpc_chttp2_stream() {
@ -703,6 +738,7 @@ grpc_chttp2_stream::~grpc_chttp2_stream() {
GPR_ASSERT(grpc_chttp2_stream_map_find(&t->stream_map, id) == nullptr); GPR_ASSERT(grpc_chttp2_stream_map_find(&t->stream_map, id) == nullptr);
} }
grpc_slice_buffer_destroy_internal(&unprocessed_incoming_frames_buffer);
grpc_slice_buffer_destroy_internal(&frame_storage); grpc_slice_buffer_destroy_internal(&frame_storage);
for (int i = 0; i < STREAM_LIST_COUNT; i++) { for (int i = 0; i < STREAM_LIST_COUNT; i++) {
@ -714,6 +750,7 @@ grpc_chttp2_stream::~grpc_chttp2_stream() {
} }
GPR_ASSERT(send_initial_metadata_finished == nullptr); GPR_ASSERT(send_initial_metadata_finished == nullptr);
GPR_ASSERT(fetching_send_message == nullptr);
GPR_ASSERT(send_trailing_metadata_finished == nullptr); GPR_ASSERT(send_trailing_metadata_finished == nullptr);
GPR_ASSERT(recv_initial_metadata_ready == nullptr); GPR_ASSERT(recv_initial_metadata_ready == nullptr);
GPR_ASSERT(recv_message_ready == nullptr); GPR_ASSERT(recv_message_ready == nullptr);
@ -721,6 +758,8 @@ grpc_chttp2_stream::~grpc_chttp2_stream() {
grpc_slice_buffer_destroy_internal(&flow_controlled_buffer); grpc_slice_buffer_destroy_internal(&flow_controlled_buffer);
GRPC_ERROR_UNREF(read_closed_error); GRPC_ERROR_UNREF(read_closed_error);
GRPC_ERROR_UNREF(write_closed_error); GRPC_ERROR_UNREF(write_closed_error);
GRPC_ERROR_UNREF(byte_stream_error);
flow_control.Destroy();
GRPC_CHTTP2_UNREF_TRANSPORT(t, "stream"); GRPC_CHTTP2_UNREF_TRANSPORT(t, "stream");
grpc_core::ExecCtx::Run(DEBUG_LOCATION, destroy_stream_arg, GRPC_ERROR_NONE); grpc_core::ExecCtx::Run(DEBUG_LOCATION, destroy_stream_arg, GRPC_ERROR_NONE);
} }
@ -1286,6 +1325,94 @@ static bool contains_non_ok_status(grpc_metadata_batch* batch) {
GRPC_STATUS_OK; GRPC_STATUS_OK;
} }
static void maybe_become_writable_due_to_send_msg(grpc_chttp2_transport* t,
grpc_chttp2_stream* s) {
if (s->id != 0 && (!s->write_buffering ||
s->flow_controlled_buffer.length > t->write_buffer_size)) {
grpc_chttp2_mark_stream_writable(t, s);
grpc_chttp2_initiate_write(t, GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE);
}
}
static void add_fetched_slice_locked(grpc_chttp2_transport* t,
grpc_chttp2_stream* s) {
s->fetched_send_message_length +=
static_cast<uint32_t> GRPC_SLICE_LENGTH(s->fetching_slice);
grpc_slice_buffer_add(&s->flow_controlled_buffer, s->fetching_slice);
maybe_become_writable_due_to_send_msg(t, s);
}
static void continue_fetching_send_locked(grpc_chttp2_transport* t,
grpc_chttp2_stream* s) {
for (;;) {
if (s->fetching_send_message == nullptr) {
// Stream was cancelled before message fetch completed
abort(); /* TODO(ctiller): what cleanup here? */
}
if (s->fetched_send_message_length == s->fetching_send_message->length()) {
int64_t notify_offset = s->next_message_end_offset;
if (notify_offset <= s->flow_controlled_bytes_written) {
grpc_chttp2_complete_closure_step(
t, s, &s->fetching_send_message_finished, GRPC_ERROR_NONE,
"fetching_send_message_finished");
} else {
grpc_chttp2_write_cb* cb = t->write_cb_pool;
if (cb == nullptr) {
cb = static_cast<grpc_chttp2_write_cb*>(gpr_malloc(sizeof(*cb)));
} else {
t->write_cb_pool = cb->next;
}
cb->call_at_byte = notify_offset;
cb->closure = s->fetching_send_message_finished;
s->fetching_send_message_finished = nullptr;
grpc_chttp2_write_cb** list =
s->fetching_send_message->flags() & GRPC_WRITE_THROUGH
? &s->on_write_finished_cbs
: &s->on_flow_controlled_cbs;
cb->next = *list;
*list = cb;
}
s->fetching_send_message.reset();
return; /* early out */
} else if (s->fetching_send_message->Next(
UINT32_MAX, GRPC_CLOSURE_INIT(&s->complete_fetch_locked,
::complete_fetch, s,
grpc_schedule_on_exec_ctx))) {
grpc_error_handle error =
s->fetching_send_message->Pull(&s->fetching_slice);
if (error != GRPC_ERROR_NONE) {
s->fetching_send_message.reset();
grpc_chttp2_cancel_stream(t, s, error);
} else {
add_fetched_slice_locked(t, s);
}
}
}
}
static void complete_fetch(void* gs, grpc_error_handle error) {
grpc_chttp2_stream* s = static_cast<grpc_chttp2_stream*>(gs);
s->t->combiner->Run(GRPC_CLOSURE_INIT(&s->complete_fetch_locked,
::complete_fetch_locked, s, nullptr),
GRPC_ERROR_REF(error));
}
static void complete_fetch_locked(void* gs, grpc_error_handle error) {
grpc_chttp2_stream* s = static_cast<grpc_chttp2_stream*>(gs);
grpc_chttp2_transport* t = s->t;
if (error == GRPC_ERROR_NONE) {
error = s->fetching_send_message->Pull(&s->fetching_slice);
if (error == GRPC_ERROR_NONE) {
add_fetched_slice_locked(t, s);
continue_fetching_send_locked(t, s);
}
}
if (error != GRPC_ERROR_NONE) {
s->fetching_send_message.reset();
grpc_chttp2_cancel_stream(t, s, error);
}
}
static void log_metadata(const grpc_metadata_batch* md_batch, uint32_t id, static void log_metadata(const grpc_metadata_batch* md_batch, uint32_t id,
bool is_client, bool is_initial) { bool is_client, bool is_initial) {
const std::string prefix = absl::StrCat( const std::string prefix = absl::StrCat(
@ -1380,7 +1507,8 @@ static void perform_stream_op_locked(void* stream_op,
GPR_ASSERT(s->id != 0); GPR_ASSERT(s->id != 0);
grpc_chttp2_mark_stream_writable(t, s); grpc_chttp2_mark_stream_writable(t, s);
if (!(op->send_message && if (!(op->send_message &&
(op->payload->send_message.flags & GRPC_WRITE_BUFFER_HINT))) { (op->payload->send_message.send_message->flags() &
GRPC_WRITE_BUFFER_HINT))) {
grpc_chttp2_initiate_write( grpc_chttp2_initiate_write(
t, GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA); t, GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA);
} }
@ -1404,28 +1532,32 @@ static void perform_stream_op_locked(void* stream_op,
GRPC_STATS_INC_HTTP2_OP_SEND_MESSAGE(); GRPC_STATS_INC_HTTP2_OP_SEND_MESSAGE();
t->num_messages_in_next_write++; t->num_messages_in_next_write++;
GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE( GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE(
op->payload->send_message.send_message->Length()); op->payload->send_message.send_message->length());
on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE; on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
s->send_message_finished = add_closure_barrier(op->on_complete); s->fetching_send_message_finished = add_closure_barrier(op->on_complete);
const uint32_t flags = op_payload->send_message.flags;
if (s->write_closed) { if (s->write_closed) {
op->payload->send_message.stream_write_closed = true; op->payload->send_message.stream_write_closed = true;
// We should NOT return an error here, so as to avoid a cancel OP being // We should NOT return an error here, so as to avoid a cancel OP being
// started. The surface layer will notice that the stream has been closed // started. The surface layer will notice that the stream has been closed
// for writes and fail the send message op. // for writes and fail the send message op.
grpc_chttp2_complete_closure_step(t, s, &s->send_message_finished, op->payload->send_message.send_message.reset();
GRPC_ERROR_NONE, grpc_chttp2_complete_closure_step(
"fetching_send_message_finished"); t, s, &s->fetching_send_message_finished, GRPC_ERROR_NONE,
"fetching_send_message_finished");
} else { } else {
GPR_ASSERT(s->fetching_send_message == nullptr);
uint8_t* frame_hdr = grpc_slice_buffer_tiny_add( uint8_t* frame_hdr = grpc_slice_buffer_tiny_add(
&s->flow_controlled_buffer, GRPC_HEADER_SIZE_IN_BYTES); &s->flow_controlled_buffer, GRPC_HEADER_SIZE_IN_BYTES);
uint32_t flags = op_payload->send_message.send_message->flags();
frame_hdr[0] = (flags & GRPC_WRITE_INTERNAL_COMPRESS) != 0; frame_hdr[0] = (flags & GRPC_WRITE_INTERNAL_COMPRESS) != 0;
size_t len = op_payload->send_message.send_message->Length(); size_t len = op_payload->send_message.send_message->length();
frame_hdr[1] = static_cast<uint8_t>(len >> 24); frame_hdr[1] = static_cast<uint8_t>(len >> 24);
frame_hdr[2] = static_cast<uint8_t>(len >> 16); frame_hdr[2] = static_cast<uint8_t>(len >> 16);
frame_hdr[3] = static_cast<uint8_t>(len >> 8); frame_hdr[3] = static_cast<uint8_t>(len >> 8);
frame_hdr[4] = static_cast<uint8_t>(len); frame_hdr[4] = static_cast<uint8_t>(len);
s->fetching_send_message =
std::move(op_payload->send_message.send_message);
s->fetched_send_message_length = 0;
s->next_message_end_offset = s->next_message_end_offset =
s->flow_controlled_bytes_written + s->flow_controlled_bytes_written +
static_cast<int64_t>(s->flow_controlled_buffer.length) + static_cast<int64_t>(s->flow_controlled_buffer.length) +
@ -1436,44 +1568,8 @@ static void perform_stream_op_locked(void* stream_op,
} else { } else {
s->write_buffering = false; s->write_buffering = false;
} }
continue_fetching_send_locked(t, s);
grpc_slice* const slices = maybe_become_writable_due_to_send_msg(t, s);
op_payload->send_message.send_message->c_slice_buffer()->slices;
grpc_slice* const end =
slices + op_payload->send_message.send_message->Count();
for (grpc_slice* slice = slices; slice != end; slice++) {
grpc_slice_buffer_add(&s->flow_controlled_buffer,
grpc_slice_ref_internal(*slice));
}
int64_t notify_offset = s->next_message_end_offset;
if (notify_offset <= s->flow_controlled_bytes_written) {
grpc_chttp2_complete_closure_step(t, s, &s->send_message_finished,
GRPC_ERROR_NONE,
"fetching_send_message_finished");
} else {
grpc_chttp2_write_cb* cb = t->write_cb_pool;
if (cb == nullptr) {
cb = static_cast<grpc_chttp2_write_cb*>(gpr_malloc(sizeof(*cb)));
} else {
t->write_cb_pool = cb->next;
}
cb->call_at_byte = notify_offset;
cb->closure = s->send_message_finished;
s->send_message_finished = nullptr;
grpc_chttp2_write_cb** list = flags & GRPC_WRITE_THROUGH
? &s->on_write_finished_cbs
: &s->on_flow_controlled_cbs;
cb->next = *list;
*list = cb;
}
if (s->id != 0 &&
(!s->write_buffering ||
s->flow_controlled_buffer.length > t->write_buffer_size)) {
grpc_chttp2_mark_stream_writable(t, s);
grpc_chttp2_initiate_write(t, GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE);
}
} }
} }
@ -1527,14 +1623,28 @@ static void perform_stream_op_locked(void* stream_op,
if (op->recv_message) { if (op->recv_message) {
GRPC_STATS_INC_HTTP2_OP_RECV_MESSAGE(); GRPC_STATS_INC_HTTP2_OP_RECV_MESSAGE();
size_t before = 0;
GPR_ASSERT(s->recv_message_ready == nullptr); GPR_ASSERT(s->recv_message_ready == nullptr);
GPR_ASSERT(!s->pending_byte_stream);
s->recv_message_ready = op_payload->recv_message.recv_message_ready; s->recv_message_ready = op_payload->recv_message.recv_message_ready;
s->recv_message = op_payload->recv_message.recv_message; s->recv_message = op_payload->recv_message.recv_message;
s->recv_message->emplace();
s->recv_message_flags = op_payload->recv_message.flags;
s->call_failed_before_recv_message = s->call_failed_before_recv_message =
op_payload->recv_message.call_failed_before_recv_message; op_payload->recv_message.call_failed_before_recv_message;
grpc_chttp2_maybe_complete_recv_trailing_metadata(t, s); if (s->id != 0) {
if (!s->read_closed) {
before = s->frame_storage.length +
s->unprocessed_incoming_frames_buffer.length;
}
}
grpc_chttp2_maybe_complete_recv_message(t, s);
if (s->id != 0) {
if (!s->read_closed && s->frame_storage.length == 0) {
size_t after = s->unprocessed_incoming_frames_buffer_cached_length;
s->flow_control->IncomingByteStreamUpdate(GRPC_HEADER_SIZE_IN_BYTES,
before - after);
grpc_chttp2_act_on_flowctl_action(s->flow_control->MakeAction(), t, s);
}
}
} }
if (op->recv_trailing_metadata) { if (op->recv_trailing_metadata) {
@ -1908,6 +2018,10 @@ void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_chttp2_transport* t,
s->published_metadata[0] != GRPC_METADATA_NOT_PUBLISHED) { s->published_metadata[0] != GRPC_METADATA_NOT_PUBLISHED) {
if (s->seen_error) { if (s->seen_error) {
grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage);
if (!s->pending_byte_stream) {
grpc_slice_buffer_reset_and_unref_internal(
&s->unprocessed_incoming_frames_buffer);
}
} }
*s->recv_initial_metadata = std::move(s->initial_metadata_buffer); *s->recv_initial_metadata = std::move(s->initial_metadata_buffer);
s->recv_initial_metadata->Set(grpc_core::PeerString(), t->peer_string); s->recv_initial_metadata->Set(grpc_core::PeerString(), t->peer_string);
@ -1924,58 +2038,47 @@ void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_chttp2_transport* t,
} }
} }
void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t, void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* /*t*/,
grpc_chttp2_stream* s) { grpc_chttp2_stream* s) {
grpc_error_handle error = GRPC_ERROR_NONE; grpc_error_handle error = GRPC_ERROR_NONE;
if (s->recv_message_ready != nullptr) { if (s->recv_message_ready != nullptr) {
*s->recv_message = nullptr;
if (s->final_metadata_requested && s->seen_error) { if (s->final_metadata_requested && s->seen_error) {
grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage);
s->recv_message->reset(); if (!s->pending_byte_stream) {
} else { grpc_slice_buffer_reset_and_unref_internal(
if (s->frame_storage.length != 0) { &s->unprocessed_incoming_frames_buffer);
while (true) { }
GPR_ASSERT(s->frame_storage.length > 0); }
uint32_t min_progress_size; if (!s->pending_byte_stream) {
auto r = grpc_deframe_unprocessed_incoming_frames( while (s->unprocessed_incoming_frames_buffer.length > 0 ||
s, &min_progress_size, &**s->recv_message, s->recv_message_flags); s->frame_storage.length > 0) {
if (absl::holds_alternative<grpc_core::Pending>(r)) { if (s->unprocessed_incoming_frames_buffer.length == 0) {
if (s->read_closed) { grpc_slice_buffer_swap(&s->unprocessed_incoming_frames_buffer,
grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); &s->frame_storage);
s->recv_message->reset(); }
break; error = grpc_deframe_unprocessed_incoming_frames(
} else { &s->data_parser, s, &s->unprocessed_incoming_frames_buffer, nullptr,
s->flow_control.UpdateProgress(min_progress_size); s->recv_message);
grpc_chttp2_act_on_flowctl_action(s->flow_control.MakeAction(), t, if (error != GRPC_ERROR_NONE) {
s); s->seen_error = true;
return; grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage);
} grpc_slice_buffer_reset_and_unref_internal(
} else { &s->unprocessed_incoming_frames_buffer);
error = absl::get<grpc_error_handle>(r); break;
if (error != GRPC_ERROR_NONE) { } else if (*s->recv_message != nullptr) {
s->seen_error = true; break;
grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage);
break;
} else {
if (t->channelz_socket != nullptr) {
t->channelz_socket->RecordMessageReceived();
}
break;
}
}
} }
} else if (s->read_closed) {
s->recv_message->reset();
} else {
s->flow_control.UpdateProgress(GRPC_HEADER_SIZE_IN_BYTES);
grpc_chttp2_act_on_flowctl_action(s->flow_control.MakeAction(), t, s);
return;
} }
} }
// save the length of the buffer before handing control back to application // save the length of the buffer before handing control back to application
// threads. Needed to support correct flow control bookkeeping // threads. Needed to support correct flow control bookkeeping
if (error == GRPC_ERROR_NONE && s->recv_message->has_value()) { s->unprocessed_incoming_frames_buffer_cached_length =
s->unprocessed_incoming_frames_buffer.length;
if (error == GRPC_ERROR_NONE && *s->recv_message != nullptr) {
null_then_sched_closure(&s->recv_message_ready); null_then_sched_closure(&s->recv_message_ready);
} else if (s->published_metadata[1] != GRPC_METADATA_NOT_PUBLISHED) { } else if (s->published_metadata[1] != GRPC_METADATA_NOT_PUBLISHED) {
*s->recv_message = nullptr;
if (s->call_failed_before_recv_message != nullptr) { if (s->call_failed_before_recv_message != nullptr) {
*s->call_failed_before_recv_message = *s->call_failed_before_recv_message =
(s->published_metadata[1] != GRPC_METADATA_PUBLISHED_AT_CLOSE); (s->published_metadata[1] != GRPC_METADATA_PUBLISHED_AT_CLOSE);
@ -1993,8 +2096,26 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_chttp2_transport* t,
s->write_closed) { s->write_closed) {
if (s->seen_error || !t->is_client) { if (s->seen_error || !t->is_client) {
grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage); grpc_slice_buffer_reset_and_unref_internal(&s->frame_storage);
if (!s->pending_byte_stream) {
grpc_slice_buffer_reset_and_unref_internal(
&s->unprocessed_incoming_frames_buffer);
}
} }
if (s->read_closed && s->frame_storage.length == 0 && bool pending_data = s->pending_byte_stream ||
s->unprocessed_incoming_frames_buffer.length > 0;
if (s->read_closed && s->frame_storage.length > 0 && !pending_data &&
!s->seen_error && s->recv_trailing_metadata_finished != nullptr) {
// Maybe some SYNC_FLUSH data is left in frame_storage. Consume them and
// maybe decompress the next 5 bytes in the stream.
grpc_slice_buffer_move_first(
&s->frame_storage,
std::min(s->frame_storage.length, size_t(GRPC_HEADER_SIZE_IN_BYTES)),
&s->unprocessed_incoming_frames_buffer);
if (s->unprocessed_incoming_frames_buffer.length > 0) {
pending_data = true;
}
}
if (s->read_closed && s->frame_storage.length == 0 && !pending_data &&
s->recv_trailing_metadata_finished != nullptr) { s->recv_trailing_metadata_finished != nullptr) {
grpc_transport_move_stats(&s->stats, s->collecting_stats); grpc_transport_move_stats(&s->stats, s->collecting_stats);
s->collecting_stats = nullptr; s->collecting_stats = nullptr;
@ -2014,6 +2135,20 @@ static void remove_stream(grpc_chttp2_transport* t, uint32_t id,
t->incoming_stream = nullptr; t->incoming_stream = nullptr;
grpc_chttp2_parsing_become_skip_parser(t); grpc_chttp2_parsing_become_skip_parser(t);
} }
if (s->pending_byte_stream) {
if (s->on_next != nullptr) {
grpc_core::Chttp2IncomingByteStream* bs = s->data_parser.parsing_frame;
if (error == GRPC_ERROR_NONE) {
error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message");
}
bs->PublishError(error);
bs->Unref();
s->data_parser.parsing_frame = nullptr;
} else {
GRPC_ERROR_UNREF(s->byte_stream_error);
s->byte_stream_error = GRPC_ERROR_REF(error);
}
}
if (grpc_chttp2_stream_map_size(&t->stream_map) == 0) { if (grpc_chttp2_stream_map_size(&t->stream_map) == 0) {
post_benign_reclaimer(t); post_benign_reclaimer(t);
@ -2147,7 +2282,8 @@ void grpc_chttp2_fail_pending_writes(grpc_chttp2_transport* t,
GRPC_ERROR_REF(error), GRPC_ERROR_REF(error),
"send_trailing_metadata_finished"); "send_trailing_metadata_finished");
grpc_chttp2_complete_closure_step(t, s, &s->send_message_finished, s->fetching_send_message.reset();
grpc_chttp2_complete_closure_step(t, s, &s->fetching_send_message_finished,
GRPC_ERROR_REF(error), GRPC_ERROR_REF(error),
"fetching_send_message_finished"); "fetching_send_message_finished");
flush_write_list(t, s, &s->on_write_finished_cbs, GRPC_ERROR_REF(error)); flush_write_list(t, s, &s->on_write_finished_cbs, GRPC_ERROR_REF(error));
@ -2417,11 +2553,8 @@ void grpc_chttp2_act_on_flowctl_action(
const grpc_core::chttp2::FlowControlAction& action, const grpc_core::chttp2::FlowControlAction& action,
grpc_chttp2_transport* t, grpc_chttp2_stream* s) { grpc_chttp2_transport* t, grpc_chttp2_stream* s) {
WithUrgency(t, action.send_stream_update(), WithUrgency(t, action.send_stream_update(),
GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL, [t, s]() { GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL,
if (s->id != 0) { [t, s]() { grpc_chttp2_mark_stream_writable(t, s); });
grpc_chttp2_mark_stream_writable(t, s);
}
});
WithUrgency(t, action.send_transport_update(), WithUrgency(t, action.send_transport_update(),
GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL, []() {}); GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL, []() {});
WithUrgency(t, action.send_initial_window_update(), WithUrgency(t, action.send_initial_window_update(),
@ -2569,13 +2702,13 @@ static void continue_read_action_locked(grpc_chttp2_transport* t) {
grpc_schedule_on_exec_ctx); grpc_schedule_on_exec_ctx);
grpc_endpoint_read(t->ep, &t->read_buffer, &t->read_action_locked, urgent, grpc_endpoint_read(t->ep, &t->read_buffer, &t->read_action_locked, urgent,
/*min_progress_size=*/1); /*min_progress_size=*/1);
grpc_chttp2_act_on_flowctl_action(t->flow_control.MakeAction(), t, nullptr); grpc_chttp2_act_on_flowctl_action(t->flow_control->MakeAction(), t, nullptr);
} }
// t is reffed prior to calling the first time, and once the callback chain // t is reffed prior to calling the first time, and once the callback chain
// that kicks off finishes, it's unreffed // that kicks off finishes, it's unreffed
void schedule_bdp_ping_locked(grpc_chttp2_transport* t) { void schedule_bdp_ping_locked(grpc_chttp2_transport* t) {
t->flow_control.bdp_estimator()->SchedulePing(); t->flow_control->bdp_estimator()->SchedulePing();
send_ping_locked( send_ping_locked(
t, t,
GRPC_CLOSURE_INIT(&t->start_bdp_ping_locked, start_bdp_ping, t, GRPC_CLOSURE_INIT(&t->start_bdp_ping_locked, start_bdp_ping, t,
@ -2605,7 +2738,7 @@ static void start_bdp_ping_locked(void* tp, grpc_error_handle error) {
if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_WAITING) { if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_WAITING) {
grpc_timer_cancel(&t->keepalive_ping_timer); grpc_timer_cancel(&t->keepalive_ping_timer);
} }
t->flow_control.bdp_estimator()->StartPing(); t->flow_control->bdp_estimator()->StartPing();
t->bdp_ping_started = true; t->bdp_ping_started = true;
} }
@ -2636,8 +2769,8 @@ static void finish_bdp_ping_locked(void* tp, grpc_error_handle error) {
} }
t->bdp_ping_started = false; t->bdp_ping_started = false;
grpc_core::Timestamp next_ping = grpc_core::Timestamp next_ping =
t->flow_control.bdp_estimator()->CompletePing(); t->flow_control->bdp_estimator()->CompletePing();
grpc_chttp2_act_on_flowctl_action(t->flow_control.PeriodicUpdate(), t, grpc_chttp2_act_on_flowctl_action(t->flow_control->PeriodicUpdate(), t,
nullptr); nullptr);
GPR_ASSERT(!t->have_next_bdp_ping_timer); GPR_ASSERT(!t->have_next_bdp_ping_timer);
t->have_next_bdp_ping_timer = true; t->have_next_bdp_ping_timer = true;
@ -2664,7 +2797,7 @@ static void next_bdp_ping_timer_expired_locked(void* tp,
GRPC_CHTTP2_UNREF_TRANSPORT(t, "bdp_ping"); GRPC_CHTTP2_UNREF_TRANSPORT(t, "bdp_ping");
return; return;
} }
if (t->flow_control.bdp_estimator()->accumulator() == 0) { if (t->flow_control->bdp_estimator()->accumulator() == 0) {
// Block the bdp ping till we receive more data. // Block the bdp ping till we receive more data.
t->bdp_ping_blocked = true; t->bdp_ping_blocked = true;
GRPC_CHTTP2_UNREF_TRANSPORT(t, "bdp_ping"); GRPC_CHTTP2_UNREF_TRANSPORT(t, "bdp_ping");
@ -2900,6 +3033,187 @@ static void set_pollset_set(grpc_transport* gt, grpc_stream* /*gs*/,
grpc_endpoint_add_to_pollset_set(t->ep, pollset_set); grpc_endpoint_add_to_pollset_set(t->ep, pollset_set);
} }
//
// BYTE STREAM
//
static void reset_byte_stream(void* arg, grpc_error_handle error) {
grpc_chttp2_stream* s = static_cast<grpc_chttp2_stream*>(arg);
s->pending_byte_stream = false;
if (error == GRPC_ERROR_NONE) {
grpc_chttp2_maybe_complete_recv_message(s->t, s);
grpc_chttp2_maybe_complete_recv_trailing_metadata(s->t, s);
} else {
GPR_ASSERT(error != GRPC_ERROR_NONE);
grpc_core::ExecCtx::Run(DEBUG_LOCATION, s->on_next, GRPC_ERROR_REF(error));
s->on_next = nullptr;
GRPC_ERROR_UNREF(s->byte_stream_error);
s->byte_stream_error = GRPC_ERROR_NONE;
grpc_chttp2_cancel_stream(s->t, s, GRPC_ERROR_REF(error));
s->byte_stream_error = GRPC_ERROR_REF(error);
}
}
namespace grpc_core {
Chttp2IncomingByteStream::Chttp2IncomingByteStream(
grpc_chttp2_transport* transport, grpc_chttp2_stream* stream,
uint32_t frame_size, uint32_t flags)
: ByteStream(frame_size, flags),
transport_(transport),
stream_(stream),
refs_(2),
remaining_bytes_(frame_size) {
GRPC_ERROR_UNREF(stream->byte_stream_error);
stream->byte_stream_error = GRPC_ERROR_NONE;
}
void Chttp2IncomingByteStream::OrphanLocked(
void* arg, grpc_error_handle /*error_ignored*/) {
Chttp2IncomingByteStream* bs = static_cast<Chttp2IncomingByteStream*>(arg);
grpc_chttp2_stream* s = bs->stream_;
grpc_chttp2_transport* t = s->t;
bs->Unref();
s->pending_byte_stream = false;
grpc_chttp2_maybe_complete_recv_message(t, s);
grpc_chttp2_maybe_complete_recv_trailing_metadata(t, s);
}
void Chttp2IncomingByteStream::Orphan() {
GPR_TIMER_SCOPE("incoming_byte_stream_destroy", 0);
transport_->combiner->Run(
GRPC_CLOSURE_INIT(&destroy_action_,
&Chttp2IncomingByteStream::OrphanLocked, this, nullptr),
GRPC_ERROR_NONE);
}
void Chttp2IncomingByteStream::NextLocked(void* arg,
grpc_error_handle /*error_ignored*/) {
Chttp2IncomingByteStream* bs = static_cast<Chttp2IncomingByteStream*>(arg);
grpc_chttp2_transport* t = bs->transport_;
grpc_chttp2_stream* s = bs->stream_;
size_t cur_length = s->frame_storage.length;
if (!s->read_closed) {
s->flow_control->IncomingByteStreamUpdate(bs->next_action_.max_size_hint,
cur_length);
grpc_chttp2_act_on_flowctl_action(s->flow_control->MakeAction(), t, s);
}
GPR_ASSERT(s->unprocessed_incoming_frames_buffer.length == 0);
if (s->frame_storage.length > 0) {
grpc_slice_buffer_swap(&s->frame_storage,
&s->unprocessed_incoming_frames_buffer);
ExecCtx::Run(DEBUG_LOCATION, bs->next_action_.on_complete, GRPC_ERROR_NONE);
} else if (s->byte_stream_error != GRPC_ERROR_NONE) {
ExecCtx::Run(DEBUG_LOCATION, bs->next_action_.on_complete,
GRPC_ERROR_REF(s->byte_stream_error));
if (s->data_parser.parsing_frame != nullptr) {
s->data_parser.parsing_frame->Unref();
s->data_parser.parsing_frame = nullptr;
}
} else if (s->read_closed) {
if (bs->remaining_bytes_ != 0) {
s->byte_stream_error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
"Truncated message", &s->read_closed_error, 1);
ExecCtx::Run(DEBUG_LOCATION, bs->next_action_.on_complete,
GRPC_ERROR_REF(s->byte_stream_error));
if (s->data_parser.parsing_frame != nullptr) {
s->data_parser.parsing_frame->Unref();
s->data_parser.parsing_frame = nullptr;
}
} else {
// Should never reach here.
GPR_ASSERT(false);
}
} else {
s->on_next = bs->next_action_.on_complete;
}
bs->Unref();
}
bool Chttp2IncomingByteStream::Next(size_t max_size_hint,
grpc_closure* on_complete) {
GPR_TIMER_SCOPE("incoming_byte_stream_next", 0);
if (stream_->unprocessed_incoming_frames_buffer.length > 0) {
return true;
} else {
Ref();
next_action_.max_size_hint = max_size_hint;
next_action_.on_complete = on_complete;
transport_->combiner->Run(
GRPC_CLOSURE_INIT(&next_action_.closure,
&Chttp2IncomingByteStream::NextLocked, this, nullptr),
GRPC_ERROR_NONE);
return false;
}
}
grpc_error_handle Chttp2IncomingByteStream::Pull(grpc_slice* slice) {
GPR_TIMER_SCOPE("incoming_byte_stream_pull", 0);
grpc_error_handle error;
if (stream_->unprocessed_incoming_frames_buffer.length > 0) {
error = grpc_deframe_unprocessed_incoming_frames(
&stream_->data_parser, stream_,
&stream_->unprocessed_incoming_frames_buffer, slice, nullptr);
if (error != GRPC_ERROR_NONE) {
return error;
}
} else {
error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message");
stream_->t->combiner->Run(&stream_->reset_byte_stream,
GRPC_ERROR_REF(error));
return error;
}
return GRPC_ERROR_NONE;
}
void Chttp2IncomingByteStream::PublishError(grpc_error_handle error) {
GPR_ASSERT(error != GRPC_ERROR_NONE);
ExecCtx::Run(DEBUG_LOCATION, stream_->on_next, GRPC_ERROR_REF(error));
stream_->on_next = nullptr;
GRPC_ERROR_UNREF(stream_->byte_stream_error);
stream_->byte_stream_error = GRPC_ERROR_REF(error);
grpc_chttp2_cancel_stream(transport_, stream_, GRPC_ERROR_REF(error));
}
grpc_error_handle Chttp2IncomingByteStream::Push(const grpc_slice& slice,
grpc_slice* slice_out) {
if (remaining_bytes_ < GRPC_SLICE_LENGTH(slice)) {
grpc_error_handle error =
GRPC_ERROR_CREATE_FROM_STATIC_STRING("Too many bytes in stream");
transport_->combiner->Run(&stream_->reset_byte_stream,
GRPC_ERROR_REF(error));
grpc_slice_unref_internal(slice);
return error;
} else {
remaining_bytes_ -= static_cast<uint32_t> GRPC_SLICE_LENGTH(slice);
if (slice_out != nullptr) {
*slice_out = slice;
}
return GRPC_ERROR_NONE;
}
}
grpc_error_handle Chttp2IncomingByteStream::Finished(grpc_error_handle error,
bool reset_on_error) {
if (error == GRPC_ERROR_NONE) {
if (remaining_bytes_ != 0) {
error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message");
}
}
if (error != GRPC_ERROR_NONE && reset_on_error) {
transport_->combiner->Run(&stream_->reset_byte_stream,
GRPC_ERROR_REF(error));
}
Unref();
return error;
}
void Chttp2IncomingByteStream::Shutdown(grpc_error_handle error) {
GRPC_ERROR_UNREF(Finished(error, true /* reset_on_error */));
}
} // namespace grpc_core
// //
// RESOURCE QUOTAS // RESOURCE QUOTAS
// //

@ -23,18 +23,18 @@
#include <inttypes.h> #include <inttypes.h>
#include <limits.h> #include <limits.h>
#include <algorithm>
#include <cmath> #include <cmath>
#include <ostream>
#include <string> #include <string>
#include <vector>
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include <grpc/slice.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h> #include <grpc/support/log.h>
#include "src/core/ext/transport/chttp2/transport/frame.h"
#include "src/core/ext/transport/chttp2/transport/internal.h"
#include "src/core/lib/gpr/string.h"
#include "src/core/lib/gpr/useful.h" #include "src/core/lib/gpr/useful.h"
#include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/exec_ctx.h"
#include "src/core/lib/resource_quota/memory_quota.h" #include "src/core/lib/resource_quota/memory_quota.h"
@ -51,61 +51,140 @@ bool g_test_only_transport_flow_control_window_check;
namespace { namespace {
constexpr const int kTracePadding = 30;
constexpr const int64_t kMaxWindowUpdateSize = (1u << 31) - 1; constexpr const int64_t kMaxWindowUpdateSize = (1u << 31) - 1;
char* fmt_int64_diff_str(int64_t old_val, int64_t new_val) {
std::string str;
if (old_val != new_val) {
str = absl::StrFormat("%" PRId64 " -> %" PRId64 "", old_val, new_val);
} else {
str = absl::StrFormat("%" PRId64 "", old_val);
}
return gpr_leftpad(str.c_str(), ' ', kTracePadding);
}
char* fmt_uint32_diff_str(uint32_t old_val, uint32_t new_val) {
std::string str;
if (old_val != new_val) {
str = absl::StrFormat("%" PRIu32 " -> %" PRIu32 "", old_val, new_val);
} else {
str = absl::StrFormat("%" PRIu32 "", old_val);
}
return gpr_leftpad(str.c_str(), ' ', kTracePadding);
}
} // namespace } // namespace
void FlowControlTrace::Init(const char* reason, TransportFlowControl* tfc,
StreamFlowControl* sfc) {
tfc_ = tfc;
sfc_ = sfc;
reason_ = reason;
remote_window_ = tfc->remote_window();
target_window_ = tfc->target_window();
announced_window_ = tfc->announced_window();
if (sfc != nullptr) {
remote_window_delta_ = sfc->remote_window_delta();
local_window_delta_ = sfc->local_window_delta();
announced_window_delta_ = sfc->announced_window_delta();
}
}
void FlowControlTrace::Finish() {
uint32_t acked_local_window =
tfc_->transport()->settings[GRPC_SENT_SETTINGS]
[GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
uint32_t remote_window =
tfc_->transport()->settings[GRPC_PEER_SETTINGS]
[GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
char* trw_str = fmt_int64_diff_str(remote_window_, tfc_->remote_window());
char* tlw_str = fmt_int64_diff_str(target_window_, tfc_->target_window());
char* taw_str =
fmt_int64_diff_str(announced_window_, tfc_->announced_window());
char* srw_str;
char* slw_str;
char* saw_str;
if (sfc_ != nullptr) {
srw_str = fmt_int64_diff_str(remote_window_delta_ + remote_window,
sfc_->remote_window_delta() + remote_window);
slw_str =
fmt_int64_diff_str(local_window_delta_ + acked_local_window,
sfc_->local_window_delta() + acked_local_window);
saw_str =
fmt_int64_diff_str(announced_window_delta_ + acked_local_window,
sfc_->announced_window_delta() + acked_local_window);
} else {
srw_str = gpr_leftpad("", ' ', kTracePadding);
slw_str = gpr_leftpad("", ' ', kTracePadding);
saw_str = gpr_leftpad("", ' ', kTracePadding);
}
gpr_log(GPR_DEBUG,
"%p[%u][%s] | %s | trw:%s, tlw:%s, taw:%s, srw:%s, slw:%s, saw:%s",
tfc_, sfc_ != nullptr ? sfc_->stream()->id : 0,
tfc_->transport()->is_client ? "cli" : "svr", reason_, trw_str,
tlw_str, taw_str, srw_str, slw_str, saw_str);
gpr_free(trw_str);
gpr_free(tlw_str);
gpr_free(taw_str);
gpr_free(srw_str);
gpr_free(slw_str);
gpr_free(saw_str);
}
const char* FlowControlAction::UrgencyString(Urgency u) { const char* FlowControlAction::UrgencyString(Urgency u) {
switch (u) { switch (u) {
case Urgency::NO_ACTION_NEEDED: case Urgency::NO_ACTION_NEEDED:
return "no-action"; return "no action";
case Urgency::UPDATE_IMMEDIATELY: case Urgency::UPDATE_IMMEDIATELY:
return "now"; return "update immediately";
case Urgency::QUEUE_UPDATE: case Urgency::QUEUE_UPDATE:
return "queue"; return "queue update";
default: default:
GPR_UNREACHABLE_CODE(return "unknown"); GPR_UNREACHABLE_CODE(return "unknown");
} }
GPR_UNREACHABLE_CODE(return "unknown"); GPR_UNREACHABLE_CODE(return "unknown");
} }
std::ostream& operator<<(std::ostream& out, FlowControlAction::Urgency u) { void FlowControlAction::Trace(grpc_chttp2_transport* t) const {
return out << FlowControlAction::UrgencyString(u); char* iw_str = fmt_uint32_diff_str(
t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE],
initial_window_size_);
char* mf_str = fmt_uint32_diff_str(
t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE],
max_frame_size_);
gpr_log(GPR_DEBUG, "t[%s], s[%s], iw:%s:%s mf:%s:%s",
UrgencyString(send_transport_update_),
UrgencyString(send_stream_update_),
UrgencyString(send_initial_window_update_), iw_str,
UrgencyString(send_max_frame_size_update_), mf_str);
gpr_free(iw_str);
gpr_free(mf_str);
} }
std::string FlowControlAction::DebugString() const { TransportFlowControlDisabled::TransportFlowControlDisabled(
std::vector<std::string> segments; grpc_chttp2_transport* t) {
if (send_transport_update_ != Urgency::NO_ACTION_NEEDED) { remote_window_ = kMaxWindow;
segments.push_back( target_initial_window_size_ = kMaxWindow;
absl::StrCat("t:", UrgencyString(send_transport_update_))); announced_window_ = kMaxWindow;
} t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
if (send_stream_update_ != Urgency::NO_ACTION_NEEDED) { kFrameSize;
segments.push_back(absl::StrCat("s:", UrgencyString(send_stream_update_))); t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
} kFrameSize;
if (send_initial_window_update_ != Urgency::NO_ACTION_NEEDED) { t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] =
segments.push_back( kFrameSize;
absl::StrCat("iw=", initial_window_size_, ":", t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
UrgencyString(send_initial_window_update_))); kMaxWindow;
} t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
if (send_max_frame_size_update_ != Urgency::NO_ACTION_NEEDED) { kMaxWindow;
segments.push_back( t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
absl::StrCat("mf=", max_frame_size_, ":", kMaxWindow;
UrgencyString(send_max_frame_size_update_)));
}
if (segments.empty()) return "no action";
return absl::StrJoin(segments, ",");
} }
std::ostream& operator<<(std::ostream& out, const FlowControlAction& action) { TransportFlowControl::TransportFlowControl(const grpc_chttp2_transport* t,
return out << action.DebugString(); bool enable_bdp_probe)
} : t_(t),
TransportFlowControl::TransportFlowControl(const char* name,
bool enable_bdp_probe,
MemoryOwner* memory_owner)
: memory_owner_(memory_owner),
enable_bdp_probe_(enable_bdp_probe), enable_bdp_probe_(enable_bdp_probe),
bdp_estimator_(name), bdp_estimator_(t->peer_string.c_str()),
pid_controller_(PidController::Args() pid_controller_(PidController::Args()
.set_gain_p(4) .set_gain_p(4)
.set_gain_i(8) .set_gain_i(8)
@ -117,6 +196,7 @@ TransportFlowControl::TransportFlowControl(const char* name,
last_pid_update_(ExecCtx::Get()->Now()) {} last_pid_update_(ExecCtx::Get()->Now()) {}
uint32_t TransportFlowControl::MaybeSendUpdate(bool writing_anyway) { uint32_t TransportFlowControl::MaybeSendUpdate(bool writing_anyway) {
FlowControlTrace trace("t updt sent", this, nullptr);
const uint32_t target_announced_window = const uint32_t target_announced_window =
static_cast<uint32_t>(target_window()); static_cast<uint32_t>(target_window());
if ((writing_anyway || announced_window_ <= target_announced_window / 2) && if ((writing_anyway || announced_window_ <= target_announced_window / 2) &&
@ -130,47 +210,80 @@ uint32_t TransportFlowControl::MaybeSendUpdate(bool writing_anyway) {
return 0; return 0;
} }
absl::Status TransportFlowControl::ValidateRecvData( grpc_error_handle TransportFlowControl::ValidateRecvData(
int64_t incoming_frame_size) { int64_t incoming_frame_size) {
if (incoming_frame_size > announced_window_) { if (incoming_frame_size > announced_window_) {
return absl::InternalError(absl::StrFormat( return GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat(
"frame of size %" PRId64 " overflows local window of %" PRId64, "frame of size %" PRId64 " overflows local window of %" PRId64,
incoming_frame_size, announced_window_)); incoming_frame_size, announced_window_));
} }
return absl::OkStatus(); return GRPC_ERROR_NONE;
} }
void TransportFlowControl::CommitRecvData(int64_t incoming_frame_size) { StreamFlowControl::StreamFlowControl(TransportFlowControl* tfc,
announced_window_ -= incoming_frame_size; const grpc_chttp2_stream* s)
} : tfc_(tfc), s_(s) {}
StreamFlowControl::StreamFlowControl(TransportFlowControl* tfc) : tfc_(tfc) {} grpc_error_handle StreamFlowControl::RecvData(int64_t incoming_frame_size) {
FlowControlTrace trace(" data recv", tfc_, this);
absl::Status StreamFlowControl::RecvData(int64_t incoming_frame_size) { grpc_error_handle error = GRPC_ERROR_NONE;
absl::Status error = tfc_->ValidateRecvData(incoming_frame_size); error = tfc_->ValidateRecvData(incoming_frame_size);
if (!error.ok()) return error; if (error != GRPC_ERROR_NONE) return error;
int64_t acked_stream_window = uint32_t sent_init_window =
announced_window_delta_ + tfc_->acked_init_window(); tfc_->transport()->settings[GRPC_SENT_SETTINGS]
[GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
uint32_t acked_init_window =
tfc_->transport()->settings[GRPC_ACKED_SETTINGS]
[GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
int64_t acked_stream_window = announced_window_delta_ + acked_init_window;
int64_t sent_stream_window = announced_window_delta_ + sent_init_window;
if (incoming_frame_size > acked_stream_window) { if (incoming_frame_size > acked_stream_window) {
return absl::InternalError(absl::StrFormat( if (incoming_frame_size <= sent_stream_window) {
"frame of size %" PRId64 " overflows local window of %" PRId64, gpr_log(GPR_ERROR,
incoming_frame_size, acked_stream_window)); "Incoming frame of size %" PRId64
" exceeds local window size of %" PRId64
".\n"
"The (un-acked, future) window size would be %" PRId64
" which is not exceeded.\n"
"This would usually cause a disconnection, but allowing it due to"
"broken HTTP2 implementations in the wild.\n"
"See (for example) https://github.com/netty/netty/issues/6520.",
incoming_frame_size, acked_stream_window, sent_stream_window);
} else {
return GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat(
"frame of size %" PRId64 " overflows local window of %" PRId64,
incoming_frame_size, acked_stream_window));
}
} }
UpdateAnnouncedWindowDelta(tfc_, -incoming_frame_size); UpdateAnnouncedWindowDelta(tfc_, -incoming_frame_size);
local_window_delta_ -= incoming_frame_size; local_window_delta_ -= incoming_frame_size;
min_progress_size_ -=
std::min(static_cast<int64_t>(min_progress_size_), incoming_frame_size);
tfc_->CommitRecvData(incoming_frame_size); tfc_->CommitRecvData(incoming_frame_size);
return absl::OkStatus(); return GRPC_ERROR_NONE;
} }
uint32_t StreamFlowControl::MaybeSendUpdate() { uint32_t StreamFlowControl::MaybeSendUpdate() {
FlowControlTrace trace("s updt sent", tfc_, this);
// If a recently sent settings frame caused the stream's flow control window // If a recently sent settings frame caused the stream's flow control window
// to go in the negative (or < min_progress_size_), update the delta. // to go in the negative (or < GRPC_HEADER_SIZE_IN_BYTES), update the delta if
// In this case, we want to make sure that bytes are still flowing. // one of the following conditions is satisfied -
UpdateProgress(min_progress_size_); // 1) There is a pending byte_stream and higher layers have expressed interest
// in reading additional data through the invokation of `Next()` where the
// bytes are to be available asynchronously. 2) There is a pending
// recv_message op.
// In these cases, we want to make sure that bytes are still flowing.
if (local_window_delta_ < GRPC_HEADER_SIZE_IN_BYTES) {
if (s_->on_next != nullptr) {
GPR_DEBUG_ASSERT(s_->pending_byte_stream);
IncomingByteStreamUpdate(GRPC_HEADER_SIZE_IN_BYTES, 0);
} else if (s_->recv_message != nullptr) {
IncomingByteStreamUpdate(GRPC_HEADER_SIZE_IN_BYTES,
s_->frame_storage.length);
}
}
if (local_window_delta_ > announced_window_delta_) { if (local_window_delta_ > announced_window_delta_) {
uint32_t announce = static_cast<uint32_t>( uint32_t announce = static_cast<uint32_t>(
Clamp(local_window_delta_ - announced_window_delta_, int64_t(0), Clamp(local_window_delta_ - announced_window_delta_, int64_t(0),
@ -181,21 +294,32 @@ uint32_t StreamFlowControl::MaybeSendUpdate() {
return 0; return 0;
} }
void StreamFlowControl::UpdateProgress(uint32_t min_progress_size) { void StreamFlowControl::IncomingByteStreamUpdate(size_t max_size_hint,
size_t have_already) {
FlowControlTrace trace("app st recv", tfc_, this);
uint32_t max_recv_bytes; uint32_t max_recv_bytes;
min_progress_size_ = min_progress_size;
/* clamp max recv hint to an allowable size */ /* clamp max recv hint to an allowable size */
if (min_progress_size >= kMaxWindowDelta) { if (max_size_hint >= kMaxWindowDelta) {
max_recv_bytes = kMaxWindowDelta; max_recv_bytes = kMaxWindowDelta;
} else { } else {
max_recv_bytes = static_cast<uint32_t>(min_progress_size); max_recv_bytes = static_cast<uint32_t>(max_size_hint);
}
/* account for bytes already received but unknown to higher layers */
if (max_recv_bytes >= have_already) {
max_recv_bytes -= static_cast<uint32_t>(have_already);
} else {
max_recv_bytes = 0;
} }
/* add some small lookahead to keep pipelines flowing */ /* add some small lookahead to keep pipelines flowing */
GPR_DEBUG_ASSERT(max_recv_bytes <= GPR_DEBUG_ASSERT(
kMaxWindowUpdateSize - tfc_->sent_init_window()); max_recv_bytes <=
kMaxWindowUpdateSize -
tfc_->transport()
->settings[GRPC_SENT_SETTINGS]
[GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]);
if (local_window_delta_ < max_recv_bytes) { if (local_window_delta_ < max_recv_bytes) {
uint32_t add_max_recv_bytes = uint32_t add_max_recv_bytes =
static_cast<uint32_t>(max_recv_bytes - local_window_delta_); static_cast<uint32_t>(max_recv_bytes - local_window_delta_);
@ -203,32 +327,6 @@ void StreamFlowControl::UpdateProgress(uint32_t min_progress_size) {
} }
} }
absl::Status TransportFlowControl::RecvData(int64_t incoming_frame_size) {
absl::Status error = ValidateRecvData(incoming_frame_size);
if (error.ok()) return error;
CommitRecvData(incoming_frame_size);
return absl::OkStatus();
}
void TransportFlowControl::RecvUpdate(uint32_t size) { remote_window_ += size; }
int64_t TransportFlowControl::target_window() const {
// See comment above announced_stream_total_over_incoming_window_ for the
// logic behind this decision.
return static_cast<uint32_t>(
std::min(static_cast<int64_t>((1u << 31) - 1),
announced_stream_total_over_incoming_window_ +
target_initial_window_size_));
}
FlowControlAction TransportFlowControl::UpdateAction(FlowControlAction action) {
if (announced_window_ < target_window() / 2) {
action.set_send_transport_update(
FlowControlAction::Urgency::UPDATE_IMMEDIATELY);
}
return action;
}
// Take in a target and modifies it based on the memory pressure of the system // Take in a target and modifies it based on the memory pressure of the system
static double AdjustForMemoryPressure(double memory_pressure, double target) { static double AdjustForMemoryPressure(double memory_pressure, double target) {
// do not increase window under heavy memory pressure. // do not increase window under heavy memory pressure.
@ -247,9 +345,10 @@ static double AdjustForMemoryPressure(double memory_pressure, double target) {
} }
double TransportFlowControl::TargetLogBdp() { double TransportFlowControl::TargetLogBdp() {
return AdjustForMemoryPressure( return AdjustForMemoryPressure(t_->memory_owner.is_valid()
memory_owner_->is_valid() ? memory_owner_->InstantaneousPressure() : 0.0, ? t_->memory_owner.InstantaneousPressure()
1 + log2(bdp_estimator_.EstimateBdp())); : 0.0,
1 + log2(bdp_estimator_.EstimateBdp()));
} }
double TransportFlowControl::SmoothLogBdp(double value) { double TransportFlowControl::SmoothLogBdp(double value) {
@ -262,17 +361,15 @@ double TransportFlowControl::SmoothLogBdp(double value) {
return pid_controller_.Update(bdp_error, dt > kMaxDt ? kMaxDt : dt); return pid_controller_.Update(bdp_error, dt > kMaxDt ? kMaxDt : dt);
} }
void TransportFlowControl::UpdateSetting( FlowControlAction::Urgency TransportFlowControl::DeltaUrgency(
int64_t* desired_value, int64_t new_desired_value, int64_t value, grpc_chttp2_setting_id setting_id) {
FlowControlAction* action, int64_t delta = value - static_cast<int64_t>(
FlowControlAction& (FlowControlAction::*set)(FlowControlAction::Urgency, t_->settings[GRPC_LOCAL_SETTINGS][setting_id]);
uint32_t)) {
int64_t delta = new_desired_value - *desired_value;
// TODO(ncteisen): tune this // TODO(ncteisen): tune this
if (delta != 0 && if (delta != 0 && (delta <= -value / 5 || delta >= value / 5)) {
(delta <= -*desired_value / 5 || delta >= *desired_value / 5)) { return FlowControlAction::Urgency::QUEUE_UPDATE;
*desired_value = new_desired_value; } else {
(action->*set)(FlowControlAction::Urgency::QUEUE_UPDATE, *desired_value); return FlowControlAction::Urgency::NO_ACTION_NEEDED;
} }
} }
@ -292,51 +389,46 @@ FlowControlAction TransportFlowControl::PeriodicUpdate() {
} }
// Though initial window 'could' drop to 0, we keep the floor at // Though initial window 'could' drop to 0, we keep the floor at
// kMinInitialWindowSize // kMinInitialWindowSize
UpdateSetting( target_initial_window_size_ = static_cast<int32_t>(Clamp(
&target_initial_window_size_, target, double(kMinInitialWindowSize), double(kMaxInitialWindowSize)));
static_cast<int32_t>(Clamp(target, double(kMinInitialWindowSize), action.set_send_initial_window_update(
double(kMaxInitialWindowSize))), DeltaUrgency(target_initial_window_size_,
&action, &FlowControlAction::set_send_initial_window_update); GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE),
static_cast<uint32_t>(target_initial_window_size_));
// get bandwidth estimate and update max_frame accordingly. // get bandwidth estimate and update max_frame accordingly.
double bw_dbl = bdp_estimator_.EstimateBandwidth(); double bw_dbl = bdp_estimator_.EstimateBandwidth();
// we target the max of BDP or bandwidth in microseconds. // we target the max of BDP or bandwidth in microseconds.
UpdateSetting( int32_t frame_size = static_cast<int32_t>(Clamp(
&target_frame_size_, std::max(
static_cast<int32_t>(Clamp( static_cast<int32_t>(Clamp(bw_dbl, 0.0, double(INT_MAX))) / 1000,
std::max(static_cast<int32_t>(Clamp(bw_dbl, 0.0, double(INT_MAX))) / static_cast<int32_t>(target_initial_window_size_)),
1000, 16384, 16777215));
static_cast<int32_t>(target_initial_window_size_)), action.set_send_max_frame_size_update(
16384, 16777215)), DeltaUrgency(static_cast<int64_t>(frame_size),
&action, &FlowControlAction::set_send_max_frame_size_update); GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE),
frame_size);
} }
return UpdateAction(action); return UpdateAction(action);
} }
FlowControlAction StreamFlowControl::UpdateAction(FlowControlAction action) { FlowControlAction StreamFlowControl::UpdateAction(FlowControlAction action) {
const uint32_t sent_init_window = tfc_->sent_init_window(); // TODO(ncteisen): tune this
if (local_window_delta_ > announced_window_delta_ && if (!s_->read_closed) {
announced_window_delta_ + sent_init_window <= sent_init_window / 2) { uint32_t sent_init_window =
action.set_send_stream_update( tfc_->transport()->settings[GRPC_SENT_SETTINGS]
FlowControlAction::Urgency::UPDATE_IMMEDIATELY); [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
} else if (local_window_delta_ > announced_window_delta_) { if (local_window_delta_ > announced_window_delta_ &&
action.set_send_stream_update(FlowControlAction::Urgency::QUEUE_UPDATE); announced_window_delta_ + sent_init_window <= sent_init_window / 2) {
action.set_send_stream_update(
FlowControlAction::Urgency::UPDATE_IMMEDIATELY);
} else if (local_window_delta_ > announced_window_delta_) {
action.set_send_stream_update(FlowControlAction::Urgency::QUEUE_UPDATE);
}
} }
return action; return action;
} }
void StreamFlowControl::UpdateAnnouncedWindowDelta(TransportFlowControl* tfc,
int64_t change) {
tfc->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_);
announced_window_delta_ += change;
tfc->PostUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_);
}
void StreamFlowControl::SentData(int64_t outgoing_frame_size) {
tfc_->StreamSentData(outgoing_frame_size);
remote_window_delta_ -= outgoing_frame_size;
}
} // namespace chttp2 } // namespace chttp2
} // namespace grpc_core } // namespace grpc_core

@ -22,18 +22,20 @@
#include <grpc/support/port_platform.h> #include <grpc/support/port_platform.h>
#include <stdint.h> #include <stdint.h>
#include <stdlib.h>
#include <iosfwd> #include <algorithm>
#include <string>
#include "absl/status/status.h"
#include "src/core/ext/transport/chttp2/transport/http2_settings.h"
#include "src/core/lib/debug/trace.h" #include "src/core/lib/debug/trace.h"
#include "src/core/lib/gprpp/time.h" #include "src/core/lib/gprpp/time.h"
#include "src/core/lib/resource_quota/memory_quota.h" #include "src/core/lib/iomgr/error.h"
#include "src/core/lib/transport/bdp_estimator.h" #include "src/core/lib/transport/bdp_estimator.h"
#include "src/core/lib/transport/pid_controller.h" #include "src/core/lib/transport/pid_controller.h"
struct grpc_chttp2_transport;
struct grpc_chttp2_stream;
extern grpc_core::TraceFlag grpc_flowctl_trace; extern grpc_core::TraceFlag grpc_flowctl_trace;
namespace grpc { namespace grpc {
@ -46,7 +48,6 @@ namespace grpc_core {
namespace chttp2 { namespace chttp2 {
static constexpr uint32_t kDefaultWindow = 65535; static constexpr uint32_t kDefaultWindow = 65535;
static constexpr uint32_t kDefaultFrameSize = 16384;
static constexpr int64_t kMaxWindow = static_cast<int64_t>((1u << 31) - 1); static constexpr int64_t kMaxWindow = static_cast<int64_t>((1u << 31) - 1);
// TODO(ncteisen): Tune this // TODO(ncteisen): Tune this
static constexpr uint32_t kFrameSize = 1024 * 1024; static constexpr uint32_t kFrameSize = 1024 * 1024;
@ -108,16 +109,7 @@ class FlowControlAction {
} }
static const char* UrgencyString(Urgency u); static const char* UrgencyString(Urgency u);
std::string DebugString() const; void Trace(grpc_chttp2_transport* t) const;
bool operator==(const FlowControlAction& other) const {
return send_stream_update_ == other.send_stream_update_ &&
send_transport_update_ == other.send_transport_update_ &&
send_initial_window_update_ == other.send_initial_window_update_ &&
send_max_frame_size_update_ == other.send_max_frame_size_update_ &&
initial_window_size_ == other.initial_window_size_ &&
max_frame_size_ == other.max_frame_size_;
}
private: private:
Urgency send_stream_update_ = Urgency::NO_ACTION_NEEDED; Urgency send_stream_update_ = Urgency::NO_ACTION_NEEDED;
@ -128,16 +120,120 @@ class FlowControlAction {
uint32_t max_frame_size_ = 0; uint32_t max_frame_size_ = 0;
}; };
std::ostream& operator<<(std::ostream& out, FlowControlAction::Urgency urgency); class FlowControlTrace {
std::ostream& operator<<(std::ostream& out, const FlowControlAction& action); public:
FlowControlTrace(const char* reason, TransportFlowControl* tfc,
StreamFlowControl* sfc) {
if (enabled_) Init(reason, tfc, sfc);
}
~FlowControlTrace() {
if (enabled_) Finish();
}
private:
void Init(const char* reason, TransportFlowControl* tfc,
StreamFlowControl* sfc);
void Finish();
const bool enabled_ = GRPC_TRACE_FLAG_ENABLED(grpc_flowctl_trace);
TransportFlowControl* tfc_;
StreamFlowControl* sfc_;
const char* reason_;
int64_t remote_window_;
int64_t target_window_;
int64_t announced_window_;
int64_t remote_window_delta_;
int64_t local_window_delta_;
int64_t announced_window_delta_;
};
// Fat interface with all methods a flow control implementation needs to
// support.
class TransportFlowControlBase {
public:
TransportFlowControlBase() {}
virtual ~TransportFlowControlBase() {}
// Is flow control enabled? This is needed in other codepaths like the checks
// in parsing and in writing.
virtual bool flow_control_enabled() const = 0;
// Called to check if the transport needs to send a WINDOW_UPDATE frame
virtual uint32_t MaybeSendUpdate(bool /* writing_anyway */) = 0;
// Using the protected members, returns and Action to be taken by the
// tranport.
virtual FlowControlAction MakeAction() = 0;
// Using the protected members, returns and Action to be taken by the
// tranport. Also checks for updates to our BDP estimate and acts
// accordingly.
virtual FlowControlAction PeriodicUpdate() = 0;
// Called to do bookkeeping when a stream owned by this transport sends
// data on the wire
virtual void StreamSentData(int64_t /* size */) = 0;
// Called to do bookkeeping when a stream owned by this transport receives
// data from the wire. Also does error checking for frame size.
virtual grpc_error_handle RecvData(int64_t /* incoming_frame_size */) = 0;
// Called to do bookkeeping when we receive a WINDOW_UPDATE frame.
virtual void RecvUpdate(uint32_t /* size */) = 0;
// Returns the BdpEstimator held by this object. Caller is responsible for
// checking for nullptr. TODO(ncteisen): consider fully encapsulating all
// bdp estimator actions inside TransportFlowControl
virtual BdpEstimator* bdp_estimator() { return nullptr; }
// Getters
int64_t remote_window() const { return remote_window_; }
virtual int64_t target_window() const { return target_initial_window_size_; }
int64_t announced_window() const { return announced_window_; }
// Used in certain benchmarks in which we don't want FlowControl to be a
// factor
virtual void TestOnlyForceHugeWindow() {}
protected:
friend class grpc::testing::TrickledCHTTP2;
int64_t remote_window_ = kDefaultWindow;
int64_t target_initial_window_size_ = kDefaultWindow;
int64_t announced_window_ = kDefaultWindow;
};
// Implementation of flow control that does NOTHING. Always returns maximum
// values, never initiates writes, and assumes that the remote peer is doing
// the same. To be used to narrow down on flow control as the cause of negative
// performance.
class TransportFlowControlDisabled final : public TransportFlowControlBase {
public:
// Maxes out all values
explicit TransportFlowControlDisabled(grpc_chttp2_transport* t);
bool flow_control_enabled() const override { return false; }
// Never do anything.
uint32_t MaybeSendUpdate(bool /* writing_anyway */) override { return 0; }
FlowControlAction MakeAction() override { return FlowControlAction(); }
FlowControlAction PeriodicUpdate() override { return FlowControlAction(); }
void StreamSentData(int64_t /* size */) override {}
grpc_error_handle RecvData(int64_t /* incoming_frame_size */) override {
return GRPC_ERROR_NONE;
}
void RecvUpdate(uint32_t /* size */) override {}
};
// Implementation of flow control that abides to HTTP/2 spec and attempts // Implementation of flow control that abides to HTTP/2 spec and attempts
// to be as performant as possible. // to be as performant as possible.
class TransportFlowControl final { class TransportFlowControl final : public TransportFlowControlBase {
public: public:
explicit TransportFlowControl(const char* name, bool enable_bdp_probe, TransportFlowControl(const grpc_chttp2_transport* t, bool enable_bdp_probe);
MemoryOwner* memory_owner); ~TransportFlowControl() override {}
~TransportFlowControl() {}
bool flow_control_enabled() const override { return true; }
bool bdp_probe() const { return enable_bdp_probe_; } bool bdp_probe() const { return enable_bdp_probe_; }
@ -145,30 +241,50 @@ class TransportFlowControl final {
// else returns zero; writing_anyway indicates if a write would happen // else returns zero; writing_anyway indicates if a write would happen
// regardless of the send - if it is false and this function returns non-zero, // regardless of the send - if it is false and this function returns non-zero,
// this announce will cause a write to occur // this announce will cause a write to occur
uint32_t MaybeSendUpdate(bool writing_anyway); uint32_t MaybeSendUpdate(bool writing_anyway) override;
// Reads the flow control data and returns and actionable struct that will // Reads the flow control data and returns and actionable struct that will
// tell chttp2 exactly what it needs to do // tell chttp2 exactly what it needs to do
FlowControlAction MakeAction() { return UpdateAction(FlowControlAction()); } FlowControlAction MakeAction() override {
return UpdateAction(FlowControlAction());
}
// Call periodically (at a low-ish rate, 100ms - 10s makes sense) // Call periodically (at a low-ish rate, 100ms - 10s makes sense)
// to perform more complex flow control calculations and return an action // to perform more complex flow control calculations and return an action
// to let chttp2 change its parameters // to let chttp2 change its parameters
FlowControlAction PeriodicUpdate(); FlowControlAction PeriodicUpdate() override;
void StreamSentData(int64_t size) { remote_window_ -= size; } void StreamSentData(int64_t size) override { remote_window_ -= size; }
absl::Status ValidateRecvData(int64_t incoming_frame_size); grpc_error_handle ValidateRecvData(int64_t incoming_frame_size);
void CommitRecvData(int64_t incoming_frame_size); void CommitRecvData(int64_t incoming_frame_size) {
announced_window_ -= incoming_frame_size;
}
absl::Status RecvData(int64_t incoming_frame_size); grpc_error_handle RecvData(int64_t incoming_frame_size) override {
FlowControlTrace trace(" data recv", this, nullptr);
grpc_error_handle error = ValidateRecvData(incoming_frame_size);
if (error != GRPC_ERROR_NONE) return error;
CommitRecvData(incoming_frame_size);
return GRPC_ERROR_NONE;
}
// we have received a WINDOW_UPDATE frame for a transport // we have received a WINDOW_UPDATE frame for a transport
void RecvUpdate(uint32_t size); void RecvUpdate(uint32_t size) override {
FlowControlTrace trace("t updt recv", this, nullptr);
remote_window_ += size;
}
int64_t target_window() const; // See comment above announced_stream_total_over_incoming_window_ for the
// logic behind this decision.
int64_t target_window() const override {
return static_cast<uint32_t>(
std::min(static_cast<int64_t>((1u << 31) - 1),
announced_stream_total_over_incoming_window_ +
target_initial_window_size_));
}
int64_t target_frame_size() const { return target_frame_size_; } const grpc_chttp2_transport* transport() const { return t_; }
void PreUpdateAnnouncedWindowOverIncomingWindow(int64_t delta) { void PreUpdateAnnouncedWindowOverIncomingWindow(int64_t delta) {
if (delta > 0) { if (delta > 0) {
@ -182,34 +298,28 @@ class TransportFlowControl final {
} }
} }
BdpEstimator* bdp_estimator() { return &bdp_estimator_; } BdpEstimator* bdp_estimator() override { return &bdp_estimator_; }
void TestOnlyForceHugeWindow() { void TestOnlyForceHugeWindow() override {
announced_window_ = 1024 * 1024 * 1024; announced_window_ = 1024 * 1024 * 1024;
remote_window_ = 1024 * 1024 * 1024; remote_window_ = 1024 * 1024 * 1024;
} }
uint32_t acked_init_window() const { return acked_init_window_; }
uint32_t sent_init_window() const { return sent_init_window_; }
void SetSentInitialWindow(uint32_t value) { sent_init_window_ = value; }
void SetAckedInitialWindow(uint32_t value) { acked_init_window_ = value; }
// Getters
int64_t remote_window() const { return remote_window_; }
int64_t announced_window() const { return announced_window_; }
private: private:
double TargetLogBdp(); double TargetLogBdp();
double SmoothLogBdp(double value); double SmoothLogBdp(double value);
static void UpdateSetting(int64_t* desired_value, int64_t new_desired_value, FlowControlAction::Urgency DeltaUrgency(int64_t value,
FlowControlAction* action, grpc_chttp2_setting_id setting_id);
FlowControlAction& (FlowControlAction::*set)(
FlowControlAction::Urgency, uint32_t));
FlowControlAction UpdateAction(FlowControlAction action); FlowControlAction UpdateAction(FlowControlAction action) {
if (announced_window_ < target_window() / 2) {
action.set_send_transport_update(
FlowControlAction::Urgency::UPDATE_IMMEDIATELY);
}
return action;
}
MemoryOwner* const memory_owner_; const grpc_chttp2_transport* const t_;
/** calculating what we should give for local window: /** calculating what we should give for local window:
we track the total amount of flow control over initial window size we track the total amount of flow control over initial window size
@ -230,50 +340,126 @@ class TransportFlowControl final {
/* pid controller */ /* pid controller */
PidController pid_controller_; PidController pid_controller_;
Timestamp last_pid_update_; Timestamp last_pid_update_;
};
int64_t remote_window_ = kDefaultWindow; // Fat interface with all methods a stream flow control implementation needs
int64_t target_initial_window_size_ = kDefaultWindow; // to support.
int64_t target_frame_size_ = kDefaultFrameSize; class StreamFlowControlBase {
int64_t announced_window_ = kDefaultWindow; public:
uint32_t sent_init_window_ = kDefaultWindow; StreamFlowControlBase() {}
uint32_t acked_init_window_ = kDefaultWindow; virtual ~StreamFlowControlBase() {}
// Updates an action using the protected members.
virtual FlowControlAction UpdateAction(FlowControlAction /* action */) {
abort();
}
// Using the protected members, returns an Action for this stream to be
// taken by the tranport.
virtual FlowControlAction MakeAction() = 0;
// Bookkeeping for when data is sent on this stream.
virtual void SentData(int64_t /* outgoing_frame_size */) = 0;
// Bookkeeping and error checking for when data is received by this stream.
virtual grpc_error_handle RecvData(int64_t /* incoming_frame_size */) = 0;
// Called to check if this stream needs to send a WINDOW_UPDATE frame.
virtual uint32_t MaybeSendUpdate() = 0;
// Bookkeeping for receiving a WINDOW_UPDATE from for this stream.
virtual void RecvUpdate(uint32_t /* size */) = 0;
// Bookkeeping for when a call pulls bytes out of the transport. At this
// point we consider the data 'used' and can thus let out peer know we are
// ready for more data.
virtual void IncomingByteStreamUpdate(size_t /* max_size_hint */,
size_t /* have_already */) {
abort();
}
// Used in certain benchmarks in which we don't want FlowControl to be a
// factor
virtual void TestOnlyForceHugeWindow() {}
// Getters
int64_t remote_window_delta() const { return remote_window_delta_; }
int64_t local_window_delta() const { return local_window_delta_; }
int64_t announced_window_delta() const { return announced_window_delta_; }
protected:
friend class grpc::testing::TrickledCHTTP2;
int64_t remote_window_delta_ = 0;
int64_t local_window_delta_ = 0;
int64_t announced_window_delta_ = 0;
};
// Implementation of flow control that does NOTHING. Always returns maximum
// values, never initiates writes, and assumes that the remote peer is doing
// the same. To be used to narrow down on flow control as the cause of negative
// performance.
class StreamFlowControlDisabled : public StreamFlowControlBase {
public:
FlowControlAction UpdateAction(FlowControlAction action) override {
return action;
}
FlowControlAction MakeAction() override { return FlowControlAction(); }
void SentData(int64_t /* outgoing_frame_size */) override {}
grpc_error_handle RecvData(int64_t /* incoming_frame_size */) override {
return GRPC_ERROR_NONE;
}
uint32_t MaybeSendUpdate() override { return 0; }
void RecvUpdate(uint32_t /* size */) override {}
void IncomingByteStreamUpdate(size_t /* max_size_hint */,
size_t /* have_already */) override {}
}; };
// Implementation of flow control that abides to HTTP/2 spec and attempts // Implementation of flow control that abides to HTTP/2 spec and attempts
// to be as performant as possible. // to be as performant as possible.
class StreamFlowControl final { class StreamFlowControl final : public StreamFlowControlBase {
public: public:
explicit StreamFlowControl(TransportFlowControl* tfc); StreamFlowControl(TransportFlowControl* tfc, const grpc_chttp2_stream* s);
~StreamFlowControl() { ~StreamFlowControl() override {
tfc_->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_); tfc_->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_);
} }
FlowControlAction UpdateAction(FlowControlAction action); FlowControlAction UpdateAction(FlowControlAction action) override;
FlowControlAction MakeAction() { return UpdateAction(tfc_->MakeAction()); } FlowControlAction MakeAction() override {
return UpdateAction(tfc_->MakeAction());
}
// we have sent data on the wire, we must track this in our bookkeeping for // we have sent data on the wire, we must track this in our bookkeeping for
// the remote peer's flow control. // the remote peer's flow control.
void SentData(int64_t outgoing_frame_size); void SentData(int64_t outgoing_frame_size) override {
FlowControlTrace tracer(" data sent", tfc_, this);
tfc_->StreamSentData(outgoing_frame_size);
remote_window_delta_ -= outgoing_frame_size;
}
// we have received data from the wire // we have received data from the wire
absl::Status RecvData(int64_t incoming_frame_size); grpc_error_handle RecvData(int64_t incoming_frame_size) override;
// returns an announce if we should send a stream update to our peer, else // returns an announce if we should send a stream update to our peer, else
// returns zero // returns zero
uint32_t MaybeSendUpdate(); uint32_t MaybeSendUpdate() override;
// we have received a WINDOW_UPDATE frame for a stream // we have received a WINDOW_UPDATE frame for a stream
void RecvUpdate(uint32_t size) { remote_window_delta_ += size; } void RecvUpdate(uint32_t size) override {
FlowControlTrace trace("s updt recv", tfc_, this);
remote_window_delta_ += size;
}
// the application is asking for a certain amount of bytes // the application is asking for a certain amount of bytes
void UpdateProgress(uint32_t min_progress_size); void IncomingByteStreamUpdate(size_t max_size_hint,
size_t have_already) override;
int64_t remote_window_delta() const { return remote_window_delta_; } int64_t remote_window_delta() const { return remote_window_delta_; }
int64_t local_window_delta() const { return local_window_delta_; } int64_t local_window_delta() const { return local_window_delta_; }
int64_t announced_window_delta() const { return announced_window_delta_; } int64_t announced_window_delta() const { return announced_window_delta_; }
uint32_t min_progress_size() const { return min_progress_size_; }
void TestOnlyForceHugeWindow() { const grpc_chttp2_stream* stream() const { return s_; }
void TestOnlyForceHugeWindow() override {
announced_window_delta_ = 1024 * 1024 * 1024; announced_window_delta_ = 1024 * 1024 * 1024;
local_window_delta_ = 1024 * 1024 * 1024; local_window_delta_ = 1024 * 1024 * 1024;
remote_window_delta_ = 1024 * 1024 * 1024; remote_window_delta_ = 1024 * 1024 * 1024;
@ -281,12 +467,13 @@ class StreamFlowControl final {
private: private:
TransportFlowControl* const tfc_; TransportFlowControl* const tfc_;
uint32_t min_progress_size_ = 0; const grpc_chttp2_stream* const s_;
int64_t remote_window_delta_ = 0;
int64_t local_window_delta_ = 0;
int64_t announced_window_delta_ = 0;
void UpdateAnnouncedWindowDelta(TransportFlowControl* tfc, int64_t change); void UpdateAnnouncedWindowDelta(TransportFlowControl* tfc, int64_t change) {
tfc->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_);
announced_window_delta_ += change;
tfc->PostUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_);
}
}; };
class TestOnlyTransportTargetWindowEstimatesMocker { class TestOnlyTransportTargetWindowEstimatesMocker {

@ -20,24 +20,42 @@
#include "src/core/ext/transport/chttp2/transport/frame_data.h" #include "src/core/ext/transport/chttp2/transport/frame_data.h"
#include <stdlib.h> #include <string.h>
#include "absl/base/attributes.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include <grpc/slice_buffer.h> #include <grpc/slice_buffer.h>
#include <grpc/support/log.h> #include <grpc/support/log.h>
#include "src/core/ext/transport/chttp2/transport/internal.h" #include "src/core/ext/transport/chttp2/transport/internal.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/channel/channelz.h"
#include "src/core/lib/gpr/string.h"
#include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/memory.h"
#include "src/core/lib/gprpp/ref_counted_ptr.h"
#include "src/core/lib/iomgr/exec_ctx.h"
#include "src/core/lib/slice/slice_internal.h"
#include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/slice/slice_refcount.h"
#include "src/core/lib/slice/slice_string_helpers.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
absl::Status grpc_chttp2_data_parser_begin_frame(uint8_t flags, grpc_chttp2_data_parser::~grpc_chttp2_data_parser() {
uint32_t stream_id, if (parsing_frame != nullptr) {
grpc_chttp2_stream* s) { GRPC_ERROR_UNREF(parsing_frame->Finished(
GRPC_ERROR_CREATE_FROM_STATIC_STRING("Parser destroyed"), false));
}
GRPC_ERROR_UNREF(error);
}
grpc_error_handle grpc_chttp2_data_parser_begin_frame(
grpc_chttp2_data_parser* /*parser*/, uint8_t flags, uint32_t stream_id,
grpc_chttp2_stream* s) {
if (flags & ~GRPC_CHTTP2_DATA_FLAG_END_STREAM) { if (flags & ~GRPC_CHTTP2_DATA_FLAG_END_STREAM) {
return absl::InternalError(absl::StrFormat( return grpc_error_set_int(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrFormat(
"unsupported data flags: 0x%02x stream: %d", flags, stream_id)); "unsupported data flags: 0x%02x", flags)),
GRPC_ERROR_INT_STREAM_ID,
static_cast<intptr_t>(stream_id));
} }
if (flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM) { if (flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM) {
@ -47,7 +65,7 @@ absl::Status grpc_chttp2_data_parser_begin_frame(uint8_t flags,
s->received_last_frame = false; s->received_last_frame = false;
} }
return absl::OkStatus(); return GRPC_ERROR_NONE;
} }
void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer* inbuf, void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer* inbuf,
@ -78,59 +96,188 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer* inbuf,
stats->data_bytes += write_bytes; stats->data_bytes += write_bytes;
} }
grpc_core::Poll<grpc_error_handle> grpc_deframe_unprocessed_incoming_frames( grpc_error_handle grpc_deframe_unprocessed_incoming_frames(
grpc_chttp2_stream* s, uint32_t* min_progress_size, grpc_chttp2_data_parser* p, grpc_chttp2_stream* s,
grpc_core::SliceBuffer* stream_out, uint32_t* message_flags) { grpc_slice_buffer* slices, grpc_slice* slice_out,
grpc_slice_buffer* slices = &s->frame_storage; grpc_core::OrphanablePtr<grpc_core::ByteStream>* stream_out) {
grpc_error_handle error = GRPC_ERROR_NONE; grpc_error_handle error = GRPC_ERROR_NONE;
grpc_chttp2_transport* t = s->t;
if (slices->length < 5) { while (slices->count > 0) {
if (min_progress_size != nullptr) *min_progress_size = 5 - slices->length; uint8_t* beg = nullptr;
return grpc_core::Pending{}; uint8_t* end = nullptr;
} uint8_t* cur = nullptr;
uint8_t header[5];
grpc_slice_buffer_copy_first_into_buffer(slices, 5, header);
switch (header[0]) {
case 0:
if (message_flags != nullptr) *message_flags = 0;
break;
case 1:
if (message_flags != nullptr) {
*message_flags = GRPC_WRITE_INTERNAL_COMPRESS;
}
break;
default:
error = GRPC_ERROR_CREATE_FROM_CPP_STRING(
absl::StrFormat("Bad GRPC frame type 0x%02x", header[0]));
error = grpc_error_set_int(error, GRPC_ERROR_INT_STREAM_ID,
static_cast<intptr_t>(s->id));
return error;
}
uint32_t length = (static_cast<uint32_t>(header[1]) << 24) | grpc_slice* slice = grpc_slice_buffer_peek_first(slices);
(static_cast<uint32_t>(header[2]) << 16) | beg = GRPC_SLICE_START_PTR(*slice);
(static_cast<uint32_t>(header[3]) << 8) | end = GRPC_SLICE_END_PTR(*slice);
static_cast<uint32_t>(header[4]); cur = beg;
uint32_t message_flags;
if (slices->length < length + 5) { if (cur == end) {
if (min_progress_size != nullptr) { grpc_slice_buffer_remove_first(slices);
*min_progress_size = length + 5 - slices->length; continue;
} }
return grpc_core::Pending{};
}
if (min_progress_size != nullptr) *min_progress_size = 0; switch (p->state) {
case GRPC_CHTTP2_DATA_ERROR:
if (stream_out != nullptr) { p->state = GRPC_CHTTP2_DATA_ERROR;
s->stats.incoming.framing_bytes += 5; grpc_slice_buffer_remove_first(slices);
s->stats.incoming.data_bytes += length; return GRPC_ERROR_REF(p->error);
grpc_slice_buffer_move_first_into_buffer(slices, 5, header); case GRPC_CHTTP2_DATA_FH_0:
grpc_slice_buffer_move_first_no_ref(slices, length, s->stats.incoming.framing_bytes++;
stream_out->c_slice_buffer()); p->frame_type = *cur;
switch (p->frame_type) {
case 0:
p->is_frame_compressed = false; /* GPR_FALSE */
break;
case 1:
p->is_frame_compressed = true; /* GPR_TRUE */
break;
default:
p->error = GRPC_ERROR_CREATE_FROM_CPP_STRING(
absl::StrFormat("Bad GRPC frame type 0x%02x", p->frame_type));
p->error = grpc_error_set_int(p->error, GRPC_ERROR_INT_STREAM_ID,
static_cast<intptr_t>(s->id));
grpc_core::UniquePtr<char> dmp(
grpc_dump_slice(*slice, GPR_DUMP_HEX | GPR_DUMP_ASCII));
p->error = grpc_error_set_str(p->error, GRPC_ERROR_STR_RAW_BYTES,
dmp.get());
p->error =
grpc_error_set_int(p->error, GRPC_ERROR_INT_OFFSET, cur - beg);
p->state = GRPC_CHTTP2_DATA_ERROR;
grpc_slice_buffer_remove_first(slices);
return GRPC_ERROR_REF(p->error);
}
if (++cur == end) {
p->state = GRPC_CHTTP2_DATA_FH_1;
grpc_slice_buffer_remove_first(slices);
continue;
}
ABSL_FALLTHROUGH_INTENDED;
case GRPC_CHTTP2_DATA_FH_1:
s->stats.incoming.framing_bytes++;
p->frame_size = (static_cast<uint32_t>(*cur)) << 24;
if (++cur == end) {
p->state = GRPC_CHTTP2_DATA_FH_2;
grpc_slice_buffer_remove_first(slices);
continue;
}
ABSL_FALLTHROUGH_INTENDED;
case GRPC_CHTTP2_DATA_FH_2:
s->stats.incoming.framing_bytes++;
p->frame_size |= (static_cast<uint32_t>(*cur)) << 16;
if (++cur == end) {
p->state = GRPC_CHTTP2_DATA_FH_3;
grpc_slice_buffer_remove_first(slices);
continue;
}
ABSL_FALLTHROUGH_INTENDED;
case GRPC_CHTTP2_DATA_FH_3:
s->stats.incoming.framing_bytes++;
p->frame_size |= (static_cast<uint32_t>(*cur)) << 8;
if (++cur == end) {
p->state = GRPC_CHTTP2_DATA_FH_4;
grpc_slice_buffer_remove_first(slices);
continue;
}
ABSL_FALLTHROUGH_INTENDED;
case GRPC_CHTTP2_DATA_FH_4:
s->stats.incoming.framing_bytes++;
GPR_ASSERT(stream_out != nullptr);
GPR_ASSERT(p->parsing_frame == nullptr);
p->frame_size |= (static_cast<uint32_t>(*cur));
if (t->channelz_socket != nullptr) {
t->channelz_socket->RecordMessageReceived();
}
p->state = GRPC_CHTTP2_DATA_FRAME;
++cur;
message_flags = 0;
if (p->is_frame_compressed) {
message_flags |= GRPC_WRITE_INTERNAL_COMPRESS;
}
p->parsing_frame = new grpc_core::Chttp2IncomingByteStream(
t, s, p->frame_size, message_flags);
stream_out->reset(p->parsing_frame);
if (p->parsing_frame->remaining_bytes() == 0) {
GRPC_ERROR_UNREF(p->parsing_frame->Finished(GRPC_ERROR_NONE, true));
p->parsing_frame = nullptr;
p->state = GRPC_CHTTP2_DATA_FH_0;
}
s->pending_byte_stream = true;
if (cur != end) {
grpc_slice_buffer_sub_first(slices, static_cast<size_t>(cur - beg),
static_cast<size_t>(end - beg));
} else {
grpc_slice_buffer_remove_first(slices);
}
return GRPC_ERROR_NONE;
case GRPC_CHTTP2_DATA_FRAME: {
GPR_ASSERT(p->parsing_frame != nullptr);
GPR_ASSERT(slice_out != nullptr);
if (cur == end) {
grpc_slice_buffer_remove_first(slices);
continue;
}
uint32_t remaining = static_cast<uint32_t>(end - cur);
if (remaining == p->frame_size) {
s->stats.incoming.data_bytes += remaining;
if (GRPC_ERROR_NONE !=
(error = p->parsing_frame->Push(
grpc_slice_sub(*slice, static_cast<size_t>(cur - beg),
static_cast<size_t>(end - beg)),
slice_out))) {
grpc_slice_buffer_remove_first(slices);
return error;
}
if (GRPC_ERROR_NONE !=
(error = p->parsing_frame->Finished(GRPC_ERROR_NONE, true))) {
grpc_slice_buffer_remove_first(slices);
return error;
}
p->parsing_frame = nullptr;
p->state = GRPC_CHTTP2_DATA_FH_0;
grpc_slice_buffer_remove_first(slices);
return GRPC_ERROR_NONE;
} else if (remaining < p->frame_size) {
s->stats.incoming.data_bytes += remaining;
if (GRPC_ERROR_NONE !=
(error = p->parsing_frame->Push(
grpc_slice_sub(*slice, static_cast<size_t>(cur - beg),
static_cast<size_t>(end - beg)),
slice_out))) {
return error;
}
p->frame_size -= remaining;
grpc_slice_buffer_remove_first(slices);
return GRPC_ERROR_NONE;
} else {
GPR_ASSERT(remaining > p->frame_size);
s->stats.incoming.data_bytes += p->frame_size;
if (GRPC_ERROR_NONE !=
p->parsing_frame->Push(
grpc_slice_sub(
*slice, static_cast<size_t>(cur - beg),
static_cast<size_t>(cur + p->frame_size - beg)),
slice_out)) {
grpc_slice_buffer_remove_first(slices);
return error;
}
if (GRPC_ERROR_NONE !=
(error = p->parsing_frame->Finished(GRPC_ERROR_NONE, true))) {
grpc_slice_buffer_remove_first(slices);
return error;
}
p->parsing_frame = nullptr;
p->state = GRPC_CHTTP2_DATA_FH_0;
cur += p->frame_size;
grpc_slice_buffer_sub_first(slices, static_cast<size_t>(cur - beg),
static_cast<size_t>(end - beg));
return GRPC_ERROR_NONE;
}
}
}
} }
return GRPC_ERROR_NONE; return GRPC_ERROR_NONE;
} }
@ -139,9 +286,20 @@ grpc_error_handle grpc_chttp2_data_parser_parse(void* /*parser*/,
grpc_chttp2_stream* s, grpc_chttp2_stream* s,
const grpc_slice& slice, const grpc_slice& slice,
int is_last) { int is_last) {
grpc_slice_ref_internal(slice); if (!s->pending_byte_stream) {
grpc_slice_buffer_add(&s->frame_storage, slice); grpc_slice_ref_internal(slice);
grpc_chttp2_maybe_complete_recv_message(t, s); grpc_slice_buffer_add(&s->frame_storage, slice);
grpc_chttp2_maybe_complete_recv_message(t, s);
} else if (s->on_next) {
GPR_ASSERT(s->frame_storage.length == 0);
grpc_slice_ref_internal(slice);
grpc_slice_buffer_add(&s->unprocessed_incoming_frames_buffer, slice);
grpc_core::ExecCtx::Run(DEBUG_LOCATION, s->on_next, GRPC_ERROR_NONE);
s->on_next = nullptr;
} else {
grpc_slice_ref_internal(slice);
grpc_slice_buffer_add(&s->frame_storage, slice);
}
if (is_last && s->received_last_frame) { if (is_last && s->received_last_frame) {
grpc_chttp2_mark_stream_closed( grpc_chttp2_mark_stream_closed(

@ -25,20 +25,45 @@
#include <stdint.h> #include <stdint.h>
#include "absl/status/status.h"
#include <grpc/slice.h> #include <grpc/slice.h>
#include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/ext/transport/chttp2/transport/frame.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/error.h"
#include "src/core/lib/promise/poll.h" #include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/slice/slice_buffer.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
typedef enum {
GRPC_CHTTP2_DATA_FH_0,
GRPC_CHTTP2_DATA_FH_1,
GRPC_CHTTP2_DATA_FH_2,
GRPC_CHTTP2_DATA_FH_3,
GRPC_CHTTP2_DATA_FH_4,
GRPC_CHTTP2_DATA_FRAME,
GRPC_CHTTP2_DATA_ERROR
} grpc_chttp2_stream_state;
namespace grpc_core {
class Chttp2IncomingByteStream;
} // namespace grpc_core
struct grpc_chttp2_data_parser {
grpc_chttp2_data_parser() = default;
~grpc_chttp2_data_parser();
grpc_chttp2_stream_state state = GRPC_CHTTP2_DATA_FH_0;
uint8_t frame_type = 0;
uint32_t frame_size = 0;
grpc_error_handle error = GRPC_ERROR_NONE;
bool is_frame_compressed = false;
grpc_core::Chttp2IncomingByteStream* parsing_frame = nullptr;
};
/* start processing a new data frame */ /* start processing a new data frame */
absl::Status grpc_chttp2_data_parser_begin_frame(uint8_t flags, grpc_error_handle grpc_chttp2_data_parser_begin_frame(
uint32_t stream_id, grpc_chttp2_data_parser* parser, uint8_t flags, uint32_t stream_id,
grpc_chttp2_stream* s); grpc_chttp2_stream* s);
/* handle a slice of a data frame - is_last indicates the last slice of a /* handle a slice of a data frame - is_last indicates the last slice of a
frame */ frame */
@ -53,8 +78,9 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer* inbuf,
grpc_transport_one_way_stats* stats, grpc_transport_one_way_stats* stats,
grpc_slice_buffer* outbuf); grpc_slice_buffer* outbuf);
grpc_core::Poll<grpc_error_handle> grpc_deframe_unprocessed_incoming_frames( grpc_error_handle grpc_deframe_unprocessed_incoming_frames(
grpc_chttp2_stream* s, uint32_t* min_progress_size, grpc_chttp2_data_parser* p, grpc_chttp2_stream* s,
grpc_core::SliceBuffer* stream_out, uint32_t* message_flags); grpc_slice_buffer* slices, grpc_slice* slice_out,
grpc_core::OrphanablePtr<grpc_core::ByteStream>* stream_out);
#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_DATA_H */ #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_DATA_H */

@ -39,6 +39,7 @@
#include "src/core/lib/debug/trace.h" #include "src/core/lib/debug/trace.h"
#include "src/core/lib/gpr/useful.h" #include "src/core/lib/gpr/useful.h"
#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/manual_constructor.h"
#include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/exec_ctx.h"
static uint8_t* fill_header(uint8_t* out, uint32_t length, uint8_t flags) { static uint8_t* fill_header(uint8_t* out, uint32_t length, uint8_t flags) {
@ -126,7 +127,7 @@ void StreamFlowControlWindowCheck(void* user_data, uint32_t /* key */,
grpc_chttp2_stream* s = static_cast<grpc_chttp2_stream*>(stream); grpc_chttp2_stream* s = static_cast<grpc_chttp2_stream*>(stream);
if ((s->t->settings[GRPC_PEER_SETTINGS] if ((s->t->settings[GRPC_PEER_SETTINGS]
[GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] +
s->t->initial_window_update + s->flow_control.remote_window_delta()) > s->t->initial_window_update + s->flow_control->remote_window_delta()) >
((1u << 31) - 1)) { ((1u << 31) - 1)) {
*error = true; *error = true;
} }
@ -218,6 +219,12 @@ grpc_error_handle grpc_chttp2_settings_parser_parse(void* p,
if (grpc_wire_id_to_setting_id(parser->id, &id)) { if (grpc_wire_id_to_setting_id(parser->id, &id)) {
const grpc_chttp2_setting_parameters* sp = const grpc_chttp2_setting_parameters* sp =
&grpc_chttp2_settings_parameters[id]; &grpc_chttp2_settings_parameters[id];
// If flow control is disabled we skip these.
if (!t->flow_control->flow_control_enabled() &&
(id == GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE ||
id == GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE)) {
continue;
}
if (parser->value < sp->min_value || parser->value > sp->max_value) { if (parser->value < sp->min_value || parser->value > sp->max_value) {
switch (sp->invalid_value_behavior) { switch (sp->invalid_value_behavior) {
case GRPC_CHTTP2_CLAMP_INVALID_VALUE: case GRPC_CHTTP2_CLAMP_INVALID_VALUE:

@ -29,6 +29,7 @@
#include "src/core/ext/transport/chttp2/transport/flow_control.h" #include "src/core/ext/transport/chttp2/transport/flow_control.h"
#include "src/core/ext/transport/chttp2/transport/internal.h" #include "src/core/ext/transport/chttp2/transport/internal.h"
#include "src/core/lib/gprpp/manual_constructor.h"
grpc_slice grpc_chttp2_window_update_create( grpc_slice grpc_chttp2_window_update_create(
uint32_t id, uint32_t window_delta, grpc_transport_one_way_stats* stats) { uint32_t id, uint32_t window_delta, grpc_transport_one_way_stats* stats) {
@ -97,10 +98,10 @@ grpc_error_handle grpc_chttp2_window_update_parser_parse(
if (t->incoming_stream_id != 0) { if (t->incoming_stream_id != 0) {
if (s != nullptr) { if (s != nullptr) {
s->flow_control.RecvUpdate(received_update); s->flow_control->RecvUpdate(received_update);
if (grpc_core::chttp2:: if (grpc_core::chttp2::
g_test_only_transport_flow_control_window_check && g_test_only_transport_flow_control_window_check &&
s->flow_control.remote_window_delta() > s->flow_control->remote_window_delta() >
grpc_core::chttp2::kMaxWindowDelta) { grpc_core::chttp2::kMaxWindowDelta) {
GPR_ASSERT(false); GPR_ASSERT(false);
} }
@ -111,9 +112,9 @@ grpc_error_handle grpc_chttp2_window_update_parser_parse(
} }
} }
} else { } else {
bool was_zero = t->flow_control.remote_window() <= 0; bool was_zero = t->flow_control->remote_window() <= 0;
t->flow_control.RecvUpdate(received_update); t->flow_control->RecvUpdate(received_update);
bool is_zero = t->flow_control.remote_window() <= 0; bool is_zero = t->flow_control->remote_window() <= 0;
if (was_zero && !is_zero) { if (was_zero && !is_zero) {
grpc_chttp2_initiate_write( grpc_chttp2_initiate_write(
t, GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED); t, GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED);

@ -27,7 +27,6 @@
#include <string> #include <string>
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include <grpc/event_engine/memory_allocator.h> #include <grpc/event_engine/memory_allocator.h>
#include <grpc/impl/codegen/grpc_types.h> #include <grpc/impl/codegen/grpc_types.h>
@ -35,6 +34,7 @@
#include "src/core/ext/transport/chttp2/transport/flow_control.h" #include "src/core/ext/transport/chttp2/transport/flow_control.h"
#include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/ext/transport/chttp2/transport/frame.h"
#include "src/core/ext/transport/chttp2/transport/frame_data.h"
#include "src/core/ext/transport/chttp2/transport/frame_goaway.h" #include "src/core/ext/transport/chttp2/transport/frame_goaway.h"
#include "src/core/ext/transport/chttp2/transport/frame_ping.h" #include "src/core/ext/transport/chttp2/transport/frame_ping.h"
#include "src/core/ext/transport/chttp2/transport/frame_rst_stream.h" #include "src/core/ext/transport/chttp2/transport/frame_rst_stream.h"
@ -48,6 +48,8 @@
#include "src/core/lib/debug/trace.h" #include "src/core/lib/debug/trace.h"
#include "src/core/lib/gprpp/bitset.h" #include "src/core/lib/gprpp/bitset.h"
#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/manual_constructor.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted.h"
#include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/ref_counted_ptr.h"
#include "src/core/lib/gprpp/time.h" #include "src/core/lib/gprpp/time.h"
@ -58,7 +60,7 @@
#include "src/core/lib/iomgr/timer.h" #include "src/core/lib/iomgr/timer.h"
#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/arena.h"
#include "src/core/lib/resource_quota/memory_quota.h" #include "src/core/lib/resource_quota/memory_quota.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/connectivity_state.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -228,6 +230,76 @@ typedef struct grpc_chttp2_write_cb {
struct grpc_chttp2_write_cb* next; struct grpc_chttp2_write_cb* next;
} grpc_chttp2_write_cb; } grpc_chttp2_write_cb;
namespace grpc_core {
class Chttp2IncomingByteStream : public ByteStream {
public:
Chttp2IncomingByteStream(grpc_chttp2_transport* transport,
grpc_chttp2_stream* stream, uint32_t frame_size,
uint32_t flags);
void Orphan() override;
bool Next(size_t max_size_hint, grpc_closure* on_complete) override;
grpc_error_handle Pull(grpc_slice* slice) override;
void Shutdown(grpc_error_handle error) override;
// TODO(roth): When I converted this class to C++, I wanted to make it
// inherit from RefCounted or InternallyRefCounted instead of continuing
// to use its own custom ref-counting code. However, that would require
// using multiple inheritance, which sucks in general. And to make matters
// worse, it causes problems with our New<> and Delete<> wrappers.
// Specifically, unless RefCounted is first in the list of parent classes,
// it will see a different value of the address of the object than the one
// we actually allocated, in which case gpr_free() will be called on a
// different address than the one we got from gpr_malloc(), thus causing a
// crash. Given the fragility of depending on that, as well as a desire to
// avoid multiple inheritance in general, I've decided to leave this
// alone for now. We can revisit this once we're able to link against
// libc++, at which point we can eliminate New<> and Delete<> and
// switch to std::shared_ptr<>.
void Ref() { refs_.Ref(); }
void Unref() {
if (GPR_UNLIKELY(refs_.Unref())) {
delete this;
}
}
void PublishError(grpc_error_handle error);
grpc_error_handle Push(const grpc_slice& slice, grpc_slice* slice_out);
grpc_error_handle Finished(grpc_error_handle error, bool reset_on_error);
uint32_t remaining_bytes() const { return remaining_bytes_; }
private:
static void NextLocked(void* arg, grpc_error_handle error_ignored);
static void OrphanLocked(void* arg, grpc_error_handle error_ignored);
grpc_chttp2_transport* transport_; // Immutable.
grpc_chttp2_stream* stream_; // Immutable.
RefCount refs_;
/* Accessed only by transport thread when stream->pending_byte_stream == false
* Accessed only by application thread when stream->pending_byte_stream ==
* true */
uint32_t remaining_bytes_;
/* Accessed only by transport thread when stream->pending_byte_stream == false
* Accessed only by application thread when stream->pending_byte_stream ==
* true */
struct {
grpc_closure closure;
size_t max_size_hint;
grpc_closure* on_complete;
} next_action_;
grpc_closure destroy_action_;
};
} // namespace grpc_core
typedef enum { typedef enum {
GRPC_CHTTP2_KEEPALIVE_STATE_WAITING, GRPC_CHTTP2_KEEPALIVE_STATE_WAITING,
GRPC_CHTTP2_KEEPALIVE_STATE_PINGING, GRPC_CHTTP2_KEEPALIVE_STATE_PINGING,
@ -356,7 +428,11 @@ struct grpc_chttp2_transport {
/** parser for goaway frames */ /** parser for goaway frames */
grpc_chttp2_goaway_parser goaway_parser; grpc_chttp2_goaway_parser goaway_parser;
grpc_core::chttp2::TransportFlowControl flow_control; grpc_core::PolymorphicManualConstructor<
grpc_core::chttp2::TransportFlowControlBase,
grpc_core::chttp2::TransportFlowControl,
grpc_core::chttp2::TransportFlowControlDisabled>
flow_control;
/** initial window change. This is tracked as we parse settings frames from /** initial window change. This is tracked as we parse settings frames from
* the remote peer. If there is a positive delta, then we will make all * the remote peer. If there is a positive delta, then we will make all
* streams readable since they may have become unstalled */ * streams readable since they may have become unstalled */
@ -490,16 +566,19 @@ struct grpc_chttp2_stream {
bool* sent_trailing_metadata_op = nullptr; bool* sent_trailing_metadata_op = nullptr;
grpc_closure* send_trailing_metadata_finished = nullptr; grpc_closure* send_trailing_metadata_finished = nullptr;
grpc_core::OrphanablePtr<grpc_core::ByteStream> fetching_send_message;
uint32_t fetched_send_message_length = 0;
grpc_slice fetching_slice = grpc_empty_slice();
int64_t next_message_end_offset; int64_t next_message_end_offset;
int64_t flow_controlled_bytes_written = 0; int64_t flow_controlled_bytes_written = 0;
int64_t flow_controlled_bytes_flowed = 0; int64_t flow_controlled_bytes_flowed = 0;
grpc_closure* send_message_finished = nullptr; grpc_closure complete_fetch_locked;
grpc_closure* fetching_send_message_finished = nullptr;
grpc_metadata_batch* recv_initial_metadata; grpc_metadata_batch* recv_initial_metadata;
grpc_closure* recv_initial_metadata_ready = nullptr; grpc_closure* recv_initial_metadata_ready = nullptr;
bool* trailing_metadata_available = nullptr; bool* trailing_metadata_available = nullptr;
absl::optional<grpc_core::SliceBuffer>* recv_message = nullptr; grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message = nullptr;
uint32_t* recv_message_flags = nullptr;
bool* call_failed_before_recv_message = nullptr; bool* call_failed_before_recv_message = nullptr;
grpc_closure* recv_message_ready = nullptr; grpc_closure* recv_message_ready = nullptr;
grpc_metadata_batch* recv_trailing_metadata; grpc_metadata_batch* recv_trailing_metadata;
@ -536,7 +615,22 @@ struct grpc_chttp2_stream {
grpc_metadata_batch initial_metadata_buffer; grpc_metadata_batch initial_metadata_buffer;
grpc_metadata_batch trailing_metadata_buffer; grpc_metadata_batch trailing_metadata_buffer;
grpc_slice_buffer frame_storage; /* protected by t combiner */ grpc_slice_buffer frame_storage; /* protected by t combiner */
grpc_closure* on_next = nullptr; /* protected by t combiner */
bool pending_byte_stream = false; /* protected by t combiner */
// cached length of buffer to be used by the transport thread in cases where
// stream->pending_byte_stream == true. The value is saved before
// application threads are allowed to modify
// unprocessed_incoming_frames_buffer
size_t unprocessed_incoming_frames_buffer_cached_length = 0;
/* Accessed only by transport thread when stream->pending_byte_stream == false
* Accessed only by application thread when stream->pending_byte_stream ==
* true */
grpc_slice_buffer unprocessed_incoming_frames_buffer;
grpc_closure reset_byte_stream;
grpc_error_handle byte_stream_error =
GRPC_ERROR_NONE; /* protected by t combiner */
bool received_last_frame = false; /* protected by t combiner */ bool received_last_frame = false; /* protected by t combiner */
grpc_core::Timestamp deadline = grpc_core::Timestamp::InfFuture(); grpc_core::Timestamp deadline = grpc_core::Timestamp::InfFuture();
@ -545,13 +639,22 @@ struct grpc_chttp2_stream {
grpc_error_handle forced_close_error = GRPC_ERROR_NONE; grpc_error_handle forced_close_error = GRPC_ERROR_NONE;
/** how many header frames have we received? */ /** how many header frames have we received? */
uint8_t header_frames_received = 0; uint8_t header_frames_received = 0;
/** parsing state for data frames */
/* Accessed only by transport thread when stream->pending_byte_stream == false
* Accessed only by application thread when stream->pending_byte_stream ==
* true */
grpc_chttp2_data_parser data_parser;
/** number of bytes received - reset at end of parse thread execution */ /** number of bytes received - reset at end of parse thread execution */
int64_t received_bytes = 0; int64_t received_bytes = 0;
bool sent_initial_metadata = false; bool sent_initial_metadata = false;
bool sent_trailing_metadata = false; bool sent_trailing_metadata = false;
grpc_core::chttp2::StreamFlowControl flow_control; grpc_core::PolymorphicManualConstructor<
grpc_core::chttp2::StreamFlowControlBase,
grpc_core::chttp2::StreamFlowControl,
grpc_core::chttp2::StreamFlowControlDisabled>
flow_control;
grpc_slice_buffer flow_controlled_buffer; grpc_slice_buffer flow_controlled_buffer;

@ -24,7 +24,6 @@
#include <string> #include <string>
#include "absl/base/attributes.h" #include "absl/base/attributes.h"
#include "absl/status/status.h"
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
@ -47,11 +46,11 @@
#include "src/core/ext/transport/chttp2/transport/stream_map.h" #include "src/core/ext/transport/chttp2/transport/stream_map.h"
#include "src/core/lib/channel/channelz.h" #include "src/core/lib/channel/channelz.h"
#include "src/core/lib/debug/trace.h" #include "src/core/lib/debug/trace.h"
#include "src/core/lib/gprpp/manual_constructor.h"
#include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/ref_counted_ptr.h"
#include "src/core/lib/gprpp/time.h" #include "src/core/lib/gprpp/time.h"
#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/error.h"
#include "src/core/lib/transport/bdp_estimator.h" #include "src/core/lib/transport/bdp_estimator.h"
#include "src/core/lib/transport/error_utils.h"
#include "src/core/lib/transport/http2_errors.h" #include "src/core/lib/transport/http2_errors.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -218,9 +217,10 @@ grpc_error_handle grpc_chttp2_perform_read(grpc_chttp2_transport* t,
return GRPC_ERROR_NONE; return GRPC_ERROR_NONE;
} }
goto dts_fh_0; /* loop */ goto dts_fh_0; /* loop */
} else if (t->incoming_frame_size > } else if (t->flow_control->flow_control_enabled() &&
t->settings[GRPC_ACKED_SETTINGS] t->incoming_frame_size >
[GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) { t->settings[GRPC_ACKED_SETTINGS]
[GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) {
return GRPC_ERROR_CREATE_FROM_CPP_STRING( return GRPC_ERROR_CREATE_FROM_CPP_STRING(
absl::StrFormat("Frame size %d is larger than max frame size %d", absl::StrFormat("Frame size %d is larger than max frame size %d",
t->incoming_frame_size, t->incoming_frame_size,
@ -384,7 +384,7 @@ void grpc_chttp2_parsing_become_skip_parser(grpc_chttp2_transport* t) {
static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) { static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) {
// Update BDP accounting since we have received a data frame. // Update BDP accounting since we have received a data frame.
grpc_core::BdpEstimator* bdp_est = t->flow_control.bdp_estimator(); grpc_core::BdpEstimator* bdp_est = t->flow_control->bdp_estimator();
if (bdp_est) { if (bdp_est) {
if (t->bdp_ping_blocked) { if (t->bdp_ping_blocked) {
t->bdp_ping_blocked = false; t->bdp_ping_blocked = false;
@ -395,17 +395,17 @@ static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) {
} }
grpc_chttp2_stream* s = grpc_chttp2_stream* s =
grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id);
absl::Status status; grpc_error_handle err = GRPC_ERROR_NONE;
grpc_core::chttp2::FlowControlAction action; grpc_core::chttp2::FlowControlAction action;
if (s == nullptr) { if (s == nullptr) {
status = t->flow_control.RecvData(t->incoming_frame_size); err = t->flow_control->RecvData(t->incoming_frame_size);
action = t->flow_control.MakeAction(); action = t->flow_control->MakeAction();
} else { } else {
status = s->flow_control.RecvData(t->incoming_frame_size); err = s->flow_control->RecvData(t->incoming_frame_size);
action = s->flow_control.MakeAction(); action = s->flow_control->MakeAction();
} }
grpc_chttp2_act_on_flowctl_action(action, t, s); grpc_chttp2_act_on_flowctl_action(action, t, s);
if (!status.ok()) { if (err != GRPC_ERROR_NONE) {
goto error_handler; goto error_handler;
} }
if (s == nullptr) { if (s == nullptr) {
@ -413,29 +413,33 @@ static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) {
} }
s->received_bytes += t->incoming_frame_size; s->received_bytes += t->incoming_frame_size;
s->stats.incoming.framing_bytes += 9; s->stats.incoming.framing_bytes += 9;
if (s->read_closed) { if (err == GRPC_ERROR_NONE && s->read_closed) {
return init_non_header_skip_frame_parser(t); return init_non_header_skip_frame_parser(t);
} }
status = if (err == GRPC_ERROR_NONE) {
grpc_chttp2_data_parser_begin_frame(t->incoming_frame_flags, s->id, s); err = grpc_chttp2_data_parser_begin_frame(
&s->data_parser, t->incoming_frame_flags, s->id, s);
}
error_handler: error_handler:
if (status.ok()) { intptr_t unused;
if (err == GRPC_ERROR_NONE) {
t->incoming_stream = s; t->incoming_stream = s;
/* t->parser = grpc_chttp2_data_parser_parse;*/ /* t->parser = grpc_chttp2_data_parser_parse;*/
t->parser = grpc_chttp2_data_parser_parse; t->parser = grpc_chttp2_data_parser_parse;
t->parser_data = nullptr; t->parser_data = &s->data_parser;
t->ping_state.last_ping_sent_time = grpc_core::Timestamp::InfPast(); t->ping_state.last_ping_sent_time = grpc_core::Timestamp::InfPast();
return GRPC_ERROR_NONE; return GRPC_ERROR_NONE;
} else if (s != nullptr) { } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, &unused)) {
/* handle stream errors by closing the stream */ /* handle stream errors by closing the stream */
grpc_chttp2_mark_stream_closed(t, s, true, false, if (s != nullptr) {
absl_status_to_grpc_error(status)); grpc_chttp2_mark_stream_closed(t, s, true, false, err);
}
grpc_chttp2_add_rst_stream_to_next_write(t, t->incoming_stream_id, grpc_chttp2_add_rst_stream_to_next_write(t, t->incoming_stream_id,
GRPC_HTTP2_PROTOCOL_ERROR, GRPC_HTTP2_PROTOCOL_ERROR,
&s->stats.outgoing); &s->stats.outgoing);
return init_non_header_skip_frame_parser(t); return init_non_header_skip_frame_parser(t);
} else { } else {
return absl_status_to_grpc_error(status); return err;
} }
} }
@ -564,10 +568,6 @@ static grpc_error_handle init_header_frame_parser(grpc_chttp2_transport* t,
gpr_log(GPR_ERROR, "too many header frames received"); gpr_log(GPR_ERROR, "too many header frames received");
return init_header_skip_frame_parser(t, priority_type); return init_header_skip_frame_parser(t, priority_type);
} }
if (frame_type == HPackParser::LogInfo::kTrailers && !t->header_eof) {
return GRPC_ERROR_CREATE_FROM_STATIC_STRING(
"Trailing metadata frame received without an end-o-stream");
}
t->hpack_parser.BeginFrame( t->hpack_parser.BeginFrame(
incoming_metadata_buffer, incoming_metadata_buffer,
t->settings[GRPC_ACKED_SETTINGS] t->settings[GRPC_ACKED_SETTINGS]
@ -647,10 +647,6 @@ static grpc_error_handle init_settings_frame_parser(grpc_chttp2_transport* t) {
t->hpack_parser.hpack_table()->SetMaxBytes( t->hpack_parser.hpack_table()->SetMaxBytes(
t->settings[GRPC_ACKED_SETTINGS] t->settings[GRPC_ACKED_SETTINGS]
[GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]); [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]);
t->flow_control.SetAckedInitialWindow(
t->settings[GRPC_ACKED_SETTINGS]
[GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]);
grpc_chttp2_act_on_flowctl_action(t->flow_control.MakeAction(), t, nullptr);
t->sent_local_settings = false; t->sent_local_settings = false;
} }
t->parser = grpc_chttp2_settings_parser_parse; t->parser = grpc_chttp2_settings_parser_parse;

@ -20,10 +20,12 @@
#include <grpc/support/log.h> #include <grpc/support/log.h>
#include "src/core/ext/transport/chttp2/transport/flow_control.h"
#include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/ext/transport/chttp2/transport/frame.h"
#include "src/core/ext/transport/chttp2/transport/internal.h" #include "src/core/ext/transport/chttp2/transport/internal.h"
#include "src/core/lib/debug/trace.h" #include "src/core/lib/debug/trace.h"
#include "src/core/lib/gprpp/bitset.h" #include "src/core/lib/gprpp/bitset.h"
#include "src/core/lib/gprpp/manual_constructor.h"
static const char* stream_list_id_string(grpc_chttp2_stream_list_id id) { static const char* stream_list_id_string(grpc_chttp2_stream_list_id id) {
switch (id) { switch (id) {
@ -187,6 +189,7 @@ void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport* t,
void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport* t, void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport* t,
grpc_chttp2_stream* s) { grpc_chttp2_stream* s) {
GPR_ASSERT(t->flow_control->flow_control_enabled());
stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);
} }
@ -202,6 +205,7 @@ void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport* t,
void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport* t, void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport* t,
grpc_chttp2_stream* s) { grpc_chttp2_stream* s) {
GPR_ASSERT(t->flow_control->flow_control_enabled());
stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM);
} }

@ -22,6 +22,7 @@
#include <stddef.h> #include <stddef.h>
#include <algorithm> #include <algorithm>
#include <memory>
#include <string> #include <string>
#include "absl/types/optional.h" #include "absl/types/optional.h"
@ -49,6 +50,7 @@
#include "src/core/lib/debug/stats.h" #include "src/core/lib/debug/stats.h"
#include "src/core/lib/debug/trace.h" #include "src/core/lib/debug/trace.h"
#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/manual_constructor.h"
#include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted.h"
#include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/ref_counted_ptr.h"
#include "src/core/lib/gprpp/time.h" #include "src/core/lib/gprpp/time.h"
@ -218,14 +220,14 @@ static void report_stall(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
s->flow_controlled_buffer.length, s->flow_controlled_bytes_flowed, s->flow_controlled_buffer.length, s->flow_controlled_bytes_flowed,
t->settings[GRPC_ACKED_SETTINGS] t->settings[GRPC_ACKED_SETTINGS]
[GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE],
t->flow_control.remote_window(), t->flow_control->remote_window(),
static_cast<uint32_t>(std::max( static_cast<uint32_t>(std::max(
int64_t(0), int64_t(0),
s->flow_control.remote_window_delta() + s->flow_control->remote_window_delta() +
static_cast<int64_t>( static_cast<int64_t>(
t->settings[GRPC_PEER_SETTINGS] t->settings[GRPC_PEER_SETTINGS]
[GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]))), [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]))),
s->flow_control.remote_window_delta()); s->flow_control->remote_window_delta());
} }
} }
@ -281,11 +283,6 @@ class WriteContext {
void FlushSettings() { void FlushSettings() {
if (t_->dirtied_local_settings && !t_->sent_local_settings) { if (t_->dirtied_local_settings && !t_->sent_local_settings) {
t_->flow_control.SetSentInitialWindow(
t_->settings[GRPC_LOCAL_SETTINGS]
[GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]);
grpc_chttp2_act_on_flowctl_action(t_->flow_control.MakeAction(), t_,
nullptr);
grpc_slice_buffer_add( grpc_slice_buffer_add(
&t_->outbuf, grpc_chttp2_settings_create( &t_->outbuf, grpc_chttp2_settings_create(
t_->settings[GRPC_SENT_SETTINGS], t_->settings[GRPC_SENT_SETTINGS],
@ -307,7 +304,7 @@ class WriteContext {
void FlushWindowUpdates() { void FlushWindowUpdates() {
uint32_t transport_announce = uint32_t transport_announce =
t_->flow_control.MaybeSendUpdate(t_->outbuf.count > 0); t_->flow_control->MaybeSendUpdate(t_->outbuf.count > 0);
if (transport_announce) { if (transport_announce) {
grpc_transport_one_way_stats throwaway_stats; grpc_transport_one_way_stats throwaway_stats;
grpc_slice_buffer_add( grpc_slice_buffer_add(
@ -395,7 +392,7 @@ class DataSendContext {
uint32_t stream_remote_window() const { uint32_t stream_remote_window() const {
return static_cast<uint32_t>(std::max( return static_cast<uint32_t>(std::max(
int64_t(0), int64_t(0),
s_->flow_control.remote_window_delta() + s_->flow_control->remote_window_delta() +
static_cast<int64_t>( static_cast<int64_t>(
t_->settings[GRPC_PEER_SETTINGS] t_->settings[GRPC_PEER_SETTINGS]
[GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]))); [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE])));
@ -405,7 +402,7 @@ class DataSendContext {
return static_cast<uint32_t>(std::min( return static_cast<uint32_t>(std::min(
t_->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], t_->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE],
static_cast<uint32_t>(std::min(int64_t(stream_remote_window()), static_cast<uint32_t>(std::min(int64_t(stream_remote_window()),
t_->flow_control.remote_window())))); t_->flow_control->remote_window()))));
} }
bool AnyOutgoing() const { return max_outgoing() > 0; } bool AnyOutgoing() const { return max_outgoing() > 0; }
@ -414,11 +411,12 @@ class DataSendContext {
uint32_t send_bytes = static_cast<uint32_t>( uint32_t send_bytes = static_cast<uint32_t>(
std::min(size_t(max_outgoing()), s_->flow_controlled_buffer.length)); std::min(size_t(max_outgoing()), s_->flow_controlled_buffer.length));
is_last_frame_ = send_bytes == s_->flow_controlled_buffer.length && is_last_frame_ = send_bytes == s_->flow_controlled_buffer.length &&
s_->fetching_send_message == nullptr &&
s_->send_trailing_metadata != nullptr && s_->send_trailing_metadata != nullptr &&
s_->send_trailing_metadata->empty(); s_->send_trailing_metadata->empty();
grpc_chttp2_encode_data(s_->id, &s_->flow_controlled_buffer, send_bytes, grpc_chttp2_encode_data(s_->id, &s_->flow_controlled_buffer, send_bytes,
is_last_frame_, &s_->stats.outgoing, &t_->outbuf); is_last_frame_, &s_->stats.outgoing, &t_->outbuf);
s_->flow_control.SentData(send_bytes); s_->flow_control->SentData(send_bytes);
s_->sending_bytes += send_bytes; s_->sending_bytes += send_bytes;
} }
@ -450,8 +448,8 @@ class StreamWriteContext {
gpr_log(GPR_INFO, "W:%p %s[%d] im-(sent,send)=(%d,%d) announce=%d", t_, gpr_log(GPR_INFO, "W:%p %s[%d] im-(sent,send)=(%d,%d) announce=%d", t_,
t_->is_client ? "CLIENT" : "SERVER", s->id, t_->is_client ? "CLIENT" : "SERVER", s->id,
s->sent_initial_metadata, s->send_initial_metadata != nullptr, s->sent_initial_metadata, s->send_initial_metadata != nullptr,
(int)(s->flow_control.local_window_delta() - (int)(s->flow_control->local_window_delta() -
s->flow_control.announced_window_delta()))); s->flow_control->announced_window_delta())));
} }
void FlushInitialMetadata() { void FlushInitialMetadata() {
@ -464,7 +462,8 @@ class StreamWriteContext {
// trailing metadata. This results in a Trailers-Only response, // trailing metadata. This results in a Trailers-Only response,
// which is required for retries, as per: // which is required for retries, as per:
// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#when-retries-are-valid
if (!t_->is_client && s_->flow_controlled_buffer.length == 0 && if (!t_->is_client && s_->fetching_send_message == nullptr &&
s_->flow_controlled_buffer.length == 0 &&
s_->send_trailing_metadata != nullptr && s_->send_trailing_metadata != nullptr &&
is_default_initial_metadata(s_->send_initial_metadata)) { is_default_initial_metadata(s_->send_initial_metadata)) {
ConvertInitialMetadataToTrailingMetadata(); ConvertInitialMetadataToTrailingMetadata();
@ -497,7 +496,7 @@ class StreamWriteContext {
void FlushWindowUpdates() { void FlushWindowUpdates() {
/* send any window updates */ /* send any window updates */
const uint32_t stream_announce = s_->flow_control.MaybeSendUpdate(); const uint32_t stream_announce = s_->flow_control->MaybeSendUpdate();
if (stream_announce == 0) return; if (stream_announce == 0) return;
grpc_slice_buffer_add( grpc_slice_buffer_add(
@ -517,7 +516,7 @@ class StreamWriteContext {
DataSendContext data_send_context(write_context_, t_, s_); DataSendContext data_send_context(write_context_, t_, s_);
if (!data_send_context.AnyOutgoing()) { if (!data_send_context.AnyOutgoing()) {
if (t_->flow_control.remote_window() <= 0) { if (t_->flow_control->remote_window() <= 0) {
report_stall(t_, s_, "transport"); report_stall(t_, s_, "transport");
grpc_chttp2_list_add_stalled_by_transport(t_, s_); grpc_chttp2_list_add_stalled_by_transport(t_, s_);
} else if (data_send_context.stream_remote_window() <= 0) { } else if (data_send_context.stream_remote_window() <= 0) {
@ -548,6 +547,7 @@ class StreamWriteContext {
if (!s_->sent_initial_metadata) return; if (!s_->sent_initial_metadata) return;
if (s_->send_trailing_metadata == nullptr) return; if (s_->send_trailing_metadata == nullptr) return;
if (s_->fetching_send_message != nullptr) return;
if (s_->flow_controlled_buffer.length != 0) return; if (s_->flow_controlled_buffer.length != 0) return;
GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "sending trailing_metadata")); GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "sending trailing_metadata"));
@ -635,7 +635,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write(
ctx.FlushQueuedBuffers(); ctx.FlushQueuedBuffers();
ctx.EnactHpackSettings(); ctx.EnactHpackSettings();
if (t->flow_control.remote_window() > 0) { if (t->flow_control->remote_window() > 0) {
ctx.UpdateStreamsNoLongerStalled(); ctx.UpdateStreamsNoLongerStalled();
} }

@ -24,6 +24,7 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <memory>
#include <new> #include <new>
#include <string> #include <string>
#include <utility> #include <utility>
@ -32,10 +33,10 @@
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "third_party/objective_c/Cronet/bidirectional_stream_c.h" #include "third_party/objective_c/Cronet/bidirectional_stream_c.h"
#include <grpc/slice.h> #include <grpc/slice.h>
#include <grpc/slice_buffer.h>
#include <grpc/status.h> #include <grpc/status.h>
#include <grpc/support/alloc.h> #include <grpc/support/alloc.h>
#include <grpc/support/log.h> #include <grpc/support/log.h>
@ -46,6 +47,8 @@
#include "src/core/ext/transport/cronet/transport/cronet_status.h" #include "src/core/ext/transport/cronet/transport/cronet_status.h"
#include "src/core/lib/debug/trace.h" #include "src/core/lib/debug/trace.h"
#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/manual_constructor.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/closure.h"
#include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/endpoint.h"
#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/error.h"
@ -54,15 +57,14 @@
#include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset.h"
#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/arena.h"
#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_internal.h"
#include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/slice/slice_refcount.h"
#include "src/core/lib/surface/validate_metadata.h" #include "src/core/lib/surface/validate_metadata.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
#include "src/core/lib/transport/transport_impl.h" #include "src/core/lib/transport/transport_impl.h"
// IWYU pragma: no_include <type_traits>
#define GRPC_HEADER_SIZE_IN_BYTES 5 #define GRPC_HEADER_SIZE_IN_BYTES 5
#define GRPC_FLUSH_READ_SIZE 4096 #define GRPC_FLUSH_READ_SIZE 4096
@ -131,7 +133,9 @@ typedef struct grpc_cronet_transport grpc_cronet_transport;
http://www.catb.org/esr/structure-packing/#_structure_reordering: */ http://www.catb.org/esr/structure-packing/#_structure_reordering: */
struct read_state { struct read_state {
explicit read_state(grpc_core::Arena* arena) explicit read_state(grpc_core::Arena* arena)
: trailing_metadata(arena), initial_metadata(arena) {} : trailing_metadata(arena), initial_metadata(arena) {
grpc_slice_buffer_init(&read_slice_buffer);
}
/* vars to store data coming from server */ /* vars to store data coming from server */
char* read_buffer = nullptr; char* read_buffer = nullptr;
@ -145,7 +149,8 @@ struct read_state {
bool read_stream_closed = false; bool read_stream_closed = false;
/* vars for holding data destined for the application */ /* vars for holding data destined for the application */
grpc_core::SliceBuffer read_slice_buffer; grpc_core::ManualConstructor<grpc_core::SliceBufferByteStream> sbs;
grpc_slice_buffer read_slice_buffer;
/* vars for trailing metadata */ /* vars for trailing metadata */
grpc_metadata_batch trailing_metadata; grpc_metadata_batch trailing_metadata;
@ -1083,17 +1088,38 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) {
result = NO_ACTION_POSSIBLE; result = NO_ACTION_POSSIBLE;
CRONET_LOG(GPR_DEBUG, "Stream is either cancelled, failed or finished"); CRONET_LOG(GPR_DEBUG, "Stream is either cancelled, failed or finished");
} else { } else {
grpc_slice_buffer write_slice_buffer;
grpc_slice slice;
grpc_slice_buffer_init(&write_slice_buffer);
while (write_slice_buffer.length <
stream_op->payload->send_message.send_message->length()) {
/* TODO(roth): When we add support for incremental sending,this code
* will need to be changed to support asynchronous delivery of the
* send_message payload. */
if (!stream_op->payload->send_message.send_message->Next(
stream_op->payload->send_message.send_message->length(),
nullptr)) {
/* Should never reach here */
GPR_ASSERT(false);
}
if (GRPC_ERROR_NONE !=
stream_op->payload->send_message.send_message->Pull(&slice)) {
/* Should never reach here */
GPR_ASSERT(false);
}
grpc_slice_buffer_add(&write_slice_buffer, slice);
}
size_t write_buffer_size; size_t write_buffer_size;
create_grpc_frame( create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer,
stream_op->payload->send_message.send_message->c_slice_buffer(), &write_buffer_size,
&stream_state->ws.write_buffer, &write_buffer_size, stream_op->payload->send_message.send_message->flags());
stream_op->payload->send_message.flags);
if (write_buffer_size > 0) { if (write_buffer_size > 0) {
CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, %p)", s->cbs, CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, %p)", s->cbs,
stream_state->ws.write_buffer); stream_state->ws.write_buffer);
stream_state->state_callback_received[OP_SEND_MESSAGE] = false; stream_state->state_callback_received[OP_SEND_MESSAGE] = false;
bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer,
static_cast<int>(write_buffer_size), false); static_cast<int>(write_buffer_size), false);
grpc_slice_buffer_destroy_internal(&write_slice_buffer);
if (t->use_packet_coalescing) { if (t->use_packet_coalescing) {
if (!stream_op->send_trailing_metadata) { if (!stream_op->send_trailing_metadata) {
CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs); CRONET_LOG(GPR_DEBUG, "bidirectional_stream_flush (%p)", s->cbs);
@ -1113,6 +1139,7 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) {
} }
stream_state->state_op_done[OP_SEND_MESSAGE] = true; stream_state->state_op_done[OP_SEND_MESSAGE] = true;
oas->state.state_op_done[OP_SEND_MESSAGE] = true; oas->state.state_op_done[OP_SEND_MESSAGE] = true;
stream_op->payload->send_message.send_message.reset();
} else if (stream_op->send_trailing_metadata && } else if (stream_op->send_trailing_metadata &&
op_can_be_run(stream_op, s, &oas->state, op_can_be_run(stream_op, s, &oas->state,
OP_SEND_TRAILING_METADATA)) { OP_SEND_TRAILING_METADATA)) {
@ -1224,14 +1251,16 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) {
stream_state->rs.remaining_bytes = 0; stream_state->rs.remaining_bytes = 0;
CRONET_LOG(GPR_DEBUG, "read operation complete. Empty response."); CRONET_LOG(GPR_DEBUG, "read operation complete. Empty response.");
/* Clean up read_slice_buffer in case there is unread data. */ /* Clean up read_slice_buffer in case there is unread data. */
stream_state->rs.read_slice_buffer.Clear(); grpc_slice_buffer_destroy_internal(
&stream_state->rs.read_slice_buffer);
grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer);
uint32_t flags = 0; uint32_t flags = 0;
if (stream_state->rs.compressed) { if (stream_state->rs.compressed) {
flags |= GRPC_WRITE_INTERNAL_COMPRESS; flags |= GRPC_WRITE_INTERNAL_COMPRESS;
} }
*stream_op->payload->recv_message.flags = flags; stream_state->rs.sbs.Init(&stream_state->rs.read_slice_buffer, flags);
*stream_op->payload->recv_message.recv_message = stream_op->payload->recv_message.recv_message->reset(
std::move(stream_state->rs.read_slice_buffer); stream_state->rs.sbs.get());
grpc_core::ExecCtx::Run( grpc_core::ExecCtx::Run(
DEBUG_LOCATION, DEBUG_LOCATION,
stream_op->payload->recv_message.recv_message_ready, stream_op->payload->recv_message.recv_message_ready,
@ -1270,16 +1299,17 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) {
static_cast<size_t>(stream_state->rs.length_field)); static_cast<size_t>(stream_state->rs.length_field));
null_and_maybe_free_read_buffer(s); null_and_maybe_free_read_buffer(s);
/* Clean up read_slice_buffer in case there is unread data. */ /* Clean up read_slice_buffer in case there is unread data. */
stream_state->rs.read_slice_buffer.Clear(); grpc_slice_buffer_destroy_internal(&stream_state->rs.read_slice_buffer);
stream_state->rs.read_slice_buffer.Append( grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer);
grpc_core::Slice(read_data_slice)); grpc_slice_buffer_add(&stream_state->rs.read_slice_buffer,
read_data_slice);
uint32_t flags = 0; uint32_t flags = 0;
if (stream_state->rs.compressed) { if (stream_state->rs.compressed) {
flags = GRPC_WRITE_INTERNAL_COMPRESS; flags = GRPC_WRITE_INTERNAL_COMPRESS;
} }
*stream_op->payload->recv_message.flags = flags; stream_state->rs.sbs.Init(&stream_state->rs.read_slice_buffer, flags);
*stream_op->payload->recv_message.recv_message = stream_op->payload->recv_message.recv_message->reset(
std::move(stream_state->rs.read_slice_buffer); stream_state->rs.sbs.get());
grpc_core::ExecCtx::Run( grpc_core::ExecCtx::Run(
DEBUG_LOCATION, stream_op->payload->recv_message.recv_message_ready, DEBUG_LOCATION, stream_op->payload->recv_message.recv_message_ready,
GRPC_ERROR_NONE); GRPC_ERROR_NONE);
@ -1395,6 +1425,8 @@ inline stream_obj::stream_obj(grpc_transport* gt, grpc_stream* gs,
inline stream_obj::~stream_obj() { inline stream_obj::~stream_obj() {
null_and_maybe_free_read_buffer(this); null_and_maybe_free_read_buffer(this);
/* Clean up read_slice_buffer in case there is unread data. */
grpc_slice_buffer_destroy_internal(&state.rs.read_slice_buffer);
GRPC_ERROR_UNREF(state.cancel_error); GRPC_ERROR_UNREF(state.cancel_error);
} }

@ -21,6 +21,7 @@
#include "src/core/ext/transport/inproc/inproc_transport.h" #include "src/core/ext/transport/inproc/inproc_transport.h"
#include <stdint.h> #include <stdint.h>
#include <string.h>
#include <algorithm> #include <algorithm>
#include <memory> #include <memory>
@ -33,10 +34,11 @@
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "absl/types/optional.h" #include "absl/types/optional.h"
#include "absl/utility/utility.h"
#include <grpc/grpc.h> #include <grpc/grpc.h>
#include <grpc/impl/codegen/connectivity_state.h> #include <grpc/impl/codegen/connectivity_state.h>
#include <grpc/slice.h>
#include <grpc/slice_buffer.h>
#include <grpc/status.h> #include <grpc/status.h>
#include <grpc/support/alloc.h> #include <grpc/support/alloc.h>
#include <grpc/support/log.h> #include <grpc/support/log.h>
@ -48,6 +50,8 @@
#include "src/core/lib/config/core_configuration.h" #include "src/core/lib/config/core_configuration.h"
#include "src/core/lib/gpr/useful.h" #include "src/core/lib/gpr/useful.h"
#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/manual_constructor.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/ref_counted_ptr.h"
#include "src/core/lib/gprpp/time.h" #include "src/core/lib/gprpp/time.h"
#include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/closure.h"
@ -58,11 +62,12 @@
#include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset.h"
#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/arena.h"
#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_internal.h"
#include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/api_trace.h"
#include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/channel.h"
#include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/surface/channel_stack_type.h"
#include "src/core/lib/surface/server.h" #include "src/core/lib/surface/server.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/connectivity_state.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -87,10 +92,6 @@ void fill_in_metadata(inproc_stream* s, const grpc_metadata_batch* metadata,
uint32_t flags, grpc_metadata_batch* out_md, uint32_t flags, grpc_metadata_batch* out_md,
uint32_t* outflags, bool* markfilled); uint32_t* outflags, bool* markfilled);
void ResetSendMessage(grpc_transport_stream_op_batch* batch) {
absl::exchange(batch->payload->send_message.send_message, nullptr)->Clear();
}
struct shared_mu { struct shared_mu {
shared_mu() { shared_mu() {
// Share one lock between both sides since both sides get affected // Share one lock between both sides since both sides get affected
@ -225,6 +226,10 @@ struct inproc_stream {
GRPC_ERROR_UNREF(cancel_self_error); GRPC_ERROR_UNREF(cancel_self_error);
GRPC_ERROR_UNREF(cancel_other_error); GRPC_ERROR_UNREF(cancel_other_error);
if (recv_inited) {
grpc_slice_buffer_destroy_internal(&recv_message);
}
t->unref(); t->unref();
} }
@ -278,6 +283,10 @@ struct inproc_stream {
grpc_transport_stream_op_batch* recv_message_op = nullptr; grpc_transport_stream_op_batch* recv_message_op = nullptr;
grpc_transport_stream_op_batch* recv_trailing_md_op = nullptr; grpc_transport_stream_op_batch* recv_trailing_md_op = nullptr;
grpc_slice_buffer recv_message;
grpc_core::ManualConstructor<grpc_core::SliceBufferByteStream> recv_stream;
bool recv_inited = false;
bool initial_md_sent = false; bool initial_md_sent = false;
bool trailing_md_sent = false; bool trailing_md_sent = false;
bool initial_md_recvd = false; bool initial_md_recvd = false;
@ -351,8 +360,8 @@ void fill_in_metadata(inproc_stream* s, const grpc_metadata_batch* metadata,
} }
// TODO(ctiller): copy the metadata batch, don't rely on a bespoke copy // TODO(ctiller): copy the metadata batch, don't rely on a bespoke copy
// function. Can only do this once mdelems are out of the way though, too // function. Can only do this once mdelems are out of the way though, too many
// many edge cases otherwise. // edge cases otherwise.
out_md->Clear(); out_md->Clear();
CopySink sink(out_md); CopySink sink(out_md);
metadata->Encode(&sink); metadata->Encode(&sink);
@ -468,8 +477,8 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) {
if (s->recv_initial_md_op) { if (s->recv_initial_md_op) {
grpc_error_handle err; grpc_error_handle err;
if (!s->t->is_client) { if (!s->t->is_client) {
// If this is a server, provide initial metadata with a path and // If this is a server, provide initial metadata with a path and authority
// authority since it expects that as well as no error yet // since it expects that as well as no error yet
grpc_metadata_batch fake_md(s->arena); grpc_metadata_batch fake_md(s->arena);
fake_md.Set(grpc_core::HttpPathMetadata(), fake_md.Set(grpc_core::HttpPathMetadata(),
grpc_core::Slice::FromStaticString("/")); grpc_core::Slice::FromStaticString("/"));
@ -528,7 +537,7 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) {
s->recv_message_op = nullptr; s->recv_message_op = nullptr;
} }
if (s->send_message_op) { if (s->send_message_op) {
ResetSendMessage(s->send_message_op); s->send_message_op->payload->send_message.send_message.reset();
complete_if_batch_end_locked( complete_if_batch_end_locked(
s, error, s->send_message_op, s, error, s->send_message_op,
"fail_helper scheduling send-message-on-complete"); "fail_helper scheduling send-message-on-complete");
@ -570,11 +579,35 @@ void fail_helper_locked(inproc_stream* s, grpc_error_handle error) {
// synchronously. That assumption is true today but may not always be // synchronously. That assumption is true today but may not always be
// true in the future. // true in the future.
void message_transfer_locked(inproc_stream* sender, inproc_stream* receiver) { void message_transfer_locked(inproc_stream* sender, inproc_stream* receiver) {
*receiver->recv_message_op->payload->recv_message.recv_message = size_t remaining =
std::move(*sender->send_message_op->payload->send_message.send_message); sender->send_message_op->payload->send_message.send_message->length();
*receiver->recv_message_op->payload->recv_message.flags = if (receiver->recv_inited) {
sender->send_message_op->payload->send_message.flags; grpc_slice_buffer_destroy_internal(&receiver->recv_message);
}
grpc_slice_buffer_init(&receiver->recv_message);
receiver->recv_inited = true;
do {
grpc_slice message_slice;
grpc_closure unused;
GPR_ASSERT(
sender->send_message_op->payload->send_message.send_message->Next(
SIZE_MAX, &unused));
grpc_error_handle error =
sender->send_message_op->payload->send_message.send_message->Pull(
&message_slice);
if (error != GRPC_ERROR_NONE) {
cancel_stream_locked(sender, GRPC_ERROR_REF(error));
break;
}
GPR_ASSERT(error == GRPC_ERROR_NONE);
remaining -= GRPC_SLICE_LENGTH(message_slice);
grpc_slice_buffer_add(&receiver->recv_message, message_slice);
} while (remaining > 0);
sender->send_message_op->payload->send_message.send_message.reset();
receiver->recv_stream.Init(&receiver->recv_message, 0);
receiver->recv_message_op->payload->recv_message.recv_message->reset(
receiver->recv_stream.get());
INPROC_LOG(GPR_INFO, "message_transfer_locked %p scheduling message-ready", INPROC_LOG(GPR_INFO, "message_transfer_locked %p scheduling message-ready",
receiver); receiver);
grpc_core::ExecCtx::Run( grpc_core::ExecCtx::Run(
@ -623,7 +656,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
maybe_process_ops_locked(other, GRPC_ERROR_NONE); maybe_process_ops_locked(other, GRPC_ERROR_NONE);
} else if (!s->t->is_client && s->trailing_md_sent) { } else if (!s->t->is_client && s->trailing_md_sent) {
// A server send will never be matched if the server already sent status // A server send will never be matched if the server already sent status
ResetSendMessage(s->send_message_op); s->send_message_op->payload->send_message.send_message.reset();
complete_if_batch_end_locked( complete_if_batch_end_locked(
s, GRPC_ERROR_NONE, s->send_message_op, s, GRPC_ERROR_NONE, s->send_message_op,
"op_state_machine scheduling send-message-on-complete case 1"); "op_state_machine scheduling send-message-on-complete case 1");
@ -764,7 +797,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
if (s->recv_message_op != nullptr) { if (s->recv_message_op != nullptr) {
// This message needs to be wrapped up because it will never be // This message needs to be wrapped up because it will never be
// satisfied // satisfied
s->recv_message_op->payload->recv_message.recv_message->reset(); *s->recv_message_op->payload->recv_message.recv_message = nullptr;
INPROC_LOG(GPR_INFO, "op_state_machine %p scheduling message-ready", s); INPROC_LOG(GPR_INFO, "op_state_machine %p scheduling message-ready", s);
grpc_core::ExecCtx::Run( grpc_core::ExecCtx::Run(
DEBUG_LOCATION, DEBUG_LOCATION,
@ -778,7 +811,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
if ((s->trailing_md_sent || s->t->is_client) && s->send_message_op) { if ((s->trailing_md_sent || s->t->is_client) && s->send_message_op) {
// Nothing further will try to receive from this stream, so finish off // Nothing further will try to receive from this stream, so finish off
// any outstanding send_message op // any outstanding send_message op
ResetSendMessage(s->send_message_op); s->send_message_op->payload->send_message.send_message.reset();
s->send_message_op->payload->send_message.stream_write_closed = true; s->send_message_op->payload->send_message.stream_write_closed = true;
complete_if_batch_end_locked( complete_if_batch_end_locked(
s, new_err, s->send_message_op, s, new_err, s->send_message_op,
@ -798,8 +831,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
// We should schedule the recv_trailing_md_op completion if // We should schedule the recv_trailing_md_op completion if
// 1. this stream is the client-side // 1. this stream is the client-side
// 2. this stream is the server-side AND has already sent its trailing md // 2. this stream is the server-side AND has already sent its trailing md
// (If the server hasn't already sent its trailing md, it doesn't // (If the server hasn't already sent its trailing md, it doesn't have
// have
// a final status, so don't mark this op complete) // a final status, so don't mark this op complete)
if (s->t->is_client || s->trailing_md_sent) { if (s->t->is_client || s->trailing_md_sent) {
grpc_core::ExecCtx::Run( grpc_core::ExecCtx::Run(
@ -844,7 +876,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
// No further message will come on this stream, so finish off the // No further message will come on this stream, so finish off the
// recv_message_op // recv_message_op
INPROC_LOG(GPR_INFO, "op_state_machine %p scheduling message-ready", s); INPROC_LOG(GPR_INFO, "op_state_machine %p scheduling message-ready", s);
s->recv_message_op->payload->recv_message.recv_message->reset(); *s->recv_message_op->payload->recv_message.recv_message = nullptr;
grpc_core::ExecCtx::Run( grpc_core::ExecCtx::Run(
DEBUG_LOCATION, DEBUG_LOCATION,
s->recv_message_op->payload->recv_message.recv_message_ready, s->recv_message_op->payload->recv_message.recv_message_ready,
@ -857,7 +889,7 @@ void op_state_machine_locked(inproc_stream* s, grpc_error_handle error) {
if (s->trailing_md_recvd && s->send_message_op && s->t->is_client) { if (s->trailing_md_recvd && s->send_message_op && s->t->is_client) {
// Nothing further will try to receive from this stream, so finish off // Nothing further will try to receive from this stream, so finish off
// any outstanding send_message op // any outstanding send_message op
ResetSendMessage(s->send_message_op); s->send_message_op->payload->send_message.send_message.reset();
complete_if_batch_end_locked( complete_if_batch_end_locked(
s, new_err, s->send_message_op, s, new_err, s->send_message_op,
"op_state_machine scheduling send-message-on-complete case 3"); "op_state_machine scheduling send-message-on-complete case 3");
@ -1067,10 +1099,10 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
} }
} else { } else {
if (error != GRPC_ERROR_NONE) { if (error != GRPC_ERROR_NONE) {
// Consume any send message that was sent here but that we are not // Consume any send message that was sent here but that we are not pushing
// pushing to the other side // to the other side
if (op->send_message) { if (op->send_message) {
ResetSendMessage(op); op->payload->send_message.send_message.reset();
} }
// Schedule op's closures that we didn't push to op state machine // Schedule op's closures that we didn't push to op state machine
if (op->recv_initial_metadata) { if (op->recv_initial_metadata) {
@ -1106,10 +1138,10 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
GRPC_ERROR_REF(error)); GRPC_ERROR_REF(error));
} }
if (op->recv_trailing_metadata) { if (op->recv_trailing_metadata) {
INPROC_LOG(GPR_INFO, INPROC_LOG(
"perform_stream_op error %p scheduling " GPR_INFO,
"trailing-metadata-ready %s", "perform_stream_op error %p scheduling trailing-metadata-ready %s",
s, grpc_error_std_string(error).c_str()); s, grpc_error_std_string(error).c_str());
grpc_core::ExecCtx::Run( grpc_core::ExecCtx::Run(
DEBUG_LOCATION, DEBUG_LOCATION,
op->payload->recv_trailing_metadata.recv_trailing_metadata_ready, op->payload->recv_trailing_metadata.recv_trailing_metadata_ready,

@ -29,7 +29,7 @@
#include <grpc/support/atm.h> #include <grpc/support/atm.h>
#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/error.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -58,13 +58,13 @@ class CallTracer {
virtual void RecordOnDoneSendInitialMetadata(gpr_atm* peer_string) = 0; virtual void RecordOnDoneSendInitialMetadata(gpr_atm* peer_string) = 0;
virtual void RecordSendTrailingMetadata( virtual void RecordSendTrailingMetadata(
grpc_metadata_batch* send_trailing_metadata) = 0; grpc_metadata_batch* send_trailing_metadata) = 0;
virtual void RecordSendMessage(const SliceBuffer& send_message) = 0; virtual void RecordSendMessage(const ByteStream& send_message) = 0;
// The `RecordReceivedInitialMetadata()` and `RecordReceivedMessage()` // The `RecordReceivedInitialMetadata()` and `RecordReceivedMessage()`
// methods should only be invoked when the metadata/message was // methods should only be invoked when the metadata/message was
// successfully received, i.e., without any error. // successfully received, i.e., without any error.
virtual void RecordReceivedInitialMetadata( virtual void RecordReceivedInitialMetadata(
grpc_metadata_batch* recv_initial_metadata, uint32_t flags) = 0; grpc_metadata_batch* recv_initial_metadata, uint32_t flags) = 0;
virtual void RecordReceivedMessage(const SliceBuffer& recv_message) = 0; virtual void RecordReceivedMessage(const ByteStream& recv_message) = 0;
// If the call was cancelled before the recv_trailing_metadata op // If the call was cancelled before the recv_trailing_metadata op
// was started, recv_trailing_metadata and transport_stream_stats // was started, recv_trailing_metadata and transport_stream_stats
// will be null. // will be null.

@ -25,9 +25,12 @@
#include <stddef.h> #include <stddef.h>
#include <new>
#include <type_traits> #include <type_traits>
#include <utility> #include <utility>
#include <grpc/support/log.h>
#include "src/core/lib/gprpp/construct_destruct.h" #include "src/core/lib/gprpp/construct_destruct.h"
namespace grpc_core { namespace grpc_core {
@ -99,6 +102,70 @@ class max_align_of<A, B...> {
} // namespace manual_ctor_impl } // namespace manual_ctor_impl
template <class BaseType, class... DerivedTypes>
class PolymorphicManualConstructor {
public:
// No constructor or destructor because one of the most useful uses of
// this class is as part of a union, and members of a union could not have
// constructors or destructors till C++11. And, anyway, the whole point of
// this class is to bypass constructor and destructor.
BaseType* get() { return reinterpret_cast<BaseType*>(&space_); }
const BaseType* get() const {
return reinterpret_cast<const BaseType*>(&space_);
}
BaseType* operator->() { return get(); }
const BaseType* operator->() const { return get(); }
BaseType& operator*() { return *get(); }
const BaseType& operator*() const { return *get(); }
template <class DerivedType>
void Init() {
FinishInit(new (&space_) DerivedType);
}
// Init() constructs the Type instance using the given arguments
// (which are forwarded to Type's constructor).
//
// Note that Init() with no arguments performs default-initialization,
// not zero-initialization (i.e it behaves the same as "new Type;", not
// "new Type();"), so it will leave non-class types uninitialized.
template <class DerivedType, typename... Ts>
void Init(Ts&&... args) {
FinishInit(new (&space_) DerivedType(std::forward<Ts>(args)...));
}
// Init() that is equivalent to copy and move construction.
// Enables usage like this:
// ManualConstructor<std::vector<int>> v;
// v.Init({1, 2, 3});
template <class DerivedType>
void Init(const DerivedType& x) {
FinishInit(new (&space_) DerivedType(x));
}
template <class DerivedType>
void Init(DerivedType&& x) {
FinishInit(new (&space_) DerivedType(std::forward<DerivedType>(x)));
}
void Destroy() { get()->~BaseType(); }
private:
template <class DerivedType>
void FinishInit(DerivedType* p) {
static_assert(
manual_ctor_impl::is_one_of<DerivedType, DerivedTypes...>::value,
"DerivedType must be one of the predeclared DerivedTypes");
GPR_ASSERT(static_cast<BaseType*>(p) == p);
}
typename std::aligned_storage<
manual_ctor_impl::max_size_of<DerivedTypes...>::value,
manual_ctor_impl::max_align_of<DerivedTypes...>::value>::type space_;
};
template <typename Type> template <typename Type>
class ManualConstructor { class ManualConstructor {
public: public:

@ -950,7 +950,6 @@ ssize_t tcp_send(int fd, const struct msghdr* msg, int additional_flags = 0) {
ssize_t sent_length; ssize_t sent_length;
do { do {
/* TODO(klempner): Cork if this is a partial write */ /* TODO(klempner): Cork if this is a partial write */
errno = 0;
GRPC_STATS_INC_SYSCALL_WRITE(); GRPC_STATS_INC_SYSCALL_WRITE();
sent_length = sendmsg(fd, msg, SENDMSG_FLAGS | additional_flags); sent_length = sendmsg(fd, msg, SENDMSG_FLAGS | additional_flags);
} while (sent_length < 0 && errno == EINTR); } while (sent_length < 0 && errno == EINTR);

@ -51,21 +51,10 @@ void SliceBuffer::Prepend(Slice slice) {
grpc_slice_buffer_undo_take_first(&slice_buffer_, slice.TakeCSlice()); grpc_slice_buffer_undo_take_first(&slice_buffer_, slice.TakeCSlice());
} }
Slice SliceBuffer::RefSlice(size_t index) const { Slice SliceBuffer::RefSlice(size_t index) {
return Slice(grpc_slice_ref_internal(slice_buffer_.slices[index])); return Slice(grpc_slice_ref_internal(slice_buffer_.slices[index]));
} }
std::string SliceBuffer::JoinIntoString() const {
std::string result;
result.reserve(slice_buffer_.length);
for (size_t i = 0; i < slice_buffer_.count; i++) {
result.append(reinterpret_cast<const char*>(
GRPC_SLICE_START_PTR(slice_buffer_.slices[i])),
GRPC_SLICE_LENGTH(slice_buffer_.slices[i]));
}
return result;
}
} // namespace grpc_core } // namespace grpc_core
/* grow a buffer; requires GRPC_SLICE_BUFFER_INLINE_ELEMENTS > 1 */ /* grow a buffer; requires GRPC_SLICE_BUFFER_INLINE_ELEMENTS > 1 */
@ -381,24 +370,6 @@ void grpc_slice_buffer_move_first_into_buffer(grpc_slice_buffer* src, size_t n,
} }
} }
void grpc_slice_buffer_copy_first_into_buffer(grpc_slice_buffer* src, size_t n,
void* dst) {
uint8_t* dstp = static_cast<uint8_t*>(dst);
GPR_ASSERT(src->length >= n);
for (size_t i = 0; i < src->count; i++) {
grpc_slice slice = src->slices[i];
size_t slice_len = GRPC_SLICE_LENGTH(slice);
if (slice_len >= n) {
memcpy(dstp, GRPC_SLICE_START_PTR(slice), n);
return;
}
memcpy(dstp, GRPC_SLICE_START_PTR(slice), slice_len);
dstp += slice_len;
n -= slice_len;
}
}
void grpc_slice_buffer_trim_end(grpc_slice_buffer* sb, size_t n, void grpc_slice_buffer_trim_end(grpc_slice_buffer* sb, size_t n,
grpc_slice_buffer* garbage) { grpc_slice_buffer* garbage) {
GPR_ASSERT(n <= sb->length); GPR_ASSERT(n <= sb->length);

@ -19,8 +19,6 @@
#include <string.h> #include <string.h>
#include <string>
#include <grpc/slice.h> #include <grpc/slice.h>
#include <grpc/slice_buffer.h> #include <grpc/slice_buffer.h>
@ -47,15 +45,14 @@ class SliceBuffer {
SliceBuffer(const SliceBuffer& other) = delete; SliceBuffer(const SliceBuffer& other) = delete;
SliceBuffer(SliceBuffer&& other) noexcept { SliceBuffer(SliceBuffer&& other) noexcept {
grpc_slice_buffer_init(&slice_buffer_); grpc_slice_buffer_init(&slice_buffer_);
grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_); grpc_slice_buffer_move_into(&slice_buffer_, &other.slice_buffer_);
} }
/// Upon destruction, the underlying raw slice buffer is cleaned out and all /// Upon destruction, the underlying raw slice buffer is cleaned out and all
/// slices are unreffed. /// slices are unreffed.
~SliceBuffer() { grpc_slice_buffer_destroy(&slice_buffer_); } ~SliceBuffer() { grpc_slice_buffer_destroy(&slice_buffer_); }
SliceBuffer& operator=(const SliceBuffer&) = delete;
SliceBuffer& operator=(SliceBuffer&& other) noexcept { SliceBuffer& operator=(SliceBuffer&& other) noexcept {
grpc_slice_buffer_swap(&slice_buffer_, &other.slice_buffer_); grpc_slice_buffer_move_into(&slice_buffer_, &other.slice_buffer_);
return *this; return *this;
} }
@ -68,7 +65,7 @@ class SliceBuffer {
size_t AppendIndexed(Slice slice); size_t AppendIndexed(Slice slice);
/// Returns the number of slices held by the SliceBuffer. /// Returns the number of slices held by the SliceBuffer.
size_t Count() const { return slice_buffer_.count; } size_t Count() { return slice_buffer_.count; }
/// Removes/deletes the last n bytes in the SliceBuffer. /// Removes/deletes the last n bytes in the SliceBuffer.
void RemoveLastNBytes(size_t n) { void RemoveLastNBytes(size_t n) {
@ -91,37 +88,13 @@ class SliceBuffer {
/// Increased the ref-count of slice at the specified index and returns the /// Increased the ref-count of slice at the specified index and returns the
/// associated slice. /// associated slice.
Slice RefSlice(size_t index) const; Slice RefSlice(size_t index);
/// The total number of bytes held by the SliceBuffer /// The total number of bytes held by the SliceBuffer
size_t Length() const { return slice_buffer_.length; } size_t Length() { return slice_buffer_.length; }
/// Swap with another slice buffer
void Swap(SliceBuffer* other) {
grpc_slice_buffer_swap(c_slice_buffer(), other->c_slice_buffer());
}
/// Concatenate all slices and return the resulting string.
std::string JoinIntoString() const;
// Return a copy of the slice buffer
SliceBuffer Copy() const {
SliceBuffer copy;
for (size_t i = 0; i < Count(); i++) {
copy.Append(RefSlice(i));
}
return copy;
}
/// Return a pointer to the back raw grpc_slice_buffer /// Return a pointer to the back raw grpc_slice_buffer
grpc_slice_buffer* c_slice_buffer() { return &slice_buffer_; } grpc_slice_buffer* RawSliceBuffer() { return &slice_buffer_; }
/// Return a pointer to the back raw grpc_slice_buffer
const grpc_slice_buffer* c_slice_buffer() const { return &slice_buffer_; }
const grpc_slice& c_slice_at(size_t index) {
return slice_buffer_.slices[index];
}
private: private:
/// The backing raw slice buffer. /// The backing raw slice buffer.
@ -130,8 +103,4 @@ class SliceBuffer {
} // namespace grpc_core } // namespace grpc_core
// Copy the first n bytes of src into memory pointed to by dst.
void grpc_slice_buffer_copy_first_into_buffer(grpc_slice_buffer* src, size_t n,
void* dst);
#endif // GRPC_CORE_LIB_SLICE_SLICE_BUFFER_H #endif // GRPC_CORE_LIB_SLICE_SLICE_BUFFER_H

@ -25,6 +25,7 @@
#include <algorithm> #include <algorithm>
#include <atomic> #include <atomic>
#include <memory>
#include <new> #include <new>
#include <string> #include <string>
#include <utility> #include <utility>
@ -58,6 +59,8 @@
#include "src/core/lib/gpr/time_precise.h" #include "src/core/lib/gpr/time_precise.h"
#include "src/core/lib/gprpp/cpp_impl_of.h" #include "src/core/lib/gprpp/cpp_impl_of.h"
#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/manual_constructor.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted.h"
#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/gprpp/sync.h"
#include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/call_combiner.h"
@ -65,7 +68,6 @@
#include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/iomgr/polling_entity.h"
#include "src/core/lib/profiling/timers.h" #include "src/core/lib/profiling/timers.h"
#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/arena.h"
#include "src/core/lib/slice/slice_buffer.h"
#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_internal.h"
#include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/slice/slice_refcount.h"
#include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/api_trace.h"
@ -74,6 +76,7 @@
#include "src/core/lib/surface/completion_queue.h" #include "src/core/lib/surface/completion_queue.h"
#include "src/core/lib/surface/server.h" #include "src/core/lib/surface/server.h"
#include "src/core/lib/surface/validate_metadata.h" #include "src/core/lib/surface/validate_metadata.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/error_utils.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -306,6 +309,8 @@ class FilterStackCall final : public Call {
void PostCompletion(); void PostCompletion();
void FinishStep(); void FinishStep();
void ContinueReceivingSlices();
void ReceivingSliceReady(grpc_error_handle error);
void ProcessDataAfterMetadata(); void ProcessDataAfterMetadata();
void ReceivingStreamReady(grpc_error_handle error); void ReceivingStreamReady(grpc_error_handle error);
void ValidateFilteredMetadata(); void ValidateFilteredMetadata();
@ -395,13 +400,13 @@ class FilterStackCall final : public Call {
/* Contexts for various subsystems (security, tracing, ...). */ /* Contexts for various subsystems (security, tracing, ...). */
grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {}; grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
SliceBuffer send_slice_buffer_; ManualConstructor<SliceBufferByteStream> sending_stream_;
absl::optional<SliceBuffer> receiving_slice_buffer_;
uint32_t receiving_stream_flags_;
OrphanablePtr<ByteStream> receiving_stream_;
bool call_failed_before_recv_message_ = false; bool call_failed_before_recv_message_ = false;
grpc_byte_buffer** receiving_buffer_ = nullptr; grpc_byte_buffer** receiving_buffer_ = nullptr;
grpc_slice receiving_slice_ = grpc_empty_slice(); grpc_slice receiving_slice_ = grpc_empty_slice();
grpc_closure receiving_slice_ready_;
grpc_closure receiving_stream_ready_; grpc_closure receiving_stream_ready_;
grpc_closure receiving_initial_metadata_ready_; grpc_closure receiving_initial_metadata_ready_;
grpc_closure receiving_trailing_metadata_ready_; grpc_closure receiving_trailing_metadata_ready_;
@ -648,7 +653,7 @@ void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
auto* c = static_cast<FilterStackCall*>(call); auto* c = static_cast<FilterStackCall*>(call);
c->recv_initial_metadata_.Clear(); c->recv_initial_metadata_.Clear();
c->recv_trailing_metadata_.Clear(); c->recv_trailing_metadata_.Clear();
c->receiving_slice_buffer_.reset(); c->receiving_stream_.reset();
ParentCall* pc = c->parent_call(); ParentCall* pc = c->parent_call();
if (pc != nullptr) { if (pc != nullptr) {
pc->~ParentCall(); pc->~ParentCall();
@ -1085,7 +1090,6 @@ void FilterStackCall::BatchControl::PostCompletion() {
"Attempt to send message after stream was closed.")); "Attempt to send message after stream was closed."));
} }
call->sending_message_ = false; call->sending_message_ = false;
call->send_slice_buffer_.Clear();
} }
if (op_.send_trailing_metadata) { if (op_.send_trailing_metadata) {
call->send_trailing_metadata_.Clear(); call->send_trailing_metadata_.Clear();
@ -1131,27 +1135,95 @@ void FilterStackCall::BatchControl::FinishStep() {
} }
} }
void FilterStackCall::BatchControl::ContinueReceivingSlices() {
grpc_error_handle error;
FilterStackCall* call = call_;
for (;;) {
size_t remaining = call->receiving_stream_->length() -
(*call->receiving_buffer_)->data.raw.slice_buffer.length;
if (remaining == 0) {
call->receiving_message_ = false;
call->receiving_stream_.reset();
FinishStep();
return;
}
if (call->receiving_stream_->Next(remaining,
&call->receiving_slice_ready_)) {
error = call->receiving_stream_->Pull(&call->receiving_slice_);
if (error == GRPC_ERROR_NONE) {
grpc_slice_buffer_add(
&(*call->receiving_buffer_)->data.raw.slice_buffer,
call->receiving_slice_);
} else {
call->receiving_stream_.reset();
grpc_byte_buffer_destroy(*call->receiving_buffer_);
*call->receiving_buffer_ = nullptr;
call->receiving_message_ = false;
FinishStep();
GRPC_ERROR_UNREF(error);
return;
}
} else {
return;
}
}
}
void FilterStackCall::BatchControl::ReceivingSliceReady(
grpc_error_handle error) {
FilterStackCall* call = call_;
bool release_error = false;
if (error == GRPC_ERROR_NONE) {
grpc_slice slice;
error = call->receiving_stream_->Pull(&slice);
if (error == GRPC_ERROR_NONE) {
grpc_slice_buffer_add(&(*call->receiving_buffer_)->data.raw.slice_buffer,
slice);
ContinueReceivingSlices();
} else {
/* Error returned by ByteStream::Pull() needs to be released manually */
release_error = true;
}
}
if (error != GRPC_ERROR_NONE) {
if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures)) {
GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error));
}
call->receiving_stream_.reset();
grpc_byte_buffer_destroy(*call->receiving_buffer_);
*call->receiving_buffer_ = nullptr;
call->receiving_message_ = false;
FinishStep();
if (release_error) {
GRPC_ERROR_UNREF(error);
}
}
}
void FilterStackCall::BatchControl::ProcessDataAfterMetadata() { void FilterStackCall::BatchControl::ProcessDataAfterMetadata() {
FilterStackCall* call = call_; FilterStackCall* call = call_;
if (!call->receiving_slice_buffer_.has_value()) { if (call->receiving_stream_ == nullptr) {
*call->receiving_buffer_ = nullptr; *call->receiving_buffer_ = nullptr;
call->receiving_message_ = false; call->receiving_message_ = false;
FinishStep(); FinishStep();
} else { } else {
call->test_only_last_message_flags_ = call->receiving_stream_flags_; call->test_only_last_message_flags_ = call->receiving_stream_->flags();
if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) && if ((call->receiving_stream_->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
(call->incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) { (call->incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) {
*call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create( *call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create(
nullptr, 0, call->incoming_compression_algorithm_); nullptr, 0, call->incoming_compression_algorithm_);
} else { } else {
*call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0); *call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0);
} }
grpc_slice_buffer_move_into( GRPC_CLOSURE_INIT(
call->receiving_slice_buffer_->c_slice_buffer(), &call->receiving_slice_ready_,
&(*call->receiving_buffer_)->data.raw.slice_buffer); [](void* bctl, grpc_error_handle error) {
call->receiving_message_ = false; static_cast<BatchControl*>(bctl)->ReceivingSliceReady(error);
call->receiving_slice_buffer_.reset(); },
FinishStep(); this, grpc_schedule_on_exec_ctx);
ContinueReceivingSlices();
} }
} }
@ -1159,7 +1231,7 @@ void FilterStackCall::BatchControl::ReceivingStreamReady(
grpc_error_handle error) { grpc_error_handle error) {
FilterStackCall* call = call_; FilterStackCall* call = call_;
if (error != GRPC_ERROR_NONE) { if (error != GRPC_ERROR_NONE) {
call->receiving_slice_buffer_.reset(); call->receiving_stream_.reset();
if (batch_error_.ok()) { if (batch_error_.ok()) {
batch_error_.set(error); batch_error_.set(error);
} }
@ -1168,7 +1240,7 @@ void FilterStackCall::BatchControl::ReceivingStreamReady(
/* If recv_state is kRecvNone, we will save the batch_control /* If recv_state is kRecvNone, we will save the batch_control
* object with rel_cas, and will not use it after the cas. Its corresponding * object with rel_cas, and will not use it after the cas. Its corresponding
* acq_load is in receiving_initial_metadata_ready() */ * acq_load is in receiving_initial_metadata_ready() */
if (error != GRPC_ERROR_NONE || !call->receiving_slice_buffer_.has_value() || if (error != GRPC_ERROR_NONE || call->receiving_stream_ == nullptr ||
!gpr_atm_rel_cas(&call->recv_state_, kRecvNone, !gpr_atm_rel_cas(&call->recv_state_, kRecvNone,
reinterpret_cast<gpr_atm>(this))) { reinterpret_cast<gpr_atm>(this))) {
ProcessDataAfterMetadata(); ProcessDataAfterMetadata();
@ -1449,12 +1521,10 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
} }
stream_op->send_message = true; stream_op->send_message = true;
sending_message_ = true; sending_message_ = true;
send_slice_buffer_.Clear(); sending_stream_.Init(
grpc_slice_buffer_move_into( &op->data.send_message.send_message->data.raw.slice_buffer, flags);
&op->data.send_message.send_message->data.raw.slice_buffer, stream_op_payload->send_message.send_message.reset(
send_slice_buffer_.c_slice_buffer()); sending_stream_.get());
stream_op_payload->send_message.flags = flags;
stream_op_payload->send_message.send_message = &send_slice_buffer_;
has_send_ops = true; has_send_ops = true;
break; break;
} }
@ -1591,11 +1661,8 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
} }
receiving_message_ = true; receiving_message_ = true;
stream_op->recv_message = true; stream_op->recv_message = true;
receiving_slice_buffer_.reset();
receiving_buffer_ = op->data.recv_message.recv_message; receiving_buffer_ = op->data.recv_message.recv_message;
stream_op_payload->recv_message.recv_message = &receiving_slice_buffer_; stream_op_payload->recv_message.recv_message = &receiving_stream_;
receiving_stream_flags_ = 0;
stream_op_payload->recv_message.flags = &receiving_stream_flags_;
stream_op_payload->recv_message.call_failed_before_recv_message = stream_op_payload->recv_message.call_failed_before_recv_message =
&call_failed_before_recv_message_; &call_failed_before_recv_message_;
GRPC_CLOSURE_INIT( GRPC_CLOSURE_INIT(
@ -1720,6 +1787,10 @@ done_with_error:
} }
if (stream_op->send_message) { if (stream_op->send_message) {
sending_message_ = false; sending_message_ = false;
// No need to invoke call->sending_stream->Orphan() explicitly.
// stream_op_payload->send_message.send_message.reset() calls Deletor
// of call->sending_stream which in-turn invokes the Orphan() method.
stream_op_payload->send_message.send_message.reset();
} }
if (stream_op->send_trailing_metadata) { if (stream_op->send_trailing_metadata) {
sent_final_op_ = false; sent_final_op_ = false;

@ -0,0 +1,165 @@
/*
*
* Copyright 2015 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#include <grpc/support/port_platform.h>
#include "src/core/lib/transport/byte_stream.h"
#include <memory>
#include <utility>
#include <grpc/slice_buffer.h>
#include <grpc/support/log.h>
#include "src/core/lib/slice/slice_internal.h"
#include "src/core/lib/slice/slice_refcount.h"
namespace grpc_core {
//
// SliceBufferByteStream
//
SliceBufferByteStream::SliceBufferByteStream(grpc_slice_buffer* slice_buffer,
uint32_t flags)
: ByteStream(static_cast<uint32_t>(slice_buffer->length), flags) {
GPR_ASSERT(slice_buffer->length <= UINT32_MAX);
grpc_slice_buffer_init(&backing_buffer_);
grpc_slice_buffer_swap(slice_buffer, &backing_buffer_);
if (backing_buffer_.count == 0) {
grpc_slice_buffer_add_indexed(&backing_buffer_, grpc_empty_slice());
GPR_ASSERT(backing_buffer_.count > 0);
}
}
SliceBufferByteStream::~SliceBufferByteStream() {}
void SliceBufferByteStream::Orphan() {
grpc_slice_buffer_destroy_internal(&backing_buffer_);
GRPC_ERROR_UNREF(shutdown_error_);
shutdown_error_ = GRPC_ERROR_NONE;
// Note: We do not actually delete the object here, since
// SliceBufferByteStream is usually allocated as part of a larger
// object and has an OrphanablePtr of itself passed down through the
// filter stack.
}
bool SliceBufferByteStream::Next(size_t /*max_size_hint*/,
grpc_closure* /*on_complete*/) {
GPR_DEBUG_ASSERT(backing_buffer_.count > 0);
return true;
}
grpc_error_handle SliceBufferByteStream::Pull(grpc_slice* slice) {
if (GPR_UNLIKELY(shutdown_error_ != GRPC_ERROR_NONE)) {
return GRPC_ERROR_REF(shutdown_error_);
}
*slice = grpc_slice_buffer_take_first(&backing_buffer_);
return GRPC_ERROR_NONE;
}
void SliceBufferByteStream::Shutdown(grpc_error_handle error) {
GRPC_ERROR_UNREF(shutdown_error_);
shutdown_error_ = error;
}
//
// ByteStreamCache
//
ByteStreamCache::ByteStreamCache(OrphanablePtr<ByteStream> underlying_stream)
: underlying_stream_(std::move(underlying_stream)),
length_(underlying_stream_->length()),
flags_(underlying_stream_->flags()) {
grpc_slice_buffer_init(&cache_buffer_);
}
ByteStreamCache::~ByteStreamCache() { Destroy(); }
void ByteStreamCache::Destroy() {
underlying_stream_.reset();
if (cache_buffer_.length > 0) {
grpc_slice_buffer_destroy_internal(&cache_buffer_);
}
}
//
// ByteStreamCache::CachingByteStream
//
ByteStreamCache::CachingByteStream::CachingByteStream(ByteStreamCache* cache)
: ByteStream(cache->length_, cache->flags_), cache_(cache) {}
ByteStreamCache::CachingByteStream::~CachingByteStream() {}
void ByteStreamCache::CachingByteStream::Orphan() {
GRPC_ERROR_UNREF(shutdown_error_);
shutdown_error_ = GRPC_ERROR_NONE;
// Note: We do not actually delete the object here, since
// CachingByteStream is usually allocated as part of a larger
// object and has an OrphanablePtr of itself passed down through the
// filter stack.
}
bool ByteStreamCache::CachingByteStream::Next(size_t max_size_hint,
grpc_closure* on_complete) {
if (shutdown_error_ != GRPC_ERROR_NONE) return true;
if (cursor_ < cache_->cache_buffer_.count) return true;
GPR_ASSERT(cache_->underlying_stream_ != nullptr);
return cache_->underlying_stream_->Next(max_size_hint, on_complete);
}
grpc_error_handle ByteStreamCache::CachingByteStream::Pull(grpc_slice* slice) {
if (shutdown_error_ != GRPC_ERROR_NONE) {
return GRPC_ERROR_REF(shutdown_error_);
}
if (cursor_ < cache_->cache_buffer_.count) {
*slice = grpc_slice_ref_internal(cache_->cache_buffer_.slices[cursor_]);
++cursor_;
offset_ += GRPC_SLICE_LENGTH(*slice);
return GRPC_ERROR_NONE;
}
GPR_ASSERT(cache_->underlying_stream_ != nullptr);
grpc_error_handle error = cache_->underlying_stream_->Pull(slice);
if (error == GRPC_ERROR_NONE) {
grpc_slice_buffer_add(&cache_->cache_buffer_,
grpc_slice_ref_internal(*slice));
++cursor_;
offset_ += GRPC_SLICE_LENGTH(*slice);
// Orphan the underlying stream if it's been drained.
if (offset_ == cache_->underlying_stream_->length()) {
cache_->underlying_stream_.reset();
}
}
return error;
}
void ByteStreamCache::CachingByteStream::Shutdown(grpc_error_handle error) {
GRPC_ERROR_UNREF(shutdown_error_);
shutdown_error_ = GRPC_ERROR_REF(error);
if (cache_->underlying_stream_ != nullptr) {
cache_->underlying_stream_->Shutdown(error);
}
}
void ByteStreamCache::CachingByteStream::Reset() {
cursor_ = 0;
offset_ = 0;
}
} // namespace grpc_core

@ -0,0 +1,170 @@
/*
*
* Copyright 2015 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#ifndef GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H
#define GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H
#include <grpc/support/port_platform.h>
#include <stddef.h>
#include <stdint.h>
#include <grpc/slice.h>
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/iomgr/closure.h"
#include "src/core/lib/iomgr/error.h"
/** Internal bit flag for grpc_begin_message's \a flags signaling the use of
* compression for the message. (Does not apply for stream compression.) */
#define GRPC_WRITE_INTERNAL_COMPRESS (0x80000000u)
/** Internal bit flag for determining whether the message was compressed and had
* to be decompressed by the message_decompress filter. (Does not apply for
* stream compression.) */
#define GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED (0x40000000u)
/** Mask of all valid internal flags. */
#define GRPC_WRITE_INTERNAL_USED_MASK \
(GRPC_WRITE_INTERNAL_COMPRESS | GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED)
namespace grpc_core {
class ByteStream : public Orphanable {
public:
~ByteStream() override {}
// Returns true if the bytes are available immediately (in which case
// on_complete will not be called), or false if the bytes will be available
// asynchronously (in which case on_complete will be called when they
// are available). Should not be called if there is no data left on the
// stream.
//
// max_size_hint can be set as a hint as to the maximum number
// of bytes that would be acceptable to read.
virtual bool Next(size_t max_size_hint, grpc_closure* on_complete) = 0;
// Returns the next slice in the byte stream when it is available, as
// indicated by Next().
//
// Once a slice is returned into *slice, it is owned by the caller.
virtual grpc_error_handle Pull(grpc_slice* slice) = 0;
// Shuts down the byte stream.
//
// If there is a pending call to on_complete from Next(), it will be
// invoked with the error passed to Shutdown().
//
// The next call to Pull() (if any) will return the error passed to
// Shutdown().
virtual void Shutdown(grpc_error_handle error) = 0;
uint32_t length() const { return length_; }
uint32_t flags() const { return flags_; }
void set_flags(uint32_t flags) { flags_ = flags; }
protected:
ByteStream(uint32_t length, uint32_t flags)
: length_(length), flags_(flags) {}
private:
const uint32_t length_;
uint32_t flags_;
};
//
// SliceBufferByteStream
//
// A ByteStream that wraps a slice buffer.
//
class SliceBufferByteStream : public ByteStream {
public:
// Removes all slices in slice_buffer, leaving it empty.
SliceBufferByteStream(grpc_slice_buffer* slice_buffer, uint32_t flags);
~SliceBufferByteStream() override;
void Orphan() override;
bool Next(size_t max_size_hint, grpc_closure* on_complete) override;
grpc_error_handle Pull(grpc_slice* slice) override;
void Shutdown(grpc_error_handle error) override;
private:
grpc_error_handle shutdown_error_ = GRPC_ERROR_NONE;
grpc_slice_buffer backing_buffer_;
};
//
// CachingByteStream
//
// A ByteStream that that wraps an underlying byte stream but caches
// the resulting slices in a slice buffer. If an initial attempt fails
// without fully draining the underlying stream, a new caching stream
// can be created from the same underlying cache, in which case it will
// return whatever is in the backing buffer before continuing to read the
// underlying stream.
//
// NOTE: No synchronization is done, so it is not safe to have multiple
// CachingByteStreams simultaneously drawing from the same underlying
// ByteStreamCache at the same time.
//
class ByteStreamCache {
public:
class CachingByteStream : public ByteStream {
public:
explicit CachingByteStream(ByteStreamCache* cache);
~CachingByteStream() override;
void Orphan() override;
bool Next(size_t max_size_hint, grpc_closure* on_complete) override;
grpc_error_handle Pull(grpc_slice* slice) override;
void Shutdown(grpc_error_handle error) override;
// Resets the byte stream to the start of the underlying stream.
void Reset();
private:
ByteStreamCache* cache_;
size_t cursor_ = 0;
size_t offset_ = 0;
grpc_error_handle shutdown_error_ = GRPC_ERROR_NONE;
};
explicit ByteStreamCache(OrphanablePtr<ByteStream> underlying_stream);
~ByteStreamCache();
// Must not be destroyed while still in use by a CachingByteStream.
void Destroy();
grpc_slice_buffer* cache_buffer() { return &cache_buffer_; }
private:
OrphanablePtr<ByteStream> underlying_stream_;
uint32_t length_;
uint32_t flags_;
grpc_slice_buffer cache_buffer_;
};
} // namespace grpc_core
#endif /* GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H */

@ -161,6 +161,9 @@ void grpc_transport_stream_op_batch_finish_with_failure(
void grpc_transport_stream_op_batch_queue_finish_with_failure( void grpc_transport_stream_op_batch_queue_finish_with_failure(
grpc_transport_stream_op_batch* batch, grpc_error_handle error, grpc_transport_stream_op_batch* batch, grpc_error_handle error,
grpc_core::CallCombinerClosureList* closures) { grpc_core::CallCombinerClosureList* closures) {
if (batch->send_message) {
batch->payload->send_message.send_message.reset();
}
if (batch->cancel_stream) { if (batch->cancel_stream) {
GRPC_ERROR_UNREF(batch->payload->cancel_stream.cancel_error); GRPC_ERROR_UNREF(batch->payload->cancel_stream.cancel_error);
} }

@ -55,7 +55,7 @@
#include "src/core/lib/promise/latch.h" #include "src/core/lib/promise/latch.h"
#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/arena.h"
#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/connectivity_state.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport_fwd.h" #include "src/core/lib/transport/transport_fwd.h"
@ -70,17 +70,6 @@ struct grpc_transport_stream_op_batch_payload;
#define GRPC_ARG_TRANSPORT "grpc.internal.transport" #define GRPC_ARG_TRANSPORT "grpc.internal.transport"
/** Internal bit flag for grpc_begin_message's \a flags signaling the use of
* compression for the message. (Does not apply for stream compression.) */
#define GRPC_WRITE_INTERNAL_COMPRESS (0x80000000u)
/** Internal bit flag for determining whether the message was compressed and had
* to be decompressed by the message_decompress filter. (Does not apply for
* stream compression.) */
#define GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED (0x40000000u)
/** Mask of all valid internal flags. */
#define GRPC_WRITE_INTERNAL_USED_MASK \
(GRPC_WRITE_INTERNAL_COMPRESS | GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED)
namespace grpc_core { namespace grpc_core {
// TODO(ctiller): eliminate once MetadataHandle is constructable directly. // TODO(ctiller): eliminate once MetadataHandle is constructable directly.
namespace promise_filter_detail { namespace promise_filter_detail {
@ -342,6 +331,12 @@ struct grpc_transport_stream_op_batch_payload {
explicit grpc_transport_stream_op_batch_payload( explicit grpc_transport_stream_op_batch_payload(
grpc_call_context_element* context) grpc_call_context_element* context)
: context(context) {} : context(context) {}
~grpc_transport_stream_op_batch_payload() {
// We don't really own `send_message`, so release ownership and let the
// owner clean the data.
(void)send_message.send_message.release();
}
struct { struct {
grpc_metadata_batch* send_initial_metadata = nullptr; grpc_metadata_batch* send_initial_metadata = nullptr;
/** Iff send_initial_metadata != NULL, flags associated with /** Iff send_initial_metadata != NULL, flags associated with
@ -370,8 +365,7 @@ struct grpc_transport_stream_op_batch_payload {
// the op gets down to the transport) takes ownership. // the op gets down to the transport) takes ownership.
// The batch's on_complete will not be called until after the byte // The batch's on_complete will not be called until after the byte
// stream is orphaned. // stream is orphaned.
grpc_core::SliceBuffer* send_message; grpc_core::OrphanablePtr<grpc_core::ByteStream> send_message;
uint32_t flags = 0;
// Set by the transport if the stream has been closed for writes. If this // Set by the transport if the stream has been closed for writes. If this
// is set and send message op is present, we set the operation to be a // is set and send message op is present, we set the operation to be a
// failure without sending a cancel OP down the stack. This is so that the // failure without sending a cancel OP down the stack. This is so that the
@ -410,11 +404,10 @@ struct grpc_transport_stream_op_batch_payload {
} recv_initial_metadata; } recv_initial_metadata;
struct { struct {
// Will be set by the transport to point to the byte stream containing a // Will be set by the transport to point to the byte stream
// received message. Will be nullopt if trailing metadata is received // containing a received message.
// instead of a message. // Will be NULL if trailing metadata is received instead of a message.
absl::optional<grpc_core::SliceBuffer>* recv_message = nullptr; grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message = nullptr;
uint32_t* flags = nullptr;
// Was this recv_message failed for reasons other than a clean end-of-stream // Was this recv_message failed for reasons other than a clean end-of-stream
bool* call_failed_before_recv_message = nullptr; bool* call_failed_before_recv_message = nullptr;
/** Should be enqueued when one message is ready to be processed. */ /** Should be enqueued when one message is ready to be processed. */

@ -33,7 +33,7 @@
#include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/channel_stack.h"
#include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/error.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/connectivity_state.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -54,9 +54,10 @@ std::string grpc_transport_stream_op_batch_string(
if (op->send_message) { if (op->send_message) {
if (op->payload->send_message.send_message != nullptr) { if (op->payload->send_message.send_message != nullptr) {
out.push_back(absl::StrFormat( out.push_back(
" SEND_MESSAGE:flags=0x%08x:len=%d", op->payload->send_message.flags, absl::StrFormat(" SEND_MESSAGE:flags=0x%08x:len=%d",
op->payload->send_message.send_message->Length())); op->payload->send_message.send_message->flags(),
op->payload->send_message.send_message->length()));
} else { } else {
// This can happen when we check a batch after the transport has // This can happen when we check a batch after the transport has
// processed and cleared the send_message op. // processed and cleared the send_message op.

@ -27,8 +27,6 @@
#include <string> #include <string>
#include <utility> #include <utility>
#include "absl/types/optional.h"
#include <grpc/grpc.h> #include <grpc/grpc.h>
#include <grpc/impl/codegen/grpc_types.h> #include <grpc/impl/codegen/grpc_types.h>
#include <grpc/support/atm.h> #include <grpc/support/atm.h>
@ -37,11 +35,12 @@
#include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_fwd.h"
#include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/channel_stack.h"
#include "src/core/lib/channel/context.h" #include "src/core/lib/channel/context.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/closure.h"
#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/error.h"
#include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/iomgr/polling_entity.h"
#include "src/core/lib/slice/slice_buffer.h"
#include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/surface/channel_stack_type.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
@ -153,21 +152,22 @@ class TransportStreamOpBatch {
op_->payload->recv_initial_metadata.recv_initial_metadata_ready = closure; op_->payload->recv_initial_metadata.recv_initial_metadata_ready = closure;
} }
grpc_core::SliceBuffer* send_message() const { grpc_core::OrphanablePtr<grpc_core::ByteStream>* send_message() const {
return op_->send_message ? op_->payload->send_message.send_message return op_->send_message ? &op_->payload->send_message.send_message
: nullptr; : nullptr;
} }
void set_send_message(
void set_send_message(grpc_core::SliceBuffer* send_message) { grpc_core::OrphanablePtr<grpc_core::ByteStream> send_message) {
op_->send_message = true; op_->send_message = true;
op_->payload->send_message.send_message = send_message; op_->payload->send_message.send_message = std::move(send_message);
} }
absl::optional<grpc_core::SliceBuffer>* recv_message() const { grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message() const {
return op_->recv_message ? op_->payload->recv_message.recv_message return op_->recv_message ? op_->payload->recv_message.recv_message
: nullptr; : nullptr;
} }
void set_recv_message(absl::optional<grpc_core::SliceBuffer>* recv_message) { void set_recv_message(
grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message) {
op_->recv_message = true; op_->recv_message = true;
op_->payload->recv_message.recv_message = recv_message; op_->payload->recv_message.recv_message = recv_message;
} }

@ -50,8 +50,8 @@
#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/gprpp/sync.h"
#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/arena.h"
#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice.h"
#include "src/core/lib/slice/slice_buffer.h"
#include "src/core/lib/slice/slice_refcount.h" #include "src/core/lib/slice/slice_refcount.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
#include "src/cpp/ext/filters/census/context.h" #include "src/cpp/ext/filters/census/context.h"
@ -137,12 +137,12 @@ void OpenCensusCallTracer::OpenCensusCallAttemptTracer::
} }
void OpenCensusCallTracer::OpenCensusCallAttemptTracer::RecordSendMessage( void OpenCensusCallTracer::OpenCensusCallAttemptTracer::RecordSendMessage(
const grpc_core::SliceBuffer& /*send_message*/) { const grpc_core::ByteStream& /*send_message*/) {
++sent_message_count_; ++sent_message_count_;
} }
void OpenCensusCallTracer::OpenCensusCallAttemptTracer::RecordReceivedMessage( void OpenCensusCallTracer::OpenCensusCallAttemptTracer::RecordReceivedMessage(
const grpc_core::SliceBuffer& /*recv_message*/) { const grpc_core::ByteStream& /*recv_message*/) {
++recv_message_count_; ++recv_message_count_;
} }

@ -38,7 +38,7 @@
#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/error.h"
#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/arena.h"
#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport.h"
#include "src/cpp/ext/filters/census/context.h" #include "src/cpp/ext/filters/census/context.h"
@ -58,12 +58,12 @@ class OpenCensusCallTracer : public grpc_core::CallTracer {
void RecordSendTrailingMetadata( void RecordSendTrailingMetadata(
grpc_metadata_batch* /*send_trailing_metadata*/) override {} grpc_metadata_batch* /*send_trailing_metadata*/) override {}
void RecordSendMessage( void RecordSendMessage(
const grpc_core::SliceBuffer& /*send_message*/) override; const grpc_core::ByteStream& /*send_message*/) override;
void RecordReceivedInitialMetadata( void RecordReceivedInitialMetadata(
grpc_metadata_batch* /*recv_initial_metadata*/, grpc_metadata_batch* /*recv_initial_metadata*/,
uint32_t /*flags*/) override {} uint32_t /*flags*/) override {}
void RecordReceivedMessage( void RecordReceivedMessage(
const grpc_core::SliceBuffer& /*recv_message*/) override; const grpc_core::ByteStream& /*recv_message*/) override;
void RecordReceivedTrailingMetadata( void RecordReceivedTrailingMetadata(
absl::Status status, grpc_metadata_batch* recv_trailing_metadata, absl::Status status, grpc_metadata_batch* recv_trailing_metadata,
const grpc_transport_stream_stats* transport_stream_stats) override; const grpc_transport_stream_stats* transport_stream_stats) override;

@ -20,6 +20,7 @@
#include "src/cpp/ext/filters/census/server_filter.h" #include "src/cpp/ext/filters/census/server_filter.h"
#include <memory>
#include <utility> #include <utility>
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
@ -81,7 +82,7 @@ void CensusServerCallData::OnDoneRecvMessageCb(void* user_data,
GPR_ASSERT(calld != nullptr); GPR_ASSERT(calld != nullptr);
GPR_ASSERT(channeld != nullptr); GPR_ASSERT(channeld != nullptr);
// Stream messages are no longer valid after receiving trailing metadata. // Stream messages are no longer valid after receiving trailing metadata.
if (calld->recv_message_->has_value()) { if ((*calld->recv_message_) != nullptr) {
++calld->recv_message_count_; ++calld->recv_message_count_;
} }
grpc_core::Closure::Run(DEBUG_LOCATION, calld->initial_on_done_recv_message_, grpc_core::Closure::Run(DEBUG_LOCATION, calld->initial_on_done_recv_message_,

@ -28,7 +28,6 @@
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "absl/time/time.h" #include "absl/time/time.h"
#include "absl/types/optional.h"
#include <grpc/grpc_security.h> #include <grpc/grpc_security.h>
#include <grpc/impl/codegen/grpc_types.h> #include <grpc/impl/codegen/grpc_types.h>
@ -36,10 +35,11 @@
#include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_fwd.h"
#include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/channel_stack.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/closure.h"
#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/error.h"
#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice.h"
#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/byte_stream.h"
#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/metadata_batch.h"
#include "src/cpp/common/channel_filter.h" #include "src/cpp/common/channel_filter.h"
#include "src/cpp/ext/filters/census/context.h" #include "src/cpp/ext/filters/census/context.h"
@ -101,7 +101,7 @@ class CensusServerCallData : public CallData {
grpc_closure on_done_recv_message_; grpc_closure on_done_recv_message_;
absl::Time start_time_; absl::Time start_time_;
absl::Duration elapsed_time_; absl::Duration elapsed_time_;
absl::optional<grpc_core::SliceBuffer>* recv_message_; grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message_;
uint64_t recv_message_count_; uint64_t recv_message_count_;
uint64_t sent_message_count_; uint64_t sent_message_count_;
// Buffer needed for grpc_slice to reference it when adding metatdata to // Buffer needed for grpc_slice to reference it when adding metatdata to

@ -678,6 +678,7 @@ CORE_SOURCE_FILES = [
'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/validate_metadata.cc',
'src/core/lib/surface/version.cc', 'src/core/lib/surface/version.cc',
'src/core/lib/transport/bdp_estimator.cc', 'src/core/lib/transport/bdp_estimator.cc',
'src/core/lib/transport/byte_stream.cc',
'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/connectivity_state.cc',
'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/error_utils.cc',
'src/core/lib/transport/handshaker.cc', 'src/core/lib/transport/handshaker.cc',

@ -200,15 +200,24 @@ def _get_compression_ratios(client_function, first_channel_kwargs,
first_server_handler, second_channel_kwargs, first_server_handler, second_channel_kwargs,
second_multicallable_kwargs, second_server_kwargs, second_multicallable_kwargs, second_server_kwargs,
second_server_handler, message): second_server_handler, message):
first_bytes_sent, first_bytes_received = _get_byte_counts( try:
first_channel_kwargs, first_multicallable_kwargs, client_function, # This test requires the byte length of each connection to be deterministic. As
first_server_kwargs, first_server_handler, message) # it turns out, flow control puts bytes on the wire in a nondeterministic
second_bytes_sent, second_bytes_received = _get_byte_counts( # manner. We disable it here in order to measure compression ratios
second_channel_kwargs, second_multicallable_kwargs, client_function, # deterministically.
second_server_kwargs, second_server_handler, message) os.environ['GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL'] = 'true'
return ((second_bytes_sent - first_bytes_sent) / float(first_bytes_sent), first_bytes_sent, first_bytes_received = _get_byte_counts(
(second_bytes_received - first_bytes_received) / first_channel_kwargs, first_multicallable_kwargs, client_function,
float(first_bytes_received)) first_server_kwargs, first_server_handler, message)
second_bytes_sent, second_bytes_received = _get_byte_counts(
second_channel_kwargs, second_multicallable_kwargs, client_function,
second_server_kwargs, second_server_handler, message)
return ((second_bytes_sent - first_bytes_sent) /
float(first_bytes_sent),
(second_bytes_received - first_bytes_received) /
float(first_bytes_received))
finally:
del os.environ['GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL']
def _unary_unary_client(channel, multicallable_kwargs, message): def _unary_unary_client(channel, multicallable_kwargs, message):
@ -293,28 +302,37 @@ class CompressionTest(unittest.TestCase):
server_handler = _GenericHandler( server_handler = _GenericHandler(
functools.partial(set_call_compression, grpc.Compression.Gzip) functools.partial(set_call_compression, grpc.Compression.Gzip)
) if server_call_compression else _GenericHandler(None) ) if server_call_compression else _GenericHandler(None)
_get_compression_ratios(client_function, {}, {}, {}, sent_ratio, received_ratio = _get_compression_ratios(
_GenericHandler(None), channel_kwargs, client_function, {}, {}, {}, _GenericHandler(None), channel_kwargs,
multicallable_kwargs, server_kwargs, multicallable_kwargs, server_kwargs, server_handler, _REQUEST)
server_handler, _REQUEST)
if client_side_compressed:
self.assertCompressed(sent_ratio)
else:
self.assertNotCompressed(sent_ratio)
if server_side_compressed:
self.assertCompressed(received_ratio)
else:
self.assertNotCompressed(received_ratio)
def testDisableNextCompressionStreaming(self): def testDisableNextCompressionStreaming(self):
server_kwargs = { server_kwargs = {
'compression': grpc.Compression.Deflate, 'compression': grpc.Compression.Deflate,
} }
_get_compression_ratios(_stream_stream_client, {}, {}, {}, _, received_ratio = _get_compression_ratios(
_GenericHandler(None), {}, {}, server_kwargs, _stream_stream_client, {}, {}, {}, _GenericHandler(None), {}, {},
_GenericHandler(disable_next_compression), server_kwargs, _GenericHandler(disable_next_compression), _REQUEST)
_REQUEST) self.assertNotCompressed(received_ratio)
def testDisableNextCompressionStreamingResets(self): def testDisableNextCompressionStreamingResets(self):
server_kwargs = { server_kwargs = {
'compression': grpc.Compression.Deflate, 'compression': grpc.Compression.Deflate,
} }
_get_compression_ratios(_stream_stream_client, {}, {}, {}, _, received_ratio = _get_compression_ratios(
_GenericHandler(None), {}, {}, server_kwargs, _stream_stream_client, {}, {}, {}, _GenericHandler(None), {}, {},
_GenericHandler(disable_first_compression), server_kwargs, _GenericHandler(disable_first_compression), _REQUEST)
_REQUEST) self.assertCompressed(received_ratio)
def _get_compression_str(name, value): def _get_compression_str(name, value):

@ -90,7 +90,7 @@ static void verifier(grpc_server* server, grpc_completion_queue* cq,
nullptr); nullptr);
GPR_ASSERT(GRPC_CALL_OK == error); GPR_ASSERT(GRPC_CALL_OK == error);
CQ_EXPECT_COMPLETION_ANY_STATUS(cqv, tag(102)); CQ_EXPECT_COMPLETION(cqv, tag(102), 1);
cq_verify(cqv); cq_verify(cqv);
memset(ops, 0, sizeof(ops)); memset(ops, 0, sizeof(ops));

@ -1,118 +0,0 @@
/*
*
* Copyright 2015 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#include <string.h>
#include <grpc/grpc_security.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/sync.h>
#include "src/core/ext/filters/client_channel/client_channel.h"
#include "src/core/ext/filters/http/server/http_server_filter.h"
#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
#include "src/core/lib/channel/connected_channel.h"
#include "src/core/lib/gprpp/host_port.h"
#include "src/core/lib/surface/channel.h"
#include "src/core/lib/surface/server.h"
#include "test/core/end2end/end2end_tests.h"
#include "test/core/util/port.h"
#include "test/core/util/test_config.h"
struct fullstack_fixture_data {
std::string localaddr;
};
static grpc_end2end_test_fixture chttp2_create_fixture_fullstack(
const grpc_channel_args* /*client_args*/,
const grpc_channel_args* /*server_args*/) {
grpc_end2end_test_fixture f;
int port = grpc_pick_unused_port_or_die();
fullstack_fixture_data* ffd = new fullstack_fixture_data();
memset(&f, 0, sizeof(f));
ffd->localaddr = grpc_core::JoinHostPort("localhost", port);
f.fixture_data = ffd;
f.cq = grpc_completion_queue_create_for_next(nullptr);
return f;
}
void chttp2_init_client_fullstack(grpc_end2end_test_fixture* f,
const grpc_channel_args* client_args) {
fullstack_fixture_data* ffd =
static_cast<fullstack_fixture_data*>(f->fixture_data);
grpc_channel_credentials* creds = grpc_insecure_credentials_create();
grpc_arg no_retry = grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_ENABLE_RETRIES), 0);
client_args = grpc_channel_args_copy_and_add(client_args, &no_retry, 1);
f->client = grpc_channel_create(ffd->localaddr.c_str(), creds, client_args);
grpc_channel_args_destroy(client_args);
grpc_channel_credentials_release(creds);
GPR_ASSERT(f->client);
}
void chttp2_init_server_fullstack(grpc_end2end_test_fixture* f,
const grpc_channel_args* server_args) {
fullstack_fixture_data* ffd =
static_cast<fullstack_fixture_data*>(f->fixture_data);
if (f->server) {
grpc_server_destroy(f->server);
}
f->server = grpc_server_create(server_args, nullptr);
grpc_server_register_completion_queue(f->server, f->cq, nullptr);
grpc_server_credentials* server_creds =
grpc_insecure_server_credentials_create();
GPR_ASSERT(grpc_server_add_http2_port(f->server, ffd->localaddr.c_str(),
server_creds));
grpc_server_credentials_release(server_creds);
grpc_server_start(f->server);
}
void chttp2_tear_down_fullstack(grpc_end2end_test_fixture* f) {
fullstack_fixture_data* ffd =
static_cast<fullstack_fixture_data*>(f->fixture_data);
delete ffd;
}
/* All test configurations */
static grpc_end2end_test_config configs[] = {
{"chttp2/fullstack",
FEATURE_MASK_SUPPORTS_DELAYED_CONNECTION |
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL |
FEATURE_MASK_SUPPORTS_AUTHORITY_HEADER,
nullptr, chttp2_create_fixture_fullstack, chttp2_init_client_fullstack,
chttp2_init_server_fullstack, chttp2_tear_down_fullstack},
};
int main(int argc, char** argv) {
size_t i;
grpc::testing::TestEnvironment env(&argc, argv);
grpc_end2end_tests_pre_init();
grpc_init();
for (i = 0; i < sizeof(configs) / sizeof(*configs); i++) {
grpc_end2end_tests(argc, argv, configs[i]);
}
grpc_shutdown();
return 0;
}

@ -36,11 +36,8 @@ def _fixture_options(
supports_write_buffering = True, supports_write_buffering = True,
client_channel = True, client_channel = True,
supports_msvc = True, supports_msvc = True,
supports_retry = None,
flaky_tests = [], flaky_tests = [],
tags = []): tags = []):
if supports_retry == None:
supports_retry = client_channel
return struct( return struct(
fullstack = fullstack, fullstack = fullstack,
includes_proxy = includes_proxy, includes_proxy = includes_proxy,
@ -57,7 +54,6 @@ def _fixture_options(
supports_msvc = supports_msvc, supports_msvc = supports_msvc,
_platforms = _platforms, _platforms = _platforms,
flaky_tests = flaky_tests, flaky_tests = flaky_tests,
supports_retry = supports_retry,
tags = tags, tags = tags,
) )
@ -77,7 +73,6 @@ END2END_FIXTURES = {
tags = ["no_test_ios"], tags = ["no_test_ios"],
), ),
"h2_full": _fixture_options(), "h2_full": _fixture_options(),
"h2_full_no_retry": _fixture_options(supports_retry = False),
"h2_full+pipe": _fixture_options(_platforms = ["linux"]), "h2_full+pipe": _fixture_options(_platforms = ["linux"]),
"h2_full+trace": _fixture_options(tracing = True), "h2_full+trace": _fixture_options(tracing = True),
"h2_http_proxy": _fixture_options(supports_proxy_auth = True), "h2_http_proxy": _fixture_options(supports_proxy_auth = True),
@ -160,7 +155,6 @@ def _test_options(
needs_proxy_auth = False, needs_proxy_auth = False,
needs_write_buffering = False, needs_write_buffering = False,
needs_client_channel = False, needs_client_channel = False,
needs_retry = False,
short_name = None, short_name = None,
exclude_pollers = []): exclude_pollers = []):
return struct( return struct(
@ -176,7 +170,6 @@ def _test_options(
needs_proxy_auth = needs_proxy_auth, needs_proxy_auth = needs_proxy_auth,
needs_write_buffering = needs_write_buffering, needs_write_buffering = needs_write_buffering,
needs_client_channel = needs_client_channel, needs_client_channel = needs_client_channel,
needs_retry = needs_retry,
short_name = short_name, short_name = short_name,
exclude_pollers = exclude_pollers, exclude_pollers = exclude_pollers,
) )
@ -257,89 +250,81 @@ END2END_TESTS = {
"registered_call": _test_options(), "registered_call": _test_options(),
"request_with_flags": _test_options(proxyable = False), "request_with_flags": _test_options(proxyable = False),
"request_with_payload": _test_options(), "request_with_payload": _test_options(),
"retry": _test_options(needs_client_channel = True, needs_retry = True), "retry": _test_options(needs_client_channel = True),
"retry_cancellation": _test_options(needs_client_channel = True, needs_retry = True), "retry_cancellation": _test_options(needs_client_channel = True),
"retry_cancel_during_delay": _test_options(needs_client_channel = True, needs_retry = True), "retry_cancel_during_delay": _test_options(needs_client_channel = True),
"retry_cancel_with_multiple_send_batches": _test_options( "retry_cancel_with_multiple_send_batches": _test_options(
# TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE
# See b/151617965 # See b/151617965
short_name = "retry_cancel3", short_name = "retry_cancel3",
needs_client_channel = True, needs_client_channel = True,
needs_retry = True,
), ),
"retry_cancel_after_first_attempt_starts": _test_options( "retry_cancel_after_first_attempt_starts": _test_options(
# TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE
# See b/151617965 # See b/151617965
short_name = "retry_cancel4", short_name = "retry_cancel4",
needs_client_channel = True, needs_client_channel = True,
needs_retry = True,
), ),
"retry_disabled": _test_options(needs_client_channel = True, needs_retry = True), "retry_disabled": _test_options(needs_client_channel = True),
"retry_exceeds_buffer_size_in_delay": _test_options(needs_client_channel = True, needs_retry = True), "retry_exceeds_buffer_size_in_delay": _test_options(needs_client_channel = True),
"retry_exceeds_buffer_size_in_initial_batch": _test_options( "retry_exceeds_buffer_size_in_initial_batch": _test_options(
needs_client_channel = True, needs_client_channel = True,
# TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE
# See b/151617965 # See b/151617965
short_name = "retry_exceeds_buffer_size_in_init", short_name = "retry_exceeds_buffer_size_in_init",
needs_retry = True,
), ),
"retry_exceeds_buffer_size_in_subsequent_batch": _test_options( "retry_exceeds_buffer_size_in_subsequent_batch": _test_options(
needs_client_channel = True, needs_client_channel = True,
# TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE
# See b/151617965 # See b/151617965
short_name = "retry_exceeds_buffer_size_in_subseq", short_name = "retry_exceeds_buffer_size_in_subseq",
needs_retry = True,
), ),
"retry_lb_drop": _test_options(needs_client_channel = True, needs_retry = True), "retry_lb_drop": _test_options(needs_client_channel = True),
"retry_lb_fail": _test_options(needs_client_channel = True, needs_retry = True), "retry_lb_fail": _test_options(needs_client_channel = True),
"retry_non_retriable_status": _test_options(needs_client_channel = True, needs_retry = True), "retry_non_retriable_status": _test_options(needs_client_channel = True),
"retry_non_retriable_status_before_recv_trailing_metadata_started": _test_options( "retry_non_retriable_status_before_recv_trailing_metadata_started": _test_options(
needs_client_channel = True, needs_client_channel = True,
# TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE
# See b/151617965 # See b/151617965
short_name = "retry_non_retriable_status2", short_name = "retry_non_retriable_status2",
needs_retry = True,
), ),
"retry_per_attempt_recv_timeout": _test_options(needs_client_channel = True, needs_retry = True), "retry_per_attempt_recv_timeout": _test_options(needs_client_channel = True),
"retry_per_attempt_recv_timeout_on_last_attempt": _test_options( "retry_per_attempt_recv_timeout_on_last_attempt": _test_options(
needs_client_channel = True, needs_client_channel = True,
# TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE
# See b/151617965 # See b/151617965
short_name = "retry_per_attempt_recv_timeout2", short_name = "retry_per_attempt_recv_timeout2",
needs_retry = True,
), ),
"retry_recv_initial_metadata": _test_options(needs_client_channel = True, needs_retry = True), "retry_recv_initial_metadata": _test_options(needs_client_channel = True),
"retry_recv_message": _test_options(needs_client_channel = True, needs_retry = True), "retry_recv_message": _test_options(needs_client_channel = True),
"retry_recv_message_replay": _test_options(needs_client_channel = True, needs_retry = True), "retry_recv_message_replay": _test_options(needs_client_channel = True),
"retry_recv_trailing_metadata_error": _test_options(needs_client_channel = True, needs_retry = True), "retry_recv_trailing_metadata_error": _test_options(needs_client_channel = True),
"retry_send_initial_metadata_refs": _test_options(needs_client_channel = True, needs_retry = True), "retry_send_initial_metadata_refs": _test_options(needs_client_channel = True),
"retry_send_op_fails": _test_options(needs_client_channel = True, needs_retry = True), "retry_send_op_fails": _test_options(needs_client_channel = True),
"retry_send_recv_batch": _test_options(needs_client_channel = True, needs_retry = True), "retry_send_recv_batch": _test_options(needs_client_channel = True),
"retry_server_pushback_delay": _test_options(needs_client_channel = True, needs_retry = True), "retry_server_pushback_delay": _test_options(needs_client_channel = True),
"retry_server_pushback_disabled": _test_options(needs_client_channel = True, needs_retry = True), "retry_server_pushback_disabled": _test_options(needs_client_channel = True),
"retry_streaming": _test_options(needs_client_channel = True, needs_retry = True), "retry_streaming": _test_options(needs_client_channel = True),
"retry_streaming_after_commit": _test_options(needs_client_channel = True, needs_retry = True), "retry_streaming_after_commit": _test_options(needs_client_channel = True),
"retry_streaming_succeeds_before_replay_finished": _test_options( "retry_streaming_succeeds_before_replay_finished": _test_options(
needs_client_channel = True, needs_client_channel = True,
# TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE
# See b/151617965 # See b/151617965
short_name = "retry_streaming2", short_name = "retry_streaming2",
needs_retry = True,
), ),
"retry_throttled": _test_options(needs_client_channel = True, needs_retry = True), "retry_throttled": _test_options(needs_client_channel = True),
"retry_too_many_attempts": _test_options(needs_client_channel = True, needs_retry = True), "retry_too_many_attempts": _test_options(needs_client_channel = True),
"retry_transparent_goaway": _test_options(needs_client_channel = True, needs_retry = True), "retry_transparent_goaway": _test_options(needs_client_channel = True),
"retry_transparent_not_sent_on_wire": _test_options(needs_client_channel = True, needs_retry = True), "retry_transparent_not_sent_on_wire": _test_options(needs_client_channel = True),
"retry_transparent_max_concurrent_streams": _test_options( "retry_transparent_max_concurrent_streams": _test_options(
needs_client_channel = True, needs_client_channel = True,
proxyable = False, proxyable = False,
# TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE
# See b/151617965 # See b/151617965
short_name = "retry_transparent_mcs", short_name = "retry_transparent_mcs",
needs_retry = True,
), ),
"retry_unref_before_finish": _test_options(needs_client_channel = True, needs_retry = True), "retry_unref_before_finish": _test_options(needs_client_channel = True),
"retry_unref_before_recv": _test_options(needs_client_channel = True, needs_retry = True), "retry_unref_before_recv": _test_options(needs_client_channel = True),
"server_finishes_request": _test_options(), "server_finishes_request": _test_options(),
"server_streaming": _test_options(needs_http2 = True), "server_streaming": _test_options(needs_http2 = True),
"shutdown_finishes_calls": _test_options(), "shutdown_finishes_calls": _test_options(),
@ -390,9 +375,6 @@ def _compatible(fopt, topt):
if topt.needs_client_channel: if topt.needs_client_channel:
if not fopt.client_channel: if not fopt.client_channel:
return False return False
if topt.needs_retry:
if not fopt.supports_retry:
return False
return True return True
def _platform_support_tags(fopt): def _platform_support_tags(fopt):

@ -25,7 +25,7 @@
#include <grpc/support/time.h> #include <grpc/support/time.h>
#include "src/core/lib/gpr/useful.h" #include "src/core/lib/gpr/useful.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/byte_stream.h"
#include "test/core/end2end/cq_verifier.h" #include "test/core/end2end/cq_verifier.h"
#include "test/core/end2end/end2end_tests.h" #include "test/core/end2end/end2end_tests.h"

@ -84,6 +84,18 @@ grpc_cc_test(
], ],
) )
grpc_cc_test(
name = "manual_constructor_test",
srcs = ["manual_constructor_test.cc"],
language = "C++",
uses_event_engine = False,
uses_polling = False,
deps = [
"//:gpr",
"//test/core/util:grpc_test_util",
],
)
grpc_cc_test( grpc_cc_test(
name = "bitset_test", name = "bitset_test",
srcs = ["bitset_test.cc"], srcs = ["bitset_test.cc"],

@ -0,0 +1,100 @@
/*
*
* Copyright 2017 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
/* Test of gpr synchronization support. */
#include "src/core/lib/gprpp/manual_constructor.h"
#include <stdio.h>
#include <stdlib.h>
#include <cstring>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/sync.h>
#include "test/core/util/test_config.h"
class A {
public:
A() {}
virtual ~A() {}
virtual const char* foo() { return "A_foo"; }
virtual const char* bar() { return "A_bar"; }
};
class B : public A {
public:
B() {}
~B() override {}
const char* foo() override { return "B_foo"; }
char get_junk() { return junk[0]; }
private:
char junk[1000];
};
class C : public B {
public:
C() {}
~C() override {}
const char* bar() override { return "C_bar"; }
char get_more_junk() { return more_junk[0]; }
private:
char more_junk[1000];
};
class D : public A {
public:
const char* bar() override { return "D_bar"; }
};
static void basic_test() {
grpc_core::PolymorphicManualConstructor<A, B> poly;
poly.Init<B>();
GPR_ASSERT(!strcmp(poly->foo(), "B_foo"));
GPR_ASSERT(!strcmp(poly->bar(), "A_bar"));
}
static void complex_test() {
grpc_core::PolymorphicManualConstructor<A, B, C, D> polyB;
polyB.Init<B>();
GPR_ASSERT(!strcmp(polyB->foo(), "B_foo"));
GPR_ASSERT(!strcmp(polyB->bar(), "A_bar"));
grpc_core::PolymorphicManualConstructor<A, B, C, D> polyC;
polyC.Init<C>();
GPR_ASSERT(!strcmp(polyC->foo(), "B_foo"));
GPR_ASSERT(!strcmp(polyC->bar(), "C_bar"));
grpc_core::PolymorphicManualConstructor<A, B, C, D> polyD;
polyD.Init<D>();
GPR_ASSERT(!strcmp(polyD->foo(), "A_foo"));
GPR_ASSERT(!strcmp(polyD->bar(), "D_bar"));
}
/* ------------------------------------------------- */
int main(int argc, char* argv[]) {
grpc::testing::TestEnvironment env(&argc, argv);
basic_test();
complex_test();
return 0;
}

@ -35,6 +35,22 @@ grpc_cc_test(
], ],
) )
grpc_cc_test(
name = "byte_stream_test",
srcs = ["byte_stream_test.cc"],
external_deps = [
"gtest",
],
language = "C++",
uses_event_engine = False,
uses_polling = False,
deps = [
"//:gpr",
"//:grpc",
"//test/core/util:grpc_test_util",
],
)
grpc_cc_test( grpc_cc_test(
name = "connectivity_state_test", name = "connectivity_state_test",
srcs = ["connectivity_state_test.cc"], srcs = ["connectivity_state_test.cc"],

@ -242,13 +242,19 @@ struct MakeSendInitialMetadata {
struct MakeSendMessage { struct MakeSendMessage {
MakeSendMessage(const std::string& message, MakeSendMessage(const std::string& message,
grpc_transport_stream_op_batch* op) { grpc_transport_stream_op_batch* op) {
send_stream.Append(grpc_core::Slice::FromCopiedString(message)); grpc_slice_buffer send_buffer;
grpc_slice_buffer_init(&send_buffer);
grpc_slice send_slice = grpc_slice_from_cpp_string(message);
grpc_slice_buffer_add(&send_buffer, send_slice);
send_stream.Init(&send_buffer, 0);
grpc_slice_buffer_destroy(&send_buffer);
op->send_message = true; op->send_message = true;
op->payload->send_message.send_message = &send_stream; op->payload->send_message.send_message.reset(send_stream.get());
} }
grpc_core::SliceBuffer send_stream; grpc_core::ManualConstructor<grpc_core::SliceBufferByteStream> send_stream;
}; };
struct MakeSendTrailingMetadata { struct MakeSendTrailingMetadata {
@ -307,7 +313,7 @@ struct MakeRecvMessage {
MockGrpcClosure ready; MockGrpcClosure ready;
absl::Notification notification; absl::Notification notification;
absl::optional<grpc_core::SliceBuffer> grpc_message; grpc_core::OrphanablePtr<grpc_core::ByteStream> grpc_message;
}; };
struct MakeRecvTrailingMetadata { struct MakeRecvTrailingMetadata {
@ -559,7 +565,13 @@ TEST_F(BinderTransportTest, PerformRecvMessage) {
grpc_core::ExecCtx::Get()->Flush(); grpc_core::ExecCtx::Get()->Flush();
recv_message.notification.WaitForNotification(); recv_message.notification.WaitForNotification();
EXPECT_EQ(kMessage, recv_message.grpc_message->JoinIntoString()); EXPECT_TRUE(recv_message.grpc_message->Next(SIZE_MAX, nullptr));
grpc_slice slice;
EXPECT_EQ(recv_message.grpc_message->Pull(&slice), GRPC_ERROR_NONE);
EXPECT_EQ(kMessage,
std::string(reinterpret_cast<char*>(GRPC_SLICE_START_PTR(slice)),
GRPC_SLICE_LENGTH(slice)));
grpc_slice_unref_internal(slice);
} }
TEST_F(BinderTransportTest, PerformRecvTrailingMetadata) { TEST_F(BinderTransportTest, PerformRecvTrailingMetadata) {
@ -617,7 +629,13 @@ TEST_F(BinderTransportTest, PerformRecvAll) {
trailing_metadata.emplace_back("grpc-status", std::to_string(kStatus)); trailing_metadata.emplace_back("grpc-status", std::to_string(kStatus));
VerifyMetadataEqual(trailing_metadata, VerifyMetadataEqual(trailing_metadata,
recv_trailing_metadata.grpc_trailing_metadata); recv_trailing_metadata.grpc_trailing_metadata);
EXPECT_EQ(kMessage, recv_message.grpc_message->JoinIntoString()); EXPECT_TRUE(recv_message.grpc_message->Next(SIZE_MAX, nullptr));
grpc_slice slice;
EXPECT_EQ(recv_message.grpc_message->Pull(&slice), GRPC_ERROR_NONE);
EXPECT_EQ(kMessage,
std::string(reinterpret_cast<char*>(GRPC_SLICE_START_PTR(slice)),
GRPC_SLICE_LENGTH(slice)));
grpc_slice_unref_internal(slice);
} }
TEST_F(BinderTransportTest, PerformAllOps) { TEST_F(BinderTransportTest, PerformAllOps) {
@ -688,7 +706,13 @@ TEST_F(BinderTransportTest, PerformAllOps) {
VerifyMetadataEqual(AppendStatus(kRecvTrailingMetadata, kStatus), VerifyMetadataEqual(AppendStatus(kRecvTrailingMetadata, kStatus),
recv_trailing_metadata.grpc_trailing_metadata); recv_trailing_metadata.grpc_trailing_metadata);
EXPECT_EQ(kRecvMessage, recv_message.grpc_message->JoinIntoString()); EXPECT_TRUE(recv_message.grpc_message->Next(SIZE_MAX, nullptr));
grpc_slice slice;
EXPECT_EQ(recv_message.grpc_message->Pull(&slice), GRPC_ERROR_NONE);
EXPECT_EQ(kRecvMessage,
std::string(reinterpret_cast<char*>(GRPC_SLICE_START_PTR(slice)),
GRPC_SLICE_LENGTH(slice)));
grpc_slice_unref_internal(slice);
} }
TEST_F(BinderTransportTest, WireWriterRpcCallErrorPropagates) { TEST_F(BinderTransportTest, WireWriterRpcCallErrorPropagates) {

@ -0,0 +1,254 @@
/*
*
* Copyright 2017 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#include "src/core/lib/transport/byte_stream.h"
#include <gtest/gtest.h>
#include <grpc/grpc.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include "src/core/lib/gpr/useful.h"
#include "src/core/lib/iomgr/exec_ctx.h"
#include "src/core/lib/slice/slice_internal.h"
#include "test/core/util/test_config.h"
namespace grpc_core {
namespace {
//
// SliceBufferByteStream tests
//
void NotCalledClosure(void* /*arg*/, grpc_error_handle /*error*/) {
GPR_ASSERT(false);
}
TEST(SliceBufferByteStream, Basic) {
ExecCtx exec_ctx;
// Create and populate slice buffer.
grpc_slice_buffer buffer;
grpc_slice_buffer_init(&buffer);
grpc_slice input[] = {
grpc_slice_from_static_string("foo"),
grpc_slice_from_static_string("bar"),
};
for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) {
grpc_slice_buffer_add(&buffer, input[i]);
}
// Create byte stream.
SliceBufferByteStream stream(&buffer, 0);
grpc_slice_buffer_destroy_internal(&buffer);
EXPECT_EQ(6U, stream.length());
grpc_closure closure;
GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr,
grpc_schedule_on_exec_ctx);
// Read each slice. Note that Next() always returns synchronously.
for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) {
ASSERT_TRUE(stream.Next(~(size_t)0, &closure));
grpc_slice output;
grpc_error_handle error = stream.Pull(&output);
EXPECT_TRUE(error == GRPC_ERROR_NONE);
EXPECT_TRUE(grpc_slice_eq(input[i], output));
grpc_slice_unref_internal(output);
}
// Clean up.
stream.Orphan();
}
TEST(SliceBufferByteStream, Shutdown) {
ExecCtx exec_ctx;
// Create and populate slice buffer.
grpc_slice_buffer buffer;
grpc_slice_buffer_init(&buffer);
grpc_slice input[] = {
grpc_slice_from_static_string("foo"),
grpc_slice_from_static_string("bar"),
};
for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) {
grpc_slice_buffer_add(&buffer, input[i]);
}
// Create byte stream.
SliceBufferByteStream stream(&buffer, 0);
grpc_slice_buffer_destroy_internal(&buffer);
EXPECT_EQ(6U, stream.length());
grpc_closure closure;
GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr,
grpc_schedule_on_exec_ctx);
// Read the first slice.
ASSERT_TRUE(stream.Next(~(size_t)0, &closure));
grpc_slice output;
grpc_error_handle error = stream.Pull(&output);
EXPECT_TRUE(error == GRPC_ERROR_NONE);
EXPECT_TRUE(grpc_slice_eq(input[0], output));
grpc_slice_unref_internal(output);
// Now shutdown.
grpc_error_handle shutdown_error =
GRPC_ERROR_CREATE_FROM_STATIC_STRING("shutdown error");
stream.Shutdown(GRPC_ERROR_REF(shutdown_error));
// After shutdown, the next pull() should return the error.
ASSERT_TRUE(stream.Next(~(size_t)0, &closure));
error = stream.Pull(&output);
EXPECT_TRUE(error == shutdown_error);
GRPC_ERROR_UNREF(error);
GRPC_ERROR_UNREF(shutdown_error);
// Clean up.
stream.Orphan();
}
//
// CachingByteStream tests
//
TEST(CachingByteStream, Basic) {
ExecCtx exec_ctx;
// Create and populate slice buffer byte stream.
grpc_slice_buffer buffer;
grpc_slice_buffer_init(&buffer);
grpc_slice input[] = {
grpc_slice_from_static_string("foo"),
grpc_slice_from_static_string("bar"),
};
for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) {
grpc_slice_buffer_add(&buffer, input[i]);
}
SliceBufferByteStream underlying_stream(&buffer, 0);
grpc_slice_buffer_destroy_internal(&buffer);
// Create cache and caching stream.
ByteStreamCache cache((OrphanablePtr<ByteStream>(&underlying_stream)));
ByteStreamCache::CachingByteStream stream(&cache);
grpc_closure closure;
GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr,
grpc_schedule_on_exec_ctx);
// Read each slice. Note that next() always returns synchronously,
// because the underlying byte stream always does.
for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) {
ASSERT_TRUE(stream.Next(~(size_t)0, &closure));
grpc_slice output;
grpc_error_handle error = stream.Pull(&output);
EXPECT_TRUE(error == GRPC_ERROR_NONE);
EXPECT_TRUE(grpc_slice_eq(input[i], output));
grpc_slice_unref_internal(output);
}
// Clean up.
stream.Orphan();
cache.Destroy();
}
TEST(CachingByteStream, Reset) {
ExecCtx exec_ctx;
// Create and populate slice buffer byte stream.
grpc_slice_buffer buffer;
grpc_slice_buffer_init(&buffer);
grpc_slice input[] = {
grpc_slice_from_static_string("foo"),
grpc_slice_from_static_string("bar"),
};
for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) {
grpc_slice_buffer_add(&buffer, input[i]);
}
SliceBufferByteStream underlying_stream(&buffer, 0);
grpc_slice_buffer_destroy_internal(&buffer);
// Create cache and caching stream.
ByteStreamCache cache((OrphanablePtr<ByteStream>(&underlying_stream)));
ByteStreamCache::CachingByteStream stream(&cache);
grpc_closure closure;
GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr,
grpc_schedule_on_exec_ctx);
// Read one slice.
ASSERT_TRUE(stream.Next(~(size_t)0, &closure));
grpc_slice output;
grpc_error_handle error = stream.Pull(&output);
EXPECT_TRUE(error == GRPC_ERROR_NONE);
EXPECT_TRUE(grpc_slice_eq(input[0], output));
grpc_slice_unref_internal(output);
// Reset the caching stream. The reads should start over from the
// first slice.
stream.Reset();
for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) {
ASSERT_TRUE(stream.Next(~(size_t)0, &closure));
error = stream.Pull(&output);
EXPECT_TRUE(error == GRPC_ERROR_NONE);
EXPECT_TRUE(grpc_slice_eq(input[i], output));
grpc_slice_unref_internal(output);
}
// Clean up.
stream.Orphan();
cache.Destroy();
}
TEST(CachingByteStream, SharedCache) {
ExecCtx exec_ctx;
// Create and populate slice buffer byte stream.
grpc_slice_buffer buffer;
grpc_slice_buffer_init(&buffer);
grpc_slice input[] = {
grpc_slice_from_static_string("foo"),
grpc_slice_from_static_string("bar"),
};
for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) {
grpc_slice_buffer_add(&buffer, input[i]);
}
SliceBufferByteStream underlying_stream(&buffer, 0);
grpc_slice_buffer_destroy_internal(&buffer);
// Create cache and two caching streams.
ByteStreamCache cache((OrphanablePtr<ByteStream>(&underlying_stream)));
ByteStreamCache::CachingByteStream stream1(&cache);
ByteStreamCache::CachingByteStream stream2(&cache);
grpc_closure closure;
GRPC_CLOSURE_INIT(&closure, NotCalledClosure, nullptr,
grpc_schedule_on_exec_ctx);
// Read one slice from stream1.
EXPECT_TRUE(stream1.Next(~(size_t)0, &closure));
grpc_slice output;
grpc_error_handle error = stream1.Pull(&output);
EXPECT_TRUE(error == GRPC_ERROR_NONE);
EXPECT_TRUE(grpc_slice_eq(input[0], output));
grpc_slice_unref_internal(output);
// Read all slices from stream2.
for (size_t i = 0; i < GPR_ARRAY_SIZE(input); ++i) {
EXPECT_TRUE(stream2.Next(~(size_t)0, &closure));
error = stream2.Pull(&output);
EXPECT_TRUE(error == GRPC_ERROR_NONE);
EXPECT_TRUE(grpc_slice_eq(input[i], output));
grpc_slice_unref_internal(output);
}
// Now read the second slice from stream1.
EXPECT_TRUE(stream1.Next(~(size_t)0, &closure));
error = stream1.Pull(&output);
EXPECT_TRUE(error == GRPC_ERROR_NONE);
EXPECT_TRUE(grpc_slice_eq(input[1], output));
grpc_slice_unref_internal(output);
// Clean up.
stream1.Orphan();
stream2.Orphan();
cache.Destroy();
}
} // namespace
} // namespace grpc_core
int main(int argc, char** argv) {
grpc::testing::TestEnvironment env(&argc, argv);
::testing::InitGoogleTest(&argc, argv);
grpc_init();
int retval = RUN_ALL_TESTS();
grpc_shutdown();
return retval;
}

@ -86,9 +86,9 @@ grpc_cc_test(
) )
grpc_cc_test( grpc_cc_test(
name = "flow_control_end2end_test", name = "flow_control_test",
size = "large", size = "large",
srcs = ["flow_control_end2end_test.cc"], srcs = ["flow_control_test.cc"],
external_deps = [ external_deps = [
"gtest", "gtest",
], ],
@ -101,20 +101,6 @@ grpc_cc_test(
], ],
) )
grpc_cc_test(
name = "flow_control_test",
srcs = ["flow_control_test.cc"],
external_deps = [
"gtest",
],
language = "C++",
deps = [
"//:chttp2_flow_control",
"//:resource_quota",
"//test/core/util:grpc_suppressions",
],
)
grpc_cc_test( grpc_cc_test(
name = "graceful_shutdown_test", name = "graceful_shutdown_test",
srcs = ["graceful_shutdown_test.cc"], srcs = ["graceful_shutdown_test.cc"],

@ -1,379 +0,0 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#include <grpc/support/port_platform.h>
#include <stdlib.h>
#include <string.h>
#include <functional>
#include <set>
#include <thread>
#include <gmock/gmock.h>
#include <grpc/grpc.h>
#include <grpc/grpc_security.h>
#include <grpc/impl/codegen/grpc_types.h>
#include <grpc/slice.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/string_util.h>
#include <grpc/support/time.h>
#include "src/core/ext/filters/client_channel/backup_poller.h"
#include "src/core/ext/transport/chttp2/transport/flow_control.h"
#include "src/core/lib/channel/channel_args.h"
#include "src/core/lib/gprpp/host_port.h"
#include "src/core/lib/surface/channel.h"
#include "test/core/end2end/cq_verifier.h"
#include "test/core/util/port.h"
#include "test/core/util/test_config.h"
namespace {
class TransportTargetWindowSizeMocker
: public grpc_core::chttp2::TestOnlyTransportTargetWindowEstimatesMocker {
public:
static constexpr uint32_t kLargeInitialWindowSize = 1u << 31;
static constexpr uint32_t kSmallInitialWindowSize = 0;
double ComputeNextTargetInitialWindowSizeFromPeriodicUpdate(
double /* current_target */) override {
// Protecting access to variable window_size_ shared between client and
// server.
grpc_core::MutexLock lock(&mu_);
if (alternating_initial_window_sizes_) {
window_size_ = (window_size_ == kLargeInitialWindowSize)
? kSmallInitialWindowSize
: kLargeInitialWindowSize;
}
return window_size_;
}
// Alternates the initial window size targets. Computes a low values if it was
// previously high, or a high value if it was previously low.
void AlternateTargetInitialWindowSizes() {
grpc_core::MutexLock lock(&mu_);
alternating_initial_window_sizes_ = true;
}
void Reset() {
// Protecting access to variable window_size_ shared between client and
// server.
grpc_core::MutexLock lock(&mu_);
alternating_initial_window_sizes_ = false;
window_size_ = kLargeInitialWindowSize;
}
private:
grpc_core::Mutex mu_;
bool alternating_initial_window_sizes_ ABSL_GUARDED_BY(mu_) = false;
double window_size_ ABSL_GUARDED_BY(mu_) = kLargeInitialWindowSize;
};
TransportTargetWindowSizeMocker* g_target_initial_window_size_mocker;
void* tag(intptr_t t) { return reinterpret_cast<void*>(t); }
void VerifyChannelReady(grpc_channel* channel, grpc_completion_queue* cq) {
grpc_connectivity_state state =
grpc_channel_check_connectivity_state(channel, 1 /* try_to_connect */);
while (state != GRPC_CHANNEL_READY) {
grpc_channel_watch_connectivity_state(
channel, state, grpc_timeout_seconds_to_deadline(5), cq, nullptr);
grpc_completion_queue_next(cq, grpc_timeout_seconds_to_deadline(5),
nullptr);
state = grpc_channel_check_connectivity_state(channel, 0);
}
}
void VerifyChannelConnected(grpc_channel* channel, grpc_completion_queue* cq) {
// Verify channel is connected. Use a ping to make sure that clients
// tries sending/receiving bytes if the channel is connected.
grpc_channel_ping(channel, cq, reinterpret_cast<void*>(2000), nullptr);
grpc_event ev = grpc_completion_queue_next(
cq, grpc_timeout_seconds_to_deadline(5), nullptr);
GPR_ASSERT(ev.type == GRPC_OP_COMPLETE);
GPR_ASSERT(ev.tag == reinterpret_cast<void*>(2000));
GPR_ASSERT(ev.success == 1);
GPR_ASSERT(grpc_channel_check_connectivity_state(channel, 0) ==
GRPC_CHANNEL_READY);
}
// Shuts down and destroys the server.
void ServerShutdownAndDestroy(grpc_server* server, grpc_completion_queue* cq) {
// Shutdown and destroy server
grpc_server_shutdown_and_notify(server, cq, reinterpret_cast<void*>(1000));
while (grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME),
nullptr)
.tag != reinterpret_cast<void*>(1000)) {
}
grpc_server_destroy(server);
}
grpc_slice LargeSlice(void) {
grpc_slice slice = grpc_slice_malloc(10000000); // ~10MB
memset(GRPC_SLICE_START_PTR(slice), 'x', GRPC_SLICE_LENGTH(slice));
return slice;
}
void PerformCallWithLargePayload(grpc_channel* channel, grpc_server* server,
grpc_completion_queue* cq) {
grpc_slice request_payload_slice = LargeSlice();
grpc_slice response_payload_slice = LargeSlice();
grpc_call* c;
grpc_call* s;
grpc_byte_buffer* request_payload =
grpc_raw_byte_buffer_create(&request_payload_slice, 1);
grpc_byte_buffer* response_payload =
grpc_raw_byte_buffer_create(&response_payload_slice, 1);
cq_verifier* cqv = cq_verifier_create(cq);
grpc_op ops[6];
grpc_op* op;
grpc_metadata_array initial_metadata_recv;
grpc_metadata_array trailing_metadata_recv;
grpc_metadata_array request_metadata_recv;
grpc_byte_buffer* request_payload_recv = nullptr;
grpc_byte_buffer* response_payload_recv = nullptr;
grpc_call_details call_details;
grpc_status_code status;
grpc_call_error error;
grpc_slice details;
int was_cancelled = 2;
gpr_timespec deadline = grpc_timeout_seconds_to_deadline(30);
c = grpc_channel_create_call(channel, nullptr, GRPC_PROPAGATE_DEFAULTS, cq,
grpc_slice_from_static_string("/foo"), nullptr,
deadline, nullptr);
GPR_ASSERT(c);
grpc_metadata_array_init(&initial_metadata_recv);
grpc_metadata_array_init(&trailing_metadata_recv);
grpc_metadata_array_init(&request_metadata_recv);
grpc_call_details_init(&call_details);
memset(ops, 0, sizeof(ops));
op = ops;
op->op = GRPC_OP_SEND_INITIAL_METADATA;
op->data.send_initial_metadata.count = 0;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_SEND_MESSAGE;
op->data.send_message.send_message = request_payload;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_INITIAL_METADATA;
op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_MESSAGE;
op->data.recv_message.recv_message = &response_payload_recv;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
op->data.recv_status_on_client.status = &status;
op->data.recv_status_on_client.status_details = &details;
op->flags = 0;
op->reserved = nullptr;
op++;
error = grpc_call_start_batch(c, ops, static_cast<size_t>(op - ops), tag(1),
nullptr);
GPR_ASSERT(GRPC_CALL_OK == error);
error = grpc_server_request_call(server, &s, &call_details,
&request_metadata_recv, cq, cq, tag(101));
GPR_ASSERT(GRPC_CALL_OK == error);
CQ_EXPECT_COMPLETION(cqv, tag(101), 1);
cq_verify(cqv);
memset(ops, 0, sizeof(ops));
op = ops;
op->op = GRPC_OP_SEND_INITIAL_METADATA;
op->data.send_initial_metadata.count = 0;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_MESSAGE;
op->data.recv_message.recv_message = &request_payload_recv;
op->flags = 0;
op->reserved = nullptr;
op++;
error = grpc_call_start_batch(s, ops, static_cast<size_t>(op - ops), tag(102),
nullptr);
GPR_ASSERT(GRPC_CALL_OK == error);
CQ_EXPECT_COMPLETION(cqv, tag(102), 1);
cq_verify(cqv);
memset(ops, 0, sizeof(ops));
op = ops;
op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
op->data.recv_close_on_server.cancelled = &was_cancelled;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_SEND_MESSAGE;
op->data.send_message.send_message = response_payload;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
op->data.send_status_from_server.trailing_metadata_count = 0;
op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED;
grpc_slice status_details = grpc_slice_from_static_string("xyz");
op->data.send_status_from_server.status_details = &status_details;
op->flags = 0;
op->reserved = nullptr;
op++;
error = grpc_call_start_batch(s, ops, static_cast<size_t>(op - ops), tag(103),
nullptr);
GPR_ASSERT(GRPC_CALL_OK == error);
CQ_EXPECT_COMPLETION(cqv, tag(103), 1);
CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
cq_verify(cqv);
GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED);
GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz"));
GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo"));
GPR_ASSERT(was_cancelled == 0);
grpc_slice_unref(details);
grpc_metadata_array_destroy(&initial_metadata_recv);
grpc_metadata_array_destroy(&trailing_metadata_recv);
grpc_metadata_array_destroy(&request_metadata_recv);
grpc_call_details_destroy(&call_details);
grpc_call_unref(c);
grpc_call_unref(s);
cq_verifier_destroy(cqv);
grpc_byte_buffer_destroy(request_payload);
grpc_byte_buffer_destroy(response_payload);
grpc_byte_buffer_destroy(request_payload_recv);
grpc_byte_buffer_destroy(response_payload_recv);
grpc_slice_unref(request_payload_slice);
grpc_slice_unref(response_payload_slice);
}
class FlowControlTest : public ::testing::Test {
protected:
void SetUp() override {
cq_ = grpc_completion_queue_create_for_next(nullptr);
// create the server
std::string server_address =
grpc_core::JoinHostPort("localhost", grpc_pick_unused_port_or_die());
grpc_arg server_args[] = {
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_HTTP2_MAX_PING_STRIKES), 0),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), -1),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH), -1)};
grpc_channel_args server_channel_args = {GPR_ARRAY_SIZE(server_args),
server_args};
server_ = grpc_server_create(&server_channel_args, nullptr);
grpc_server_register_completion_queue(server_, cq_, nullptr);
grpc_server_credentials* server_creds =
grpc_insecure_server_credentials_create();
GPR_ASSERT(grpc_server_add_http2_port(server_, server_address.c_str(),
server_creds));
grpc_server_credentials_release(server_creds);
grpc_server_start(server_);
// create the channel (bdp pings are enabled by default)
grpc_arg client_args[] = {
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA), 0),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS), 1),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), -1),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH), -1)};
grpc_channel_args client_channel_args = {GPR_ARRAY_SIZE(client_args),
client_args};
grpc_channel_credentials* creds = grpc_insecure_credentials_create();
channel_ = grpc_channel_create(server_address.c_str(), creds,
&client_channel_args);
grpc_channel_credentials_release(creds);
VerifyChannelReady(channel_, cq_);
g_target_initial_window_size_mocker->Reset();
}
void TearDown() override {
// shutdown and destroy the client and server
grpc_channel_destroy(channel_);
ServerShutdownAndDestroy(server_, cq_);
grpc_completion_queue_shutdown(cq_);
while (grpc_completion_queue_next(cq_, gpr_inf_future(GPR_CLOCK_REALTIME),
nullptr)
.type != GRPC_QUEUE_SHUTDOWN) {
}
grpc_completion_queue_destroy(cq_);
}
grpc_server* server_ = nullptr;
grpc_channel* channel_ = nullptr;
grpc_completion_queue* cq_ = nullptr;
};
TEST_F(FlowControlTest,
TestLargeWindowSizeUpdatesDoNotCauseIllegalFlowControlWindows) {
for (int i = 0; i < 10; ++i) {
PerformCallWithLargePayload(channel_, server_, cq_);
VerifyChannelConnected(channel_, cq_);
}
}
TEST_F(FlowControlTest, TestWindowSizeUpdatesDoNotCauseStalledStreams) {
g_target_initial_window_size_mocker->AlternateTargetInitialWindowSizes();
for (int i = 0; i < 100; ++i) {
PerformCallWithLargePayload(channel_, server_, cq_);
VerifyChannelConnected(channel_, cq_);
}
}
} // namespace
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
// Make sure that we will have an active poller on all client-side fd's that
// are capable of sending and receiving even in the case that we don't have an
// active RPC operation on the fd.
GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 1);
grpc_core::chttp2::g_test_only_transport_flow_control_window_check = true;
g_target_initial_window_size_mocker = new TransportTargetWindowSizeMocker();
grpc_core::chttp2::g_test_only_transport_target_window_estimates_mocker =
g_target_initial_window_size_mocker;
grpc::testing::TestEnvironment env(&argc, argv);
grpc_init();
auto result = RUN_ALL_TESTS();
grpc_shutdown();
return result;
}

@ -1,115 +1,380 @@
// Copyright 2022 gRPC authors. /*
// *
// Licensed under the Apache License, Version 2.0 (the "License"); * Copyright 2021 gRPC authors.
// you may not use this file except in compliance with the License. *
// You may obtain a copy of the License at * Licensed under the Apache License, Version 2.0 (the "License");
// * you may not use this file except in compliance with the License.
// http://www.apache.org/licenses/LICENSE-2.0 * You may obtain a copy of the License at
// *
// Unless required by applicable law or agreed to in writing, software * http://www.apache.org/licenses/LICENSE-2.0
// distributed under the License is distributed on an "AS IS" BASIS, *
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * Unless required by applicable law or agreed to in writing, software
// See the License for the specific language governing permissions and * distributed under the License is distributed on an "AS IS" BASIS,
// limitations under the License. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#include <grpc/support/port_platform.h>
#include "src/core/ext/transport/chttp2/transport/flow_control.h" #include "src/core/ext/transport/chttp2/transport/flow_control.h"
#include <gtest/gtest.h> #include <stdlib.h>
#include <string.h>
#include <functional>
#include <set>
#include <thread>
#include <gmock/gmock.h>
#include "src/core/lib/iomgr/exec_ctx.h" #include <grpc/grpc.h>
#include "src/core/lib/resource_quota/resource_quota.h" #include <grpc/grpc_security.h>
#include <grpc/impl/codegen/grpc_types.h>
#include <grpc/slice.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/string_util.h>
#include <grpc/support/time.h>
namespace grpc_core { #include "src/core/ext/filters/client_channel/backup_poller.h"
namespace chttp2 { #include "src/core/lib/channel/channel_args.h"
#include "src/core/lib/gprpp/host_port.h"
#include "src/core/lib/surface/channel.h"
#include "test/core/end2end/cq_verifier.h"
#include "test/core/util/port.h"
#include "test/core/util/test_config.h"
namespace { namespace {
auto* g_memory_owner = new MemoryOwner(
ResourceQuota::Default()->memory_quota()->CreateMemoryOwner("test"));
}
TEST(FlowControl, NoOp) { class TransportTargetWindowSizeMocker
ExecCtx exec_ctx; : public grpc_core::chttp2::TestOnlyTransportTargetWindowEstimatesMocker {
TransportFlowControl tfc("test", true, g_memory_owner); public:
StreamFlowControl sfc(&tfc); static constexpr uint32_t kLargeInitialWindowSize = 1u << 31;
// Check initial values are per http2 spec static constexpr uint32_t kSmallInitialWindowSize = 0;
EXPECT_EQ(tfc.sent_init_window(), 65535);
EXPECT_EQ(tfc.acked_init_window(), 65535); double ComputeNextTargetInitialWindowSizeFromPeriodicUpdate(
EXPECT_EQ(tfc.remote_window(), 65535); double /* current_target */) override {
EXPECT_EQ(tfc.target_frame_size(), 16384); // Protecting access to variable window_size_ shared between client and
EXPECT_EQ(sfc.remote_window_delta(), 0); // server.
EXPECT_EQ(sfc.min_progress_size(), 0); grpc_core::MutexLock lock(&mu_);
EXPECT_EQ(sfc.local_window_delta(), 0); if (alternating_initial_window_sizes_) {
EXPECT_EQ(sfc.announced_window_delta(), 0); window_size_ = (window_size_ == kLargeInitialWindowSize)
? kSmallInitialWindowSize
: kLargeInitialWindowSize;
}
return window_size_;
}
// Alternates the initial window size targets. Computes a low values if it was
// previously high, or a high value if it was previously low.
void AlternateTargetInitialWindowSizes() {
grpc_core::MutexLock lock(&mu_);
alternating_initial_window_sizes_ = true;
}
void Reset() {
// Protecting access to variable window_size_ shared between client and
// server.
grpc_core::MutexLock lock(&mu_);
alternating_initial_window_sizes_ = false;
window_size_ = kLargeInitialWindowSize;
}
private:
grpc_core::Mutex mu_;
bool alternating_initial_window_sizes_ ABSL_GUARDED_BY(mu_) = false;
double window_size_ ABSL_GUARDED_BY(mu_) = kLargeInitialWindowSize;
};
TransportTargetWindowSizeMocker* g_target_initial_window_size_mocker;
void* tag(intptr_t t) { return reinterpret_cast<void*>(t); }
void VerifyChannelReady(grpc_channel* channel, grpc_completion_queue* cq) {
grpc_connectivity_state state =
grpc_channel_check_connectivity_state(channel, 1 /* try_to_connect */);
while (state != GRPC_CHANNEL_READY) {
grpc_channel_watch_connectivity_state(
channel, state, grpc_timeout_seconds_to_deadline(5), cq, nullptr);
grpc_completion_queue_next(cq, grpc_timeout_seconds_to_deadline(5),
nullptr);
state = grpc_channel_check_connectivity_state(channel, 0);
}
} }
TEST(FlowControl, SendData) { void VerifyChannelConnected(grpc_channel* channel, grpc_completion_queue* cq) {
ExecCtx exec_ctx; // Verify channel is connected. Use a ping to make sure that clients
TransportFlowControl tfc("test", true, g_memory_owner); // tries sending/receiving bytes if the channel is connected.
StreamFlowControl sfc(&tfc); grpc_channel_ping(channel, cq, reinterpret_cast<void*>(2000), nullptr);
sfc.SentData(1024); grpc_event ev = grpc_completion_queue_next(
EXPECT_EQ(sfc.remote_window_delta(), -1024); cq, grpc_timeout_seconds_to_deadline(5), nullptr);
EXPECT_EQ(tfc.remote_window(), 65535 - 1024); GPR_ASSERT(ev.type == GRPC_OP_COMPLETE);
GPR_ASSERT(ev.tag == reinterpret_cast<void*>(2000));
GPR_ASSERT(ev.success == 1);
GPR_ASSERT(grpc_channel_check_connectivity_state(channel, 0) ==
GRPC_CHANNEL_READY);
} }
TEST(FlowControl, InitialTransportUpdate) { // Shuts down and destroys the server.
ExecCtx exec_ctx; void ServerShutdownAndDestroy(grpc_server* server, grpc_completion_queue* cq) {
TransportFlowControl tfc("test", true, g_memory_owner); // Shutdown and destroy server
EXPECT_EQ(tfc.MakeAction(), FlowControlAction()); grpc_server_shutdown_and_notify(server, cq, reinterpret_cast<void*>(1000));
while (grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME),
nullptr)
.tag != reinterpret_cast<void*>(1000)) {
}
grpc_server_destroy(server);
} }
TEST(FlowControl, InitialStreamUpdate) { grpc_slice LargeSlice(void) {
ExecCtx exec_ctx; grpc_slice slice = grpc_slice_malloc(10000000); // ~10MB
TransportFlowControl tfc("test", true, g_memory_owner); memset(GRPC_SLICE_START_PTR(slice), 'x', GRPC_SLICE_LENGTH(slice));
StreamFlowControl sfc(&tfc); return slice;
EXPECT_EQ(sfc.MakeAction(), FlowControlAction());
} }
TEST(FlowControl, RecvData) { void PerformCallWithLargePayload(grpc_channel* channel, grpc_server* server,
ExecCtx exec_ctx; grpc_completion_queue* cq) {
TransportFlowControl tfc("test", true, g_memory_owner); grpc_slice request_payload_slice = LargeSlice();
StreamFlowControl sfc(&tfc); grpc_slice response_payload_slice = LargeSlice();
EXPECT_EQ(absl::OkStatus(), sfc.RecvData(1024)); grpc_call* c;
EXPECT_EQ(tfc.announced_window(), 65535 - 1024); grpc_call* s;
EXPECT_EQ(sfc.local_window_delta(), -1024); grpc_byte_buffer* request_payload =
grpc_raw_byte_buffer_create(&request_payload_slice, 1);
grpc_byte_buffer* response_payload =
grpc_raw_byte_buffer_create(&response_payload_slice, 1);
cq_verifier* cqv = cq_verifier_create(cq);
grpc_op ops[6];
grpc_op* op;
grpc_metadata_array initial_metadata_recv;
grpc_metadata_array trailing_metadata_recv;
grpc_metadata_array request_metadata_recv;
grpc_byte_buffer* request_payload_recv = nullptr;
grpc_byte_buffer* response_payload_recv = nullptr;
grpc_call_details call_details;
grpc_status_code status;
grpc_call_error error;
grpc_slice details;
int was_cancelled = 2;
gpr_timespec deadline = grpc_timeout_seconds_to_deadline(30);
c = grpc_channel_create_call(channel, nullptr, GRPC_PROPAGATE_DEFAULTS, cq,
grpc_slice_from_static_string("/foo"), nullptr,
deadline, nullptr);
GPR_ASSERT(c);
grpc_metadata_array_init(&initial_metadata_recv);
grpc_metadata_array_init(&trailing_metadata_recv);
grpc_metadata_array_init(&request_metadata_recv);
grpc_call_details_init(&call_details);
memset(ops, 0, sizeof(ops));
op = ops;
op->op = GRPC_OP_SEND_INITIAL_METADATA;
op->data.send_initial_metadata.count = 0;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_SEND_MESSAGE;
op->data.send_message.send_message = request_payload;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_INITIAL_METADATA;
op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_MESSAGE;
op->data.recv_message.recv_message = &response_payload_recv;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
op->data.recv_status_on_client.status = &status;
op->data.recv_status_on_client.status_details = &details;
op->flags = 0;
op->reserved = nullptr;
op++;
error = grpc_call_start_batch(c, ops, static_cast<size_t>(op - ops), tag(1),
nullptr);
GPR_ASSERT(GRPC_CALL_OK == error);
error = grpc_server_request_call(server, &s, &call_details,
&request_metadata_recv, cq, cq, tag(101));
GPR_ASSERT(GRPC_CALL_OK == error);
CQ_EXPECT_COMPLETION(cqv, tag(101), 1);
cq_verify(cqv);
memset(ops, 0, sizeof(ops));
op = ops;
op->op = GRPC_OP_SEND_INITIAL_METADATA;
op->data.send_initial_metadata.count = 0;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_MESSAGE;
op->data.recv_message.recv_message = &request_payload_recv;
op->flags = 0;
op->reserved = nullptr;
op++;
error = grpc_call_start_batch(s, ops, static_cast<size_t>(op - ops), tag(102),
nullptr);
GPR_ASSERT(GRPC_CALL_OK == error);
CQ_EXPECT_COMPLETION(cqv, tag(102), 1);
cq_verify(cqv);
memset(ops, 0, sizeof(ops));
op = ops;
op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
op->data.recv_close_on_server.cancelled = &was_cancelled;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_SEND_MESSAGE;
op->data.send_message.send_message = response_payload;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
op->data.send_status_from_server.trailing_metadata_count = 0;
op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED;
grpc_slice status_details = grpc_slice_from_static_string("xyz");
op->data.send_status_from_server.status_details = &status_details;
op->flags = 0;
op->reserved = nullptr;
op++;
error = grpc_call_start_batch(s, ops, static_cast<size_t>(op - ops), tag(103),
nullptr);
GPR_ASSERT(GRPC_CALL_OK == error);
CQ_EXPECT_COMPLETION(cqv, tag(103), 1);
CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
cq_verify(cqv);
GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED);
GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz"));
GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo"));
GPR_ASSERT(was_cancelled == 0);
grpc_slice_unref(details);
grpc_metadata_array_destroy(&initial_metadata_recv);
grpc_metadata_array_destroy(&trailing_metadata_recv);
grpc_metadata_array_destroy(&request_metadata_recv);
grpc_call_details_destroy(&call_details);
grpc_call_unref(c);
grpc_call_unref(s);
cq_verifier_destroy(cqv);
grpc_byte_buffer_destroy(request_payload);
grpc_byte_buffer_destroy(response_payload);
grpc_byte_buffer_destroy(request_payload_recv);
grpc_byte_buffer_destroy(response_payload_recv);
grpc_slice_unref(request_payload_slice);
grpc_slice_unref(response_payload_slice);
} }
TEST(FlowControl, TrackMinProgressSize) { class FlowControlTest : public ::testing::Test {
ExecCtx exec_ctx; protected:
TransportFlowControl tfc("test", true, g_memory_owner); void SetUp() override {
StreamFlowControl sfc(&tfc); cq_ = grpc_completion_queue_create_for_next(nullptr);
sfc.UpdateProgress(5); // create the server
EXPECT_EQ(sfc.min_progress_size(), 5); std::string server_address =
sfc.UpdateProgress(10); grpc_core::JoinHostPort("localhost", grpc_pick_unused_port_or_die());
EXPECT_EQ(sfc.min_progress_size(), 10); grpc_arg server_args[] = {
EXPECT_EQ(absl::OkStatus(), sfc.RecvData(5)); grpc_channel_arg_integer_create(
EXPECT_EQ(sfc.min_progress_size(), 5); const_cast<char*>(GRPC_ARG_HTTP2_MAX_PING_STRIKES), 0),
EXPECT_EQ(absl::OkStatus(), sfc.RecvData(5)); grpc_channel_arg_integer_create(
EXPECT_EQ(sfc.min_progress_size(), 0); const_cast<char*>(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), -1),
EXPECT_EQ(absl::OkStatus(), sfc.RecvData(5)); grpc_channel_arg_integer_create(
EXPECT_EQ(sfc.min_progress_size(), 0); const_cast<char*>(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH), -1)};
grpc_channel_args server_channel_args = {GPR_ARRAY_SIZE(server_args),
server_args};
server_ = grpc_server_create(&server_channel_args, nullptr);
grpc_server_register_completion_queue(server_, cq_, nullptr);
grpc_server_credentials* server_creds =
grpc_insecure_server_credentials_create();
GPR_ASSERT(grpc_server_add_http2_port(server_, server_address.c_str(),
server_creds));
grpc_server_credentials_release(server_creds);
grpc_server_start(server_);
// create the channel (bdp pings are enabled by default)
grpc_arg client_args[] = {
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA), 0),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS), 1),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), -1),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH), -1)};
grpc_channel_args client_channel_args = {GPR_ARRAY_SIZE(client_args),
client_args};
grpc_channel_credentials* creds = grpc_insecure_credentials_create();
channel_ = grpc_channel_create(server_address.c_str(), creds,
&client_channel_args);
grpc_channel_credentials_release(creds);
VerifyChannelReady(channel_, cq_);
g_target_initial_window_size_mocker->Reset();
}
void TearDown() override {
// shutdown and destroy the client and server
grpc_channel_destroy(channel_);
ServerShutdownAndDestroy(server_, cq_);
grpc_completion_queue_shutdown(cq_);
while (grpc_completion_queue_next(cq_, gpr_inf_future(GPR_CLOCK_REALTIME),
nullptr)
.type != GRPC_QUEUE_SHUTDOWN) {
}
grpc_completion_queue_destroy(cq_);
}
grpc_server* server_ = nullptr;
grpc_channel* channel_ = nullptr;
grpc_completion_queue* cq_ = nullptr;
};
TEST_F(FlowControlTest,
TestLargeWindowSizeUpdatesDoNotCauseIllegalFlowControlWindows) {
for (int i = 0; i < 10; ++i) {
PerformCallWithLargePayload(channel_, server_, cq_);
VerifyChannelConnected(channel_, cq_);
}
} }
TEST(FlowControl, NoUpdateWithoutReader) { TEST_F(FlowControlTest, TestWindowSizeUpdatesDoNotCauseStalledStreams) {
ExecCtx exec_ctx; g_target_initial_window_size_mocker->AlternateTargetInitialWindowSizes();
TransportFlowControl tfc("test", true, g_memory_owner); for (int i = 0; i < 100; ++i) {
StreamFlowControl sfc(&tfc); PerformCallWithLargePayload(channel_, server_, cq_);
for (int i = 0; i < 65535; i++) { VerifyChannelConnected(channel_, cq_);
EXPECT_EQ(sfc.RecvData(1), absl::OkStatus());
EXPECT_EQ(sfc.MakeAction().send_stream_update(),
FlowControlAction::Urgency::NO_ACTION_NEEDED);
} }
// Empty window needing 1 byte to progress should trigger an immediate read.
sfc.UpdateProgress(1);
EXPECT_EQ(sfc.min_progress_size(), 1);
EXPECT_EQ(sfc.MakeAction().send_stream_update(),
FlowControlAction::Urgency::UPDATE_IMMEDIATELY);
EXPECT_GT(sfc.MaybeSendUpdate(), 0);
} }
} // namespace chttp2 } // namespace
} // namespace grpc_core
int main(int argc, char** argv) { int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS(); // Make sure that we will have an active poller on all client-side fd's that
// are capable of sending and receiving even in the case that we don't have an
// active RPC operation on the fd.
GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 1);
grpc_core::chttp2::g_test_only_transport_flow_control_window_check = true;
g_target_initial_window_size_mocker = new TransportTargetWindowSizeMocker();
grpc_core::chttp2::g_test_only_transport_target_window_estimates_mocker =
g_target_initial_window_size_mocker;
grpc::testing::TestEnvironment env(&argc, argv);
grpc_init();
auto result = RUN_ALL_TESTS();
grpc_shutdown();
return result;
} }

@ -232,7 +232,10 @@ class TestServer {
static void HandleOneRpc(grpc_call* call, grpc_completion_queue* call_cq) { static void HandleOneRpc(grpc_call* call, grpc_completion_queue* call_cq) {
// Send a large enough payload to get us stalled on outgoing flow control // Send a large enough payload to get us stalled on outgoing flow control
std::string send_payload(4 * 1024 * 1024, 'a'); std::string send_payload = "";
for (int i = 0; i < 4 * 1e6; i++) {
send_payload += "a";
}
grpc_slice request_payload_slice = grpc_slice request_payload_slice =
grpc_slice_from_copied_string(send_payload.c_str()); grpc_slice_from_copied_string(send_payload.c_str());
grpc_byte_buffer* request_payload = grpc_byte_buffer* request_payload =

@ -27,7 +27,7 @@
#include <grpcpp/client_context.h> #include <grpcpp/client_context.h>
#include "src/core/lib/surface/call_test_only.h" #include "src/core/lib/surface/call_test_only.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/byte_stream.h"
namespace grpc { namespace grpc {
namespace testing { namespace testing {

@ -29,6 +29,7 @@
#include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/string.h"
#include "src/core/lib/gpr/useful.h" #include "src/core/lib/gpr/useful.h"
#include "src/core/lib/transport/byte_stream.h"
#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/messages.pb.h"
#include "src/proto/grpc/testing/test.grpc.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h"
#include "test/cpp/util/create_test_channel.h" #include "test/cpp/util/create_test_channel.h"

@ -26,7 +26,7 @@
#include <grpcpp/security/server_credentials.h> #include <grpcpp/security/server_credentials.h>
#include "src/core/lib/surface/call_test_only.h" #include "src/core/lib/surface/call_test_only.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/byte_stream.h"
#include "test/cpp/util/test_credentials_provider.h" #include "test/cpp/util/test_credentials_provider.h"
ABSL_DECLARE_FLAG(bool, use_alts); ABSL_DECLARE_FLAG(bool, use_alts);

@ -35,7 +35,7 @@
#include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/string.h"
#include "src/core/lib/iomgr/gethostname.h" #include "src/core/lib/iomgr/gethostname.h"
#include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/byte_stream.h"
#include "src/proto/grpc/testing/empty.pb.h" #include "src/proto/grpc/testing/empty.pb.h"
#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/messages.pb.h"
#include "src/proto/grpc/testing/test.grpc.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h"

@ -430,7 +430,7 @@ static void BM_TransportStreamSend(benchmark::State& state) {
// is unreffed after each send_message op. // is unreffed after each send_message op.
grpc_slice send_slice = grpc_slice_malloc_large(state.range(0)); grpc_slice send_slice = grpc_slice_malloc_large(state.range(0));
memset(GRPC_SLICE_START_PTR(send_slice), 0, GRPC_SLICE_LENGTH(send_slice)); memset(GRPC_SLICE_START_PTR(send_slice), 0, GRPC_SLICE_LENGTH(send_slice));
grpc_core::SliceBuffer send_stream; grpc_core::ManualConstructor<grpc_core::SliceBufferByteStream> send_stream;
auto arena = grpc_core::MakeScopedArena(1024, g_memory_allocator); auto arena = grpc_core::MakeScopedArena(1024, g_memory_allocator);
grpc_metadata_batch b(arena.get()); grpc_metadata_batch b(arena.get());
RepresentativeClientInitialMetadata::Prepare(&b); RepresentativeClientInitialMetadata::Prepare(&b);
@ -444,15 +444,18 @@ static void BM_TransportStreamSend(benchmark::State& state) {
gpr_event_set(bm_done, reinterpret_cast<void*>(1)); gpr_event_set(bm_done, reinterpret_cast<void*>(1));
return; return;
} }
send_stream.Clear(); grpc_slice_buffer send_buffer;
send_stream.Append(grpc_core::Slice(grpc_slice_ref(send_slice))); grpc_slice_buffer_init(&send_buffer);
grpc_slice_buffer_add(&send_buffer, grpc_slice_ref(send_slice));
send_stream.Init(&send_buffer, 0);
grpc_slice_buffer_destroy(&send_buffer);
// force outgoing window to be yuge // force outgoing window to be yuge
s->chttp2_stream()->flow_control.TestOnlyForceHugeWindow(); s->chttp2_stream()->flow_control->TestOnlyForceHugeWindow();
f.chttp2_transport()->flow_control.TestOnlyForceHugeWindow(); f.chttp2_transport()->flow_control->TestOnlyForceHugeWindow();
reset_op(); reset_op();
op.on_complete = c.get(); op.on_complete = c.get();
op.send_message = true; op.send_message = true;
op.payload->send_message.send_message = &send_stream; op.payload->send_message.send_message.reset(send_stream.get());
s->Op(&op); s->Op(&op);
}); });
@ -554,7 +557,7 @@ static void BM_TransportStreamRecv(benchmark::State& state) {
s->Init(state); s->Init(state);
grpc_transport_stream_op_batch_payload op_payload(nullptr); grpc_transport_stream_op_batch_payload op_payload(nullptr);
grpc_transport_stream_op_batch op; grpc_transport_stream_op_batch op;
absl::optional<grpc_core::SliceBuffer> recv_stream; grpc_core::OrphanablePtr<grpc_core::ByteStream> recv_stream;
grpc_slice incoming_data = CreateIncomingDataSlice(state.range(0), 16384); grpc_slice incoming_data = CreateIncomingDataSlice(state.range(0), 16384);
auto reset_op = [&]() { auto reset_op = [&]() {
@ -571,23 +574,57 @@ static void BM_TransportStreamRecv(benchmark::State& state) {
uint32_t received; uint32_t received;
std::unique_ptr<TestClosure> drain_start;
std::unique_ptr<TestClosure> drain;
std::unique_ptr<TestClosure> drain_continue;
grpc_slice recv_slice;
std::unique_ptr<TestClosure> c = std::unique_ptr<TestClosure> c =
MakeTestClosure([&](grpc_error_handle /*error*/) { MakeTestClosure([&](grpc_error_handle /*error*/) {
if (!state.KeepRunning()) return; if (!state.KeepRunning()) return;
// force outgoing window to be yuge // force outgoing window to be yuge
s->chttp2_stream()->flow_control.TestOnlyForceHugeWindow(); s->chttp2_stream()->flow_control->TestOnlyForceHugeWindow();
f.chttp2_transport()->flow_control.TestOnlyForceHugeWindow(); f.chttp2_transport()->flow_control->TestOnlyForceHugeWindow();
received = 0; received = 0;
reset_op(); reset_op();
op.on_complete = do_nothing.get(); op.on_complete = do_nothing.get();
op.recv_message = true; op.recv_message = true;
op.payload->recv_message.recv_message = &recv_stream; op.payload->recv_message.recv_message = &recv_stream;
op.payload->recv_message.call_failed_before_recv_message = nullptr; op.payload->recv_message.call_failed_before_recv_message = nullptr;
op.payload->recv_message.recv_message_ready = c.get(); op.payload->recv_message.recv_message_ready = drain_start.get();
s->Op(&op); s->Op(&op);
f.PushInput(grpc_slice_ref(incoming_data)); f.PushInput(grpc_slice_ref(incoming_data));
}); });
drain_start = MakeTestClosure([&](grpc_error_handle /*error*/) {
if (recv_stream == nullptr) {
GPR_ASSERT(!state.KeepRunning());
return;
}
grpc_core::Closure::Run(DEBUG_LOCATION, drain.get(), GRPC_ERROR_NONE);
});
drain = MakeTestClosure([&](grpc_error_handle /*error*/) {
do {
if (received == recv_stream->length()) {
recv_stream.reset();
grpc_core::ExecCtx::Run(DEBUG_LOCATION, c.get(), GRPC_ERROR_NONE);
return;
}
} while (recv_stream->Next(recv_stream->length() - received,
drain_continue.get()) &&
GRPC_ERROR_NONE == recv_stream->Pull(&recv_slice) &&
(received += GRPC_SLICE_LENGTH(recv_slice),
grpc_slice_unref_internal(recv_slice), true));
});
drain_continue = MakeTestClosure([&](grpc_error_handle /*error*/) {
GPR_ASSERT(GRPC_LOG_IF_ERROR("Pull", recv_stream->Pull(&recv_slice)));
received += GRPC_SLICE_LENGTH(recv_slice);
grpc_slice_unref_internal(recv_slice);
grpc_core::Closure::Run(DEBUG_LOCATION, drain.get(), GRPC_ERROR_NONE);
});
reset_op(); reset_op();
auto b_recv = absl::make_unique<grpc_metadata_batch>(arena.get()); auto b_recv = absl::make_unique<grpc_metadata_batch>(arena.get());
op.send_initial_metadata = true; op.send_initial_metadata = true;

@ -2403,6 +2403,8 @@ src/core/lib/surface/validate_metadata.h \
src/core/lib/surface/version.cc \ src/core/lib/surface/version.cc \
src/core/lib/transport/bdp_estimator.cc \ src/core/lib/transport/bdp_estimator.cc \
src/core/lib/transport/bdp_estimator.h \ src/core/lib/transport/bdp_estimator.h \
src/core/lib/transport/byte_stream.cc \
src/core/lib/transport/byte_stream.h \
src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/connectivity_state.cc \
src/core/lib/transport/connectivity_state.h \ src/core/lib/transport/connectivity_state.h \
src/core/lib/transport/error_utils.cc \ src/core/lib/transport/error_utils.cc \

@ -2199,6 +2199,8 @@ src/core/lib/surface/version.cc \
src/core/lib/transport/README.md \ src/core/lib/transport/README.md \
src/core/lib/transport/bdp_estimator.cc \ src/core/lib/transport/bdp_estimator.cc \
src/core/lib/transport/bdp_estimator.h \ src/core/lib/transport/bdp_estimator.h \
src/core/lib/transport/byte_stream.cc \
src/core/lib/transport/byte_stream.h \
src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/connectivity_state.cc \
src/core/lib/transport/connectivity_state.h \ src/core/lib/transport/connectivity_state.h \
src/core/lib/transport/error_utils.cc \ src/core/lib/transport/error_utils.cc \

@ -1467,6 +1467,30 @@
], ],
"uses_polling": false "uses_polling": false
}, },
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix",
"windows"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": false,
"language": "c",
"name": "manual_constructor_test",
"platforms": [
"linux",
"mac",
"posix",
"windows"
],
"uses_polling": false
},
{ {
"args": [], "args": [],
"benchmark": false, "benchmark": false,
@ -3041,6 +3065,30 @@
], ],
"uses_polling": false "uses_polling": false
}, },
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix",
"windows"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "byte_stream_test",
"platforms": [
"linux",
"mac",
"posix",
"windows"
],
"uses_polling": false
},
{ {
"args": [], "args": [],
"benchmark": false, "benchmark": false,
@ -4189,30 +4237,6 @@
], ],
"uses_polling": true "uses_polling": true
}, },
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix",
"windows"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "flow_control_end2end_test",
"platforms": [
"linux",
"mac",
"posix",
"windows"
],
"uses_polling": true
},
{ {
"args": [], "args": [],
"benchmark": false, "benchmark": false,

Loading…
Cancel
Save