From bbeb15006af65d371a933fe432bce5078115b9e3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 30 Jan 2023 21:39:11 -0800 Subject: [PATCH] [promises] Server call (#31448) * add experiment * allow instantiation * scratchings * scratchings * sniffly * Automated change: Fix sanity tests * fix * fix * fix * Automated change: Fix sanity tests * progress * change pipe labels to enable server code to be written * better api * Automated change: Fix sanity tests * progress * [promises] Implementation of deadline for server-based-calls * compression filter compiles again * Automated change: Fix sanity tests * fix * server tracing fixes * get client initial metadata * progress * progress * server call surface progress * Automated change: Fix sanity tests * move payload * server-progress * recv-message-server-connchan * logging * fix context-gate * recv fix@top * Automated change: Fix sanity tests * recv close on server * top termination start * [promises] Move Empty to be first class * fixes * fix * flow control fix * got to orphan! * orphan * call orphan * spam cleanup * fix * new cancelation semantics * progress * large metadata fixes * fix * fix * log * better logs * fix-chanz * logging, necessaryness * fix typo * fixes * fix * fix * fix-pipe * cleanup logging * fix * build-fix * fix * Automated change: Fix sanity tests * logging * Automated change: Fix sanity tests * Automated change: Fix sanity tests * better primitive * Revert "better primitive" This reverts commit 119b5ee244763a88d2318bc5145065f4940a2295. * fix * fix * trrracing * Automated change: Fix sanity tests * get-trailing-metadata * cancellation * Automated change: Fix sanity tests * add transform pipeline to pipe * add transform pipeline to pipe * interceptor lists * new server initial md api into filters * convert connected_channel * convert call * initial promise based filter conversion * convert promise based filter * build fixes * compile fix * fixes * fix ordering * fixes * check-metadata * revert later: debug code * better debug * fix metadata ordering with messages in promise based filter * fix ordering problem between batch completion and promise completion * properly handle failure on receive message path on client * more debug, fix a repoll bug in pbf * Automated change: Fix sanity tests * fixes * Automated change: Fix sanity tests * cleanup logging * fixes * missing file * fixes * logging * Automated change: Fix sanity tests * fixes * convert logging filter * fix * Automated change: Fix sanity tests * fix bad server response test * Revert "Disable logging test (#32049)" This reverts commit 5fc92eaeae6f668ba7c2df1024f18ef8c798a319. * fix * Automated change: Fix sanity tests * fix memory leaks, logging * Automated change: Fix sanity tests * slice refcount debugging * asan-canaries * leak-fix * leak-fix * Automated change: Fix sanity tests * fix * fix * fix * fix * fix * Automated change: Fix sanity tests * fix * remove mistaken line * add-comment * fix refcounting bug * Automated change: Fix sanity tests * rename variable * renames * bleh * carry pipe close status from bottom of pipe to top to appease recv-close-on-server * backport cancellation * Revert "carry pipe close status from bottom of pipe to top to appease" This reverts commit fa33301dcddd01aff52d3bee77b16ffef9466de5. * fix * Automated change: Fix sanity tests * review-feedback * comment-ordering * monostate * renames * undo-review-feedback * fix * review-feedback * review-feedback * fix * review-feedback * drop debugloc constructor * interceptor-list-rev-feedback * interceptor-list-rev-feedback * pipe test * review-feedback * undo-mistaken-change * Automated change: Fix sanity tests * pipe error state * detect send/recv failures and report * iwyu, build * fix submodules * fix * warning * cleanup * Automated change: Fix sanity tests * fix * fix for windows * fix * null pointer fix * iwyu * gen projex --------- Co-authored-by: ctiller --- BUILD | 15 +- CMakeLists.txt | 148 +- Makefile | 8 +- bazel/experiments.bzl | 1 + build_autogenerated.yaml | 259 ++-- config.m4 | 4 +- config.w32 | 4 +- doc/environment_variables.md | 1 + gRPC-C++.podspec | 16 +- gRPC-Core.podspec | 20 +- grpc.gemspec | 12 +- grpc.gyp | 12 +- package.xml | 12 +- src/core/BUILD | 95 +- .../grpclb/client_load_reporting_filter.cc | 21 +- .../filters/http/client/http_client_filter.cc | 46 +- .../message_compress/compression_filter.cc | 223 ++- .../message_compress/compression_filter.h | 21 +- .../filters/http/server/http_server_filter.cc | 49 +- .../stateful_session_filter.cc | 51 +- .../chttp2/transport/chttp2_transport.cc | 32 + .../ext/transport/chttp2/transport/internal.h | 14 +- .../ext/transport/chttp2/transport/parsing.cc | 135 +- .../lib/channel/channel_stack_builder_impl.cc | 15 +- src/core/lib/channel/connected_channel.cc | 1212 ++++++++++++----- src/core/lib/channel/promise_based_filter.cc | 687 ++++++---- src/core/lib/channel/promise_based_filter.h | 139 +- src/core/lib/event_engine/slice.cc | 2 +- src/core/lib/experiments/experiments.cc | 4 + src/core/lib/experiments/experiments.h | 9 +- src/core/lib/experiments/experiments.yaml | 8 + src/core/lib/gprpp/debug_location.h | 3 - src/core/lib/iomgr/call_combiner.cc | 8 +- src/core/lib/iomgr/closure.cc | 27 + src/core/lib/iomgr/closure.h | 2 + src/core/lib/promise/activity.h | 4 + src/core/lib/promise/for_each.h | 22 + src/core/lib/promise/if.h | 61 + src/core/lib/promise/interceptor_list.h | 309 +++++ src/core/lib/promise/intra_activity_waiter.h | 6 + src/core/lib/promise/latch.h | 30 +- src/core/lib/promise/map_pipe.h | 32 +- src/core/lib/promise/pipe.h | 482 ++++--- src/core/lib/promise/{pipe.cc => trace.cc} | 5 +- src/core/lib/promise/trace.h | 24 + src/core/lib/promise/try_concurrently.h | 28 + src/core/lib/resource_quota/arena.h | 90 +- src/core/lib/slice/slice.cc | 6 +- src/core/lib/slice/slice.h | 15 +- src/core/lib/slice/slice_buffer.h | 8 + src/core/lib/slice/slice_refcount.cc | 20 + src/core/lib/slice/slice_refcount.h | 26 +- src/core/lib/surface/call.cc | 1059 ++++++++++---- src/core/lib/surface/call.h | 28 + src/core/lib/surface/call_trace.cc | 28 +- src/core/lib/surface/lame_client.cc | 4 +- src/core/lib/surface/server.cc | 299 +++- src/core/lib/surface/server.h | 3 + src/core/lib/transport/transport.cc | 7 +- src/core/lib/transport/transport.h | 18 +- src/cpp/ext/filters/logging/BUILD | 11 +- src/cpp/ext/filters/logging/logging_filter.cc | 142 +- src/python/grpcio/grpc_core_dependencies.py | 4 +- .../fixtures/h2_sockpair_with_minstack.cc | 15 +- test/core/end2end/generate_tests.bzl | 2 +- .../core/end2end/tests/cancel_after_accept.cc | 3 + test/core/end2end/tests/grpc_authz.cc | 1 + test/core/end2end/tests/max_message_length.cc | 3 + test/core/end2end/tests/no_logging.cc | 2 + test/core/end2end/tests/server_streaming.cc | 11 +- test/core/filters/filter_fuzzer.cc | 28 +- test/core/promise/BUILD | 20 +- test/core/promise/activity_test.cc | 4 +- test/core/promise/for_each_test.cc | 1 + test/core/promise/if_test.cc | 12 + test/core/promise/interceptor_list_test.cc | 144 ++ test/core/promise/latch_test.cc | 4 +- test/core/promise/map_pipe_test.cc | 1 + test/core/promise/pipe_test.cc | 262 +++- test/core/resource_quota/arena_test.cc | 45 + test/core/slice/slice_test.cc | 2 +- test/cpp/ext/filters/logging/logging_test.cc | 10 +- tools/doxygen/Doxyfile.c++.internal | 12 +- tools/doxygen/Doxyfile.core.internal | 12 +- tools/run_tests/generated/tests.json | 24 + 85 files changed, 4960 insertions(+), 1744 deletions(-) create mode 100644 src/core/lib/iomgr/closure.cc create mode 100644 src/core/lib/promise/interceptor_list.h rename src/core/lib/promise/{pipe.cc => trace.cc} (82%) create mode 100644 src/core/lib/promise/trace.h create mode 100644 src/core/lib/slice/slice_refcount.cc create mode 100644 test/core/promise/interceptor_list_test.cc diff --git a/BUILD b/BUILD index 5c11bfe5ffc..aa8b4af6f34 100644 --- a/BUILD +++ b/BUILD @@ -1421,6 +1421,8 @@ grpc_cc_library( "//src/core:arena", "//src/core:arena_promise", "//src/core:atomic_utils", + "//src/core:basic_join", + "//src/core:basic_seq", "//src/core:bitset", "//src/core:channel_args", "//src/core:channel_args_endpoint_config", @@ -1449,7 +1451,7 @@ grpc_cc_library( "//src/core:iomgr_fwd", "//src/core:iomgr_port", "//src/core:json", - "//src/core:latch", + "//src/core:map", "//src/core:match", "//src/core:memory_quota", "//src/core:no_destruct", @@ -1475,6 +1477,8 @@ grpc_cc_library( "//src/core:thread_quota", "//src/core:time", "//src/core:transport_fwd", + "//src/core:try_join", + "//src/core:try_seq", "//src/core:useful", ], ) @@ -3260,9 +3264,9 @@ grpc_cc_library( "grpc_public_hdrs", "grpc_trace", "promise", + "//src/core:activity", "//src/core:arena", "//src/core:arena_promise", - "//src/core:basic_seq", "//src/core:channel_args", "//src/core:channel_fwd", "//src/core:channel_init", @@ -3270,15 +3274,14 @@ grpc_cc_library( "//src/core:context", "//src/core:grpc_message_size_filter", "//src/core:latch", - "//src/core:map_pipe", + "//src/core:map", "//src/core:percent_encoding", "//src/core:pipe", - "//src/core:promise_like", - "//src/core:seq", + "//src/core:poll", + "//src/core:race", "//src/core:slice", "//src/core:slice_buffer", "//src/core:transport_fwd", - "//src/core:try_concurrently", ], ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0951d66112e..a7ece0db6c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1015,6 +1015,7 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_cxx init_test) add_dependencies(buildtests_cxx initial_settings_frame_bad_client_test) add_dependencies(buildtests_cxx insecure_security_connector_test) + add_dependencies(buildtests_cxx interceptor_list_test) add_dependencies(buildtests_cxx interop_client) add_dependencies(buildtests_cxx interop_server) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX OR _gRPC_PLATFORM_WINDOWS) @@ -2209,6 +2210,7 @@ add_library(grpc src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/cfstream_handle.cc + src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/dualstack_socket_posix.cc src/core/lib/iomgr/endpoint.cc @@ -2290,8 +2292,8 @@ add_library(grpc src/core/lib/load_balancing/lb_policy_registry.cc src/core/lib/matchers/matchers.cc src/core/lib/promise/activity.cc - src/core/lib/promise/pipe.cc src/core/lib/promise/sleep.cc + src/core/lib/promise/trace.cc src/core/lib/resolver/resolver.cc src/core/lib/resolver/resolver_registry.cc src/core/lib/resolver/server_address.cc @@ -2369,6 +2371,7 @@ add_library(grpc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc src/core/lib/slice/slice_buffer.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/surface/api_trace.cc src/core/lib/surface/builtins.cc @@ -2892,6 +2895,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/cfstream_handle.cc + src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/dualstack_socket_posix.cc src/core/lib/iomgr/endpoint.cc @@ -2971,8 +2975,8 @@ add_library(grpc_unsecure src/core/lib/load_balancing/lb_policy.cc src/core/lib/load_balancing/lb_policy_registry.cc src/core/lib/promise/activity.cc - src/core/lib/promise/pipe.cc src/core/lib/promise/sleep.cc + src/core/lib/promise/trace.cc src/core/lib/resolver/resolver.cc src/core/lib/resolver/resolver_registry.cc src/core/lib/resolver/server_address.cc @@ -3019,6 +3023,7 @@ add_library(grpc_unsecure src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc src/core/lib/slice/slice_buffer.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/surface/api_trace.cc src/core/lib/surface/builtins.cc @@ -4402,6 +4407,7 @@ add_library(grpc_authorization_provider src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/cfstream_handle.cc + src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/dualstack_socket_posix.cc src/core/lib/iomgr/endpoint.cc @@ -4481,7 +4487,7 @@ add_library(grpc_authorization_provider src/core/lib/load_balancing/lb_policy_registry.cc src/core/lib/matchers/matchers.cc src/core/lib/promise/activity.cc - src/core/lib/promise/pipe.cc + src/core/lib/promise/trace.cc src/core/lib/resolver/resolver.cc src/core/lib/resolver/resolver_registry.cc src/core/lib/resolver/server_address.cc @@ -4528,6 +4534,7 @@ add_library(grpc_authorization_provider src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc src/core/lib/slice/slice_buffer.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/surface/api_trace.cc src/core/lib/surface/builtins.cc @@ -8288,6 +8295,7 @@ add_executable(chunked_vector_test src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc + src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc @@ -8302,6 +8310,7 @@ add_executable(chunked_vector_test src/core/lib/resource_quota/trace.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/gprpp/chunked_vector_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -10577,6 +10586,7 @@ add_executable(exec_ctx_wakeup_scheduler_test src/core/lib/debug/trace.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc + src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc @@ -10585,6 +10595,7 @@ add_executable(exec_ctx_wakeup_scheduler_test src/core/lib/promise/activity.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/promise/exec_ctx_wakeup_scheduler_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -11050,6 +11061,7 @@ add_executable(flow_control_test src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc + src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc @@ -11063,6 +11075,7 @@ add_executable(flow_control_test src/core/lib/resource_quota/trace.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/transport/bdp_estimator.cc src/core/lib/transport/pid_controller.cc @@ -11119,13 +11132,14 @@ add_executable(for_each_test src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc + src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc src/core/lib/iomgr/executor.cc src/core/lib/iomgr/iomgr_internal.cc src/core/lib/promise/activity.cc - src/core/lib/promise/pipe.cc + src/core/lib/promise/trace.cc src/core/lib/resource_quota/arena.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc @@ -11134,6 +11148,7 @@ add_executable(for_each_test src/core/lib/resource_quota/trace.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/promise/for_each_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -11478,6 +11493,7 @@ add_executable(frame_test src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/cfstream_handle.cc + src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/dualstack_socket_posix.cc src/core/lib/iomgr/endpoint.cc @@ -11556,7 +11572,7 @@ add_executable(frame_test src/core/lib/load_balancing/lb_policy.cc src/core/lib/load_balancing/lb_policy_registry.cc src/core/lib/promise/activity.cc - src/core/lib/promise/pipe.cc + src/core/lib/promise/trace.cc src/core/lib/resolver/resolver.cc src/core/lib/resolver/resolver_registry.cc src/core/lib/resolver/server_address.cc @@ -11580,6 +11596,7 @@ add_executable(frame_test src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc src/core/lib/slice/slice_buffer.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/surface/api_trace.cc src/core/lib/surface/builtins.cc @@ -13917,6 +13934,77 @@ target_link_libraries(insecure_security_connector_test ) +endif() +if(gRPC_BUILD_TESTS) + +add_executable(interceptor_list_test + src/core/ext/upb-generated/google/protobuf/any.upb.c + src/core/ext/upb-generated/google/rpc/status.upb.c + src/core/lib/debug/trace.cc + src/core/lib/event_engine/memory_allocator.cc + src/core/lib/experiments/config.cc + src/core/lib/experiments/experiments.cc + src/core/lib/gprpp/status_helper.cc + src/core/lib/gprpp/time.cc + src/core/lib/iomgr/closure.cc + src/core/lib/iomgr/combiner.cc + src/core/lib/iomgr/error.cc + src/core/lib/iomgr/exec_ctx.cc + src/core/lib/iomgr/executor.cc + src/core/lib/iomgr/iomgr_internal.cc + src/core/lib/promise/activity.cc + src/core/lib/promise/trace.cc + src/core/lib/resource_quota/arena.cc + src/core/lib/resource_quota/memory_quota.cc + src/core/lib/resource_quota/periodic_update.cc + src/core/lib/resource_quota/resource_quota.cc + src/core/lib/resource_quota/thread_quota.cc + src/core/lib/resource_quota/trace.cc + src/core/lib/slice/percent_encoding.cc + src/core/lib/slice/slice.cc + src/core/lib/slice/slice_refcount.cc + src/core/lib/slice/slice_string_helpers.cc + test/core/promise/interceptor_list_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) +target_compile_features(interceptor_list_test PUBLIC cxx_std_14) +target_include_directories(interceptor_list_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} + third_party/googletest/googletest/include + third_party/googletest/googletest + third_party/googletest/googlemock/include + third_party/googletest/googlemock + ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(interceptor_list_test + ${_gRPC_BASELIB_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ZLIB_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + absl::flat_hash_set + absl::any_invocable + absl::function_ref + absl::hash + absl::type_traits + absl::statusor + absl::utility + gpr + upb +) + + endif() if(gRPC_BUILD_TESTS) @@ -14399,7 +14487,9 @@ endif() if(gRPC_BUILD_TESTS) add_executable(latch_test + src/core/lib/debug/trace.cc src/core/lib/promise/activity.cc + src/core/lib/promise/trace.cc test/core/promise/latch_test.cc third_party/googletest/googletest/src/gtest-all.cc third_party/googletest/googlemock/src/gmock-all.cc @@ -14646,13 +14736,14 @@ add_executable(map_pipe_test src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc + src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc src/core/lib/iomgr/executor.cc src/core/lib/iomgr/iomgr_internal.cc src/core/lib/promise/activity.cc - src/core/lib/promise/pipe.cc + src/core/lib/promise/trace.cc src/core/lib/resource_quota/arena.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc @@ -14661,6 +14752,7 @@ add_executable(map_pipe_test src/core/lib/resource_quota/trace.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/promise/map_pipe_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -16016,6 +16108,7 @@ add_executable(periodic_update_test src/core/lib/debug/trace.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc + src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc @@ -16024,6 +16117,7 @@ add_executable(periodic_update_test src/core/lib/resource_quota/periodic_update.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/resource_quota/periodic_update_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -16153,30 +16247,6 @@ endif() if(gRPC_BUILD_TESTS) add_executable(pipe_test - src/core/ext/upb-generated/google/protobuf/any.upb.c - src/core/ext/upb-generated/google/rpc/status.upb.c - src/core/lib/debug/trace.cc - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/experiments/config.cc - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc - src/core/lib/iomgr/executor.cc - src/core/lib/iomgr/iomgr_internal.cc - src/core/lib/promise/activity.cc - src/core/lib/promise/pipe.cc - src/core/lib/resource_quota/arena.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/resource_quota.cc - src/core/lib/resource_quota/thread_quota.cc - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_string_helpers.cc test/core/promise/pipe_test.cc third_party/googletest/googletest/src/gtest-all.cc third_party/googletest/googlemock/src/gmock-all.cc @@ -16206,15 +16276,7 @@ target_link_libraries(pipe_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ZLIB_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - absl::flat_hash_set - absl::any_invocable - absl::function_ref - absl::hash - absl::type_traits - absl::statusor - absl::utility - gpr - upb + grpc ) @@ -18768,7 +18830,9 @@ endif() if(gRPC_BUILD_TESTS) add_executable(slice_string_helpers_test + src/core/lib/debug/trace.cc src/core/lib/slice/slice.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/slice/slice_string_helpers_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -20162,12 +20226,14 @@ endif() if(gRPC_BUILD_TESTS) add_executable(test_core_event_engine_slice_buffer_test + src/core/lib/debug/trace.cc src/core/lib/event_engine/event_engine.cc src/core/lib/event_engine/resolved_address.cc src/core/lib/event_engine/slice.cc src/core/lib/event_engine/slice_buffer.cc src/core/lib/slice/slice.cc src/core/lib/slice/slice_buffer.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/event_engine/slice_buffer_test.cc third_party/googletest/googletest/src/gtest-all.cc @@ -21515,19 +21581,21 @@ add_executable(try_concurrently_test src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc + src/core/lib/iomgr/closure.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/error.cc src/core/lib/iomgr/exec_ctx.cc src/core/lib/iomgr/executor.cc src/core/lib/iomgr/iomgr_internal.cc src/core/lib/promise/activity.cc - src/core/lib/promise/pipe.cc + src/core/lib/promise/trace.cc src/core/lib/resource_quota/arena.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc src/core/lib/resource_quota/trace.cc src/core/lib/slice/percent_encoding.cc src/core/lib/slice/slice.cc + src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc test/core/promise/try_concurrently_test.cc third_party/googletest/googletest/src/gtest-all.cc diff --git a/Makefile b/Makefile index 5f6348a3a64..8c1670e4426 100644 --- a/Makefile +++ b/Makefile @@ -1473,6 +1473,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/cfstream_handle.cc \ + src/core/lib/iomgr/closure.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/dualstack_socket_posix.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -1554,8 +1555,8 @@ LIBGRPC_SRC = \ src/core/lib/load_balancing/lb_policy_registry.cc \ src/core/lib/matchers/matchers.cc \ src/core/lib/promise/activity.cc \ - src/core/lib/promise/pipe.cc \ src/core/lib/promise/sleep.cc \ + src/core/lib/promise/trace.cc \ src/core/lib/resolver/resolver.cc \ src/core/lib/resolver/resolver_registry.cc \ src/core/lib/resolver/server_address.cc \ @@ -1633,6 +1634,7 @@ LIBGRPC_SRC = \ src/core/lib/slice/percent_encoding.cc \ src/core/lib/slice/slice.cc \ src/core/lib/slice/slice_buffer.cc \ + src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/surface/api_trace.cc \ src/core/lib/surface/builtins.cc \ @@ -2009,6 +2011,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/cfstream_handle.cc \ + src/core/lib/iomgr/closure.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/dualstack_socket_posix.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -2088,8 +2091,8 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/load_balancing/lb_policy.cc \ src/core/lib/load_balancing/lb_policy_registry.cc \ src/core/lib/promise/activity.cc \ - src/core/lib/promise/pipe.cc \ src/core/lib/promise/sleep.cc \ + src/core/lib/promise/trace.cc \ src/core/lib/resolver/resolver.cc \ src/core/lib/resolver/resolver_registry.cc \ src/core/lib/resolver/server_address.cc \ @@ -2136,6 +2139,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/slice/percent_encoding.cc \ src/core/lib/slice/slice.cc \ src/core/lib/slice/slice_buffer.cc \ + src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/surface/api_trace.cc \ src/core/lib/surface/builtins.cc \ diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index bfcbebb19ee..b86d9688824 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -25,6 +25,7 @@ EXPERIMENTS = { ], "core_end2end_test": [ "promise_based_client_call", + "promise_based_server_call", ], "endpoint_test": [ "tcp_frame_size_tuning", diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index d656f1b5ba4..a4ad71956d1 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -930,25 +930,27 @@ libs: - src/core/lib/promise/activity.h - src/core/lib/promise/arena_promise.h - src/core/lib/promise/context.h + - src/core/lib/promise/detail/basic_join.h - src/core/lib/promise/detail/basic_seq.h - src/core/lib/promise/detail/promise_factory.h - src/core/lib/promise/detail/promise_like.h - src/core/lib/promise/detail/status.h - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - - src/core/lib/promise/for_each.h + - src/core/lib/promise/if.h + - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/latch.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h - - src/core/lib/promise/map_pipe.h - src/core/lib/promise/pipe.h - src/core/lib/promise/poll.h - src/core/lib/promise/promise.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - src/core/lib/promise/sleep.h - - src/core/lib/promise/try_concurrently.h + - src/core/lib/promise/trace.h + - src/core/lib/promise/try_join.h - src/core/lib/promise/try_seq.h - src/core/lib/resolver/resolver.h - src/core/lib/resolver/resolver_factory.h @@ -1608,6 +1610,7 @@ libs: - src/core/lib/iomgr/buffer_list.cc - src/core/lib/iomgr/call_combiner.cc - src/core/lib/iomgr/cfstream_handle.cc + - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/dualstack_socket_posix.cc - src/core/lib/iomgr/endpoint.cc @@ -1689,8 +1692,8 @@ libs: - src/core/lib/load_balancing/lb_policy_registry.cc - src/core/lib/matchers/matchers.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/pipe.cc - src/core/lib/promise/sleep.cc + - src/core/lib/promise/trace.cc - src/core/lib/resolver/resolver.cc - src/core/lib/resolver/resolver_registry.cc - src/core/lib/resolver/server_address.cc @@ -1768,6 +1771,7 @@ libs: - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_buffer.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/surface/api_trace.cc - src/core/lib/surface/builtins.cc @@ -2258,25 +2262,27 @@ libs: - src/core/lib/promise/activity.h - src/core/lib/promise/arena_promise.h - src/core/lib/promise/context.h + - src/core/lib/promise/detail/basic_join.h - src/core/lib/promise/detail/basic_seq.h - src/core/lib/promise/detail/promise_factory.h - src/core/lib/promise/detail/promise_like.h - src/core/lib/promise/detail/status.h - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - - src/core/lib/promise/for_each.h + - src/core/lib/promise/if.h + - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/latch.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h - - src/core/lib/promise/map_pipe.h - src/core/lib/promise/pipe.h - src/core/lib/promise/poll.h - src/core/lib/promise/promise.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - src/core/lib/promise/sleep.h - - src/core/lib/promise/try_concurrently.h + - src/core/lib/promise/trace.h + - src/core/lib/promise/try_join.h - src/core/lib/promise/try_seq.h - src/core/lib/resolver/resolver.h - src/core/lib/resolver/resolver_factory.h @@ -2552,6 +2558,7 @@ libs: - src/core/lib/iomgr/buffer_list.cc - src/core/lib/iomgr/call_combiner.cc - src/core/lib/iomgr/cfstream_handle.cc + - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/dualstack_socket_posix.cc - src/core/lib/iomgr/endpoint.cc @@ -2631,8 +2638,8 @@ libs: - src/core/lib/load_balancing/lb_policy.cc - src/core/lib/load_balancing/lb_policy_registry.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/pipe.cc - src/core/lib/promise/sleep.cc + - src/core/lib/promise/trace.cc - src/core/lib/resolver/resolver.cc - src/core/lib/resolver/resolver_registry.cc - src/core/lib/resolver/server_address.cc @@ -2679,6 +2686,7 @@ libs: - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_buffer.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/surface/api_trace.cc - src/core/lib/surface/builtins.cc @@ -3703,14 +3711,16 @@ libs: - src/core/lib/promise/activity.h - src/core/lib/promise/arena_promise.h - src/core/lib/promise/context.h + - src/core/lib/promise/detail/basic_join.h - src/core/lib/promise/detail/basic_seq.h - src/core/lib/promise/detail/promise_factory.h - src/core/lib/promise/detail/promise_like.h - src/core/lib/promise/detail/status.h - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h + - src/core/lib/promise/if.h + - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h - - src/core/lib/promise/latch.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h - src/core/lib/promise/pipe.h @@ -3718,6 +3728,8 @@ libs: - src/core/lib/promise/promise.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h + - src/core/lib/promise/trace.h + - src/core/lib/promise/try_join.h - src/core/lib/promise/try_seq.h - src/core/lib/resolver/resolver.h - src/core/lib/resolver/resolver_factory.h @@ -3878,6 +3890,7 @@ libs: - src/core/lib/iomgr/buffer_list.cc - src/core/lib/iomgr/call_combiner.cc - src/core/lib/iomgr/cfstream_handle.cc + - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/dualstack_socket_posix.cc - src/core/lib/iomgr/endpoint.cc @@ -3957,7 +3970,7 @@ libs: - src/core/lib/load_balancing/lb_policy_registry.cc - src/core/lib/matchers/matchers.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/pipe.cc + - src/core/lib/promise/trace.cc - src/core/lib/resolver/resolver.cc - src/core/lib/resolver/resolver_registry.cc - src/core/lib/resolver/server_address.cc @@ -4004,6 +4017,7 @@ libs: - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_buffer.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/surface/api_trace.cc - src/core/lib/surface/builtins.cc @@ -5799,6 +5813,7 @@ targets: - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc + - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc @@ -5813,6 +5828,7 @@ targets: - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/gprpp/chunked_vector_test.cc deps: @@ -6797,6 +6813,7 @@ targets: - src/core/lib/debug/trace.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc + - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc @@ -6805,6 +6822,7 @@ targets: - src/core/lib/promise/activity.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/promise/exec_ctx_wakeup_scheduler_test.cc deps: @@ -7092,6 +7110,7 @@ targets: - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc + - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc @@ -7105,6 +7124,7 @@ targets: - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/transport/bdp_estimator.cc - src/core/lib/transport/pid_controller.cc @@ -7156,6 +7176,8 @@ targets: - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - src/core/lib/promise/for_each.h + - src/core/lib/promise/if.h + - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/join.h - src/core/lib/promise/loop.h @@ -7164,6 +7186,7 @@ targets: - src/core/lib/promise/poll.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h + - src/core/lib/promise/trace.h - src/core/lib/promise/try_seq.h - src/core/lib/resource_quota/arena.h - src/core/lib/resource_quota/memory_quota.h @@ -7186,13 +7209,14 @@ targets: - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc + - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc - src/core/lib/iomgr/executor.cc - src/core/lib/iomgr/iomgr_internal.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/pipe.cc + - src/core/lib/promise/trace.cc - src/core/lib/resource_quota/arena.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc @@ -7201,6 +7225,7 @@ targets: - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/promise/for_each_test.cc deps: @@ -7500,14 +7525,16 @@ targets: - src/core/lib/promise/activity.h - src/core/lib/promise/arena_promise.h - src/core/lib/promise/context.h + - src/core/lib/promise/detail/basic_join.h - src/core/lib/promise/detail/basic_seq.h - src/core/lib/promise/detail/promise_factory.h - src/core/lib/promise/detail/promise_like.h - src/core/lib/promise/detail/status.h - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h + - src/core/lib/promise/if.h + - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h - - src/core/lib/promise/latch.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h - src/core/lib/promise/pipe.h @@ -7515,6 +7542,9 @@ targets: - src/core/lib/promise/promise.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h + - src/core/lib/promise/trace.h + - src/core/lib/promise/try_join.h + - src/core/lib/promise/try_seq.h - src/core/lib/resolver/resolver.h - src/core/lib/resolver/resolver_factory.h - src/core/lib/resolver/resolver_registry.h @@ -7658,6 +7688,7 @@ targets: - src/core/lib/iomgr/buffer_list.cc - src/core/lib/iomgr/call_combiner.cc - src/core/lib/iomgr/cfstream_handle.cc + - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/dualstack_socket_posix.cc - src/core/lib/iomgr/endpoint.cc @@ -7736,7 +7767,7 @@ targets: - src/core/lib/load_balancing/lb_policy.cc - src/core/lib/load_balancing/lb_policy_registry.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/pipe.cc + - src/core/lib/promise/trace.cc - src/core/lib/resolver/resolver.cc - src/core/lib/resolver/resolver_registry.cc - src/core/lib/resolver/server_address.cc @@ -7760,6 +7791,7 @@ targets: - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_buffer.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/surface/api_trace.cc - src/core/lib/surface/builtins.cc @@ -8701,6 +8733,7 @@ targets: build: test language: c++ headers: + - src/core/lib/gprpp/construct_destruct.h - src/core/lib/promise/detail/promise_factory.h - src/core/lib/promise/detail/promise_like.h - src/core/lib/promise/if.h @@ -8769,6 +8802,98 @@ targets: - test/core/util/tracer_util.cc deps: - grpc_test_util +- name: interceptor_list_test + gtest: true + build: test + language: c++ + headers: + - src/core/ext/upb-generated/google/protobuf/any.upb.h + - src/core/ext/upb-generated/google/rpc/status.upb.h + - src/core/lib/debug/trace.h + - src/core/lib/experiments/config.h + - src/core/lib/experiments/experiments.h + - src/core/lib/gpr/spinlock.h + - src/core/lib/gprpp/atomic_utils.h + - src/core/lib/gprpp/bitset.h + - src/core/lib/gprpp/cpp_impl_of.h + - src/core/lib/gprpp/manual_constructor.h + - src/core/lib/gprpp/orphanable.h + - src/core/lib/gprpp/ref_counted.h + - src/core/lib/gprpp/ref_counted_ptr.h + - src/core/lib/gprpp/status_helper.h + - src/core/lib/gprpp/time.h + - src/core/lib/iomgr/closure.h + - src/core/lib/iomgr/combiner.h + - src/core/lib/iomgr/error.h + - src/core/lib/iomgr/exec_ctx.h + - src/core/lib/iomgr/executor.h + - src/core/lib/iomgr/iomgr_internal.h + - src/core/lib/promise/activity.h + - src/core/lib/promise/context.h + - src/core/lib/promise/detail/basic_seq.h + - src/core/lib/promise/detail/promise_factory.h + - src/core/lib/promise/detail/promise_like.h + - src/core/lib/promise/detail/status.h + - src/core/lib/promise/detail/switch.h + - src/core/lib/promise/exec_ctx_wakeup_scheduler.h + - src/core/lib/promise/interceptor_list.h + - src/core/lib/promise/loop.h + - src/core/lib/promise/map.h + - src/core/lib/promise/poll.h + - src/core/lib/promise/race.h + - src/core/lib/promise/seq.h + - src/core/lib/promise/trace.h + - src/core/lib/resource_quota/arena.h + - src/core/lib/resource_quota/memory_quota.h + - src/core/lib/resource_quota/periodic_update.h + - src/core/lib/resource_quota/resource_quota.h + - src/core/lib/resource_quota/thread_quota.h + - src/core/lib/resource_quota/trace.h + - src/core/lib/slice/percent_encoding.h + - src/core/lib/slice/slice.h + - src/core/lib/slice/slice_internal.h + - src/core/lib/slice/slice_refcount.h + - src/core/lib/slice/slice_string_helpers.h + - test/core/promise/test_context.h + src: + - src/core/ext/upb-generated/google/protobuf/any.upb.c + - src/core/ext/upb-generated/google/rpc/status.upb.c + - src/core/lib/debug/trace.cc + - src/core/lib/event_engine/memory_allocator.cc + - src/core/lib/experiments/config.cc + - src/core/lib/experiments/experiments.cc + - src/core/lib/gprpp/status_helper.cc + - src/core/lib/gprpp/time.cc + - src/core/lib/iomgr/closure.cc + - src/core/lib/iomgr/combiner.cc + - src/core/lib/iomgr/error.cc + - src/core/lib/iomgr/exec_ctx.cc + - src/core/lib/iomgr/executor.cc + - src/core/lib/iomgr/iomgr_internal.cc + - src/core/lib/promise/activity.cc + - src/core/lib/promise/trace.cc + - src/core/lib/resource_quota/arena.cc + - src/core/lib/resource_quota/memory_quota.cc + - src/core/lib/resource_quota/periodic_update.cc + - src/core/lib/resource_quota/resource_quota.cc + - src/core/lib/resource_quota/thread_quota.cc + - src/core/lib/resource_quota/trace.cc + - src/core/lib/slice/percent_encoding.cc + - src/core/lib/slice/slice.cc + - src/core/lib/slice/slice_refcount.cc + - src/core/lib/slice/slice_string_helpers.cc + - test/core/promise/interceptor_list_test.cc + deps: + - absl/container:flat_hash_set + - absl/functional:any_invocable + - absl/functional:function_ref + - absl/hash:hash + - absl/meta:type_traits + - absl/status:statusor + - absl/utility:utility + - gpr + - upb + uses_polling: false - name: interop_client build: test run: false @@ -8973,6 +9098,7 @@ targets: build: test language: c++ headers: + - src/core/lib/debug/trace.h - src/core/lib/gprpp/atomic_utils.h - src/core/lib/gprpp/bitset.h - src/core/lib/gprpp/orphanable.h @@ -8991,9 +9117,12 @@ targets: - src/core/lib/promise/latch.h - src/core/lib/promise/poll.h - src/core/lib/promise/seq.h + - src/core/lib/promise/trace.h - test/core/promise/test_wakeup_schedulers.h src: + - src/core/lib/debug/trace.cc - src/core/lib/promise/activity.cc + - src/core/lib/promise/trace.cc - test/core/promise/latch_test.cc deps: - absl/meta:type_traits @@ -9112,6 +9241,8 @@ targets: - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - src/core/lib/promise/for_each.h + - src/core/lib/promise/if.h + - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/join.h - src/core/lib/promise/loop.h @@ -9121,6 +9252,7 @@ targets: - src/core/lib/promise/poll.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h + - src/core/lib/promise/trace.h - src/core/lib/promise/try_seq.h - src/core/lib/resource_quota/arena.h - src/core/lib/resource_quota/memory_quota.h @@ -9143,13 +9275,14 @@ targets: - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc + - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc - src/core/lib/iomgr/executor.cc - src/core/lib/iomgr/iomgr_internal.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/pipe.cc + - src/core/lib/promise/trace.cc - src/core/lib/resource_quota/arena.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc @@ -9158,6 +9291,7 @@ targets: - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/promise/map_pipe_test.cc deps: @@ -9706,6 +9840,7 @@ targets: - src/core/lib/debug/trace.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc + - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc @@ -9714,6 +9849,7 @@ targets: - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/resource_quota/periodic_update_test.cc deps: @@ -9774,92 +9910,12 @@ targets: build: test language: c++ headers: - - src/core/ext/upb-generated/google/protobuf/any.upb.h - - src/core/ext/upb-generated/google/rpc/status.upb.h - - src/core/lib/debug/trace.h - - src/core/lib/experiments/config.h - - src/core/lib/experiments/experiments.h - - src/core/lib/gpr/spinlock.h - - src/core/lib/gprpp/atomic_utils.h - - src/core/lib/gprpp/bitset.h - - src/core/lib/gprpp/cpp_impl_of.h - - src/core/lib/gprpp/manual_constructor.h - - src/core/lib/gprpp/orphanable.h - - src/core/lib/gprpp/ref_counted.h - - src/core/lib/gprpp/ref_counted_ptr.h - - src/core/lib/gprpp/status_helper.h - - src/core/lib/gprpp/time.h - - src/core/lib/iomgr/closure.h - - src/core/lib/iomgr/combiner.h - - src/core/lib/iomgr/error.h - - src/core/lib/iomgr/exec_ctx.h - - src/core/lib/iomgr/executor.h - - src/core/lib/iomgr/iomgr_internal.h - - src/core/lib/promise/activity.h - - src/core/lib/promise/context.h - - src/core/lib/promise/detail/basic_join.h - - src/core/lib/promise/detail/basic_seq.h - - src/core/lib/promise/detail/promise_factory.h - - src/core/lib/promise/detail/promise_like.h - - src/core/lib/promise/detail/status.h - - src/core/lib/promise/detail/switch.h - - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/join.h - - src/core/lib/promise/loop.h - - src/core/lib/promise/map.h - - src/core/lib/promise/pipe.h - - src/core/lib/promise/poll.h - - src/core/lib/promise/race.h - - src/core/lib/promise/seq.h - - src/core/lib/resource_quota/arena.h - - src/core/lib/resource_quota/memory_quota.h - - src/core/lib/resource_quota/periodic_update.h - - src/core/lib/resource_quota/resource_quota.h - - src/core/lib/resource_quota/thread_quota.h - - src/core/lib/resource_quota/trace.h - - src/core/lib/slice/percent_encoding.h - - src/core/lib/slice/slice.h - - src/core/lib/slice/slice_internal.h - - src/core/lib/slice/slice_refcount.h - - src/core/lib/slice/slice_string_helpers.h - test/core/promise/test_wakeup_schedulers.h src: - - src/core/ext/upb-generated/google/protobuf/any.upb.c - - src/core/ext/upb-generated/google/rpc/status.upb.c - - src/core/lib/debug/trace.cc - - src/core/lib/event_engine/memory_allocator.cc - - src/core/lib/experiments/config.cc - - src/core/lib/experiments/experiments.cc - - src/core/lib/gprpp/status_helper.cc - - src/core/lib/gprpp/time.cc - - src/core/lib/iomgr/combiner.cc - - src/core/lib/iomgr/error.cc - - src/core/lib/iomgr/exec_ctx.cc - - src/core/lib/iomgr/executor.cc - - src/core/lib/iomgr/iomgr_internal.cc - - src/core/lib/promise/activity.cc - - src/core/lib/promise/pipe.cc - - src/core/lib/resource_quota/arena.cc - - src/core/lib/resource_quota/memory_quota.cc - - src/core/lib/resource_quota/periodic_update.cc - - src/core/lib/resource_quota/resource_quota.cc - - src/core/lib/resource_quota/thread_quota.cc - - src/core/lib/resource_quota/trace.cc - - src/core/lib/slice/percent_encoding.cc - - src/core/lib/slice/slice.cc - - src/core/lib/slice/slice_string_helpers.cc - test/core/promise/pipe_test.cc deps: - - absl/container:flat_hash_set - - absl/functional:any_invocable - - absl/functional:function_ref - - absl/hash:hash - - absl/meta:type_traits - - absl/status:statusor - - absl/utility:utility - - gpr - - upb + - grpc uses_polling: false - name: poll_test gtest: true @@ -10932,12 +10988,15 @@ targets: build: test language: c++ headers: + - src/core/lib/debug/trace.h - src/core/lib/slice/slice.h - src/core/lib/slice/slice_internal.h - src/core/lib/slice/slice_refcount.h - src/core/lib/slice/slice_string_helpers.h src: + - src/core/lib/debug/trace.cc - src/core/lib/slice/slice.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/slice/slice_string_helpers_test.cc deps: @@ -11561,6 +11620,7 @@ targets: build: test language: c++ headers: + - src/core/lib/debug/trace.h - src/core/lib/event_engine/handle_containers.h - src/core/lib/event_engine/resolved_address_internal.h - src/core/lib/iomgr/port.h @@ -11571,12 +11631,14 @@ targets: - src/core/lib/slice/slice_refcount.h - src/core/lib/slice/slice_string_helpers.h src: + - src/core/lib/debug/trace.cc - src/core/lib/event_engine/event_engine.cc - src/core/lib/event_engine/resolved_address.cc - src/core/lib/event_engine/slice.cc - src/core/lib/event_engine/slice_buffer.cc - src/core/lib/slice/slice.cc - src/core/lib/slice/slice_buffer.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/event_engine/slice_buffer_test.cc deps: @@ -12184,6 +12246,8 @@ targets: - src/core/lib/promise/detail/switch.h - src/core/lib/promise/exec_ctx_wakeup_scheduler.h - src/core/lib/promise/for_each.h + - src/core/lib/promise/if.h + - src/core/lib/promise/interceptor_list.h - src/core/lib/promise/intra_activity_waiter.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h @@ -12192,6 +12256,7 @@ targets: - src/core/lib/promise/poll.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h + - src/core/lib/promise/trace.h - src/core/lib/promise/try_concurrently.h - src/core/lib/promise/try_seq.h - src/core/lib/resource_quota/arena.h @@ -12212,19 +12277,21 @@ targets: - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc + - src/core/lib/iomgr/closure.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/exec_ctx.cc - src/core/lib/iomgr/executor.cc - src/core/lib/iomgr/iomgr_internal.cc - src/core/lib/promise/activity.cc - - src/core/lib/promise/pipe.cc + - src/core/lib/promise/trace.cc - src/core/lib/resource_quota/arena.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/trace.cc - src/core/lib/slice/percent_encoding.cc - src/core/lib/slice/slice.cc + - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - test/core/promise/try_concurrently_test.cc deps: diff --git a/config.m4 b/config.m4 index c30182fd782..876ef5c9e36 100644 --- a/config.m4 +++ b/config.m4 @@ -598,6 +598,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/cfstream_handle.cc \ + src/core/lib/iomgr/closure.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/dualstack_socket_posix.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -679,8 +680,8 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/load_balancing/lb_policy_registry.cc \ src/core/lib/matchers/matchers.cc \ src/core/lib/promise/activity.cc \ - src/core/lib/promise/pipe.cc \ src/core/lib/promise/sleep.cc \ + src/core/lib/promise/trace.cc \ src/core/lib/resolver/resolver.cc \ src/core/lib/resolver/resolver_registry.cc \ src/core/lib/resolver/server_address.cc \ @@ -758,6 +759,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/slice/percent_encoding.cc \ src/core/lib/slice/slice.cc \ src/core/lib/slice/slice_buffer.cc \ + src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/surface/api_trace.cc \ src/core/lib/surface/builtins.cc \ diff --git a/config.w32 b/config.w32 index 8131eadf4b1..616170f76e2 100644 --- a/config.w32 +++ b/config.w32 @@ -564,6 +564,7 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\iomgr\\buffer_list.cc " + "src\\core\\lib\\iomgr\\call_combiner.cc " + "src\\core\\lib\\iomgr\\cfstream_handle.cc " + + "src\\core\\lib\\iomgr\\closure.cc " + "src\\core\\lib\\iomgr\\combiner.cc " + "src\\core\\lib\\iomgr\\dualstack_socket_posix.cc " + "src\\core\\lib\\iomgr\\endpoint.cc " + @@ -645,8 +646,8 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\load_balancing\\lb_policy_registry.cc " + "src\\core\\lib\\matchers\\matchers.cc " + "src\\core\\lib\\promise\\activity.cc " + - "src\\core\\lib\\promise\\pipe.cc " + "src\\core\\lib\\promise\\sleep.cc " + + "src\\core\\lib\\promise\\trace.cc " + "src\\core\\lib\\resolver\\resolver.cc " + "src\\core\\lib\\resolver\\resolver_registry.cc " + "src\\core\\lib\\resolver\\server_address.cc " + @@ -724,6 +725,7 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\slice\\percent_encoding.cc " + "src\\core\\lib\\slice\\slice.cc " + "src\\core\\lib\\slice\\slice_buffer.cc " + + "src\\core\\lib\\slice\\slice_refcount.cc " + "src\\core\\lib\\slice\\slice_string_helpers.cc " + "src\\core\\lib\\surface\\api_trace.cc " + "src\\core\\lib\\surface\\builtins.cc " + diff --git a/doc/environment_variables.md b/doc/environment_variables.md index d2d0ba9e798..f5f10a285f3 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -118,6 +118,7 @@ some configuration as environment variables that can be set. - queue_refcount - error_refcount - stream_refcount + - slice_refcount - workqueue_refcount - fd_refcount - cq_refcount diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index adf5330fdb1..cb816108ad5 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -907,25 +907,27 @@ Pod::Spec.new do |s| 'src/core/lib/promise/activity.h', 'src/core/lib/promise/arena_promise.h', 'src/core/lib/promise/context.h', + 'src/core/lib/promise/detail/basic_join.h', 'src/core/lib/promise/detail/basic_seq.h', 'src/core/lib/promise/detail/promise_factory.h', 'src/core/lib/promise/detail/promise_like.h', 'src/core/lib/promise/detail/status.h', 'src/core/lib/promise/detail/switch.h', 'src/core/lib/promise/exec_ctx_wakeup_scheduler.h', - 'src/core/lib/promise/for_each.h', + 'src/core/lib/promise/if.h', + 'src/core/lib/promise/interceptor_list.h', 'src/core/lib/promise/intra_activity_waiter.h', 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', - 'src/core/lib/promise/map_pipe.h', 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', 'src/core/lib/promise/race.h', 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.h', - 'src/core/lib/promise/try_concurrently.h', + 'src/core/lib/promise/trace.h', + 'src/core/lib/promise/try_join.h', 'src/core/lib/promise/try_seq.h', 'src/core/lib/resolver/resolver.h', 'src/core/lib/resolver/resolver_factory.h', @@ -1834,25 +1836,27 @@ Pod::Spec.new do |s| 'src/core/lib/promise/activity.h', 'src/core/lib/promise/arena_promise.h', 'src/core/lib/promise/context.h', + 'src/core/lib/promise/detail/basic_join.h', 'src/core/lib/promise/detail/basic_seq.h', 'src/core/lib/promise/detail/promise_factory.h', 'src/core/lib/promise/detail/promise_like.h', 'src/core/lib/promise/detail/status.h', 'src/core/lib/promise/detail/switch.h', 'src/core/lib/promise/exec_ctx_wakeup_scheduler.h', - 'src/core/lib/promise/for_each.h', + 'src/core/lib/promise/if.h', + 'src/core/lib/promise/interceptor_list.h', 'src/core/lib/promise/intra_activity_waiter.h', 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', - 'src/core/lib/promise/map_pipe.h', 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', 'src/core/lib/promise/race.h', 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.h', - 'src/core/lib/promise/try_concurrently.h', + 'src/core/lib/promise/trace.h', + 'src/core/lib/promise/try_join.h', 'src/core/lib/promise/try_seq.h', 'src/core/lib/resolver/resolver.h', 'src/core/lib/resolver/resolver_factory.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index c5b87c8131b..6e7f4d423e8 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1321,6 +1321,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/call_combiner.h', 'src/core/lib/iomgr/cfstream_handle.cc', 'src/core/lib/iomgr/cfstream_handle.h', + 'src/core/lib/iomgr/closure.cc', 'src/core/lib/iomgr/closure.h', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/combiner.h', @@ -1476,19 +1477,19 @@ Pod::Spec.new do |s| 'src/core/lib/promise/activity.h', 'src/core/lib/promise/arena_promise.h', 'src/core/lib/promise/context.h', + 'src/core/lib/promise/detail/basic_join.h', 'src/core/lib/promise/detail/basic_seq.h', 'src/core/lib/promise/detail/promise_factory.h', 'src/core/lib/promise/detail/promise_like.h', 'src/core/lib/promise/detail/status.h', 'src/core/lib/promise/detail/switch.h', 'src/core/lib/promise/exec_ctx_wakeup_scheduler.h', - 'src/core/lib/promise/for_each.h', + 'src/core/lib/promise/if.h', + 'src/core/lib/promise/interceptor_list.h', 'src/core/lib/promise/intra_activity_waiter.h', 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', - 'src/core/lib/promise/map_pipe.h', - 'src/core/lib/promise/pipe.cc', 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', @@ -1496,7 +1497,9 @@ Pod::Spec.new do |s| 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.cc', 'src/core/lib/promise/sleep.h', - 'src/core/lib/promise/try_concurrently.h', + 'src/core/lib/promise/trace.cc', + 'src/core/lib/promise/trace.h', + 'src/core/lib/promise/try_join.h', 'src/core/lib/promise/try_seq.h', 'src/core/lib/resolver/resolver.cc', 'src/core/lib/resolver/resolver.h', @@ -1651,6 +1654,7 @@ Pod::Spec.new do |s| 'src/core/lib/slice/slice_buffer.cc', 'src/core/lib/slice/slice_buffer.h', 'src/core/lib/slice/slice_internal.h', + 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_refcount.h', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/slice/slice_string_helpers.h', @@ -2517,25 +2521,27 @@ Pod::Spec.new do |s| 'src/core/lib/promise/activity.h', 'src/core/lib/promise/arena_promise.h', 'src/core/lib/promise/context.h', + 'src/core/lib/promise/detail/basic_join.h', 'src/core/lib/promise/detail/basic_seq.h', 'src/core/lib/promise/detail/promise_factory.h', 'src/core/lib/promise/detail/promise_like.h', 'src/core/lib/promise/detail/status.h', 'src/core/lib/promise/detail/switch.h', 'src/core/lib/promise/exec_ctx_wakeup_scheduler.h', - 'src/core/lib/promise/for_each.h', + 'src/core/lib/promise/if.h', + 'src/core/lib/promise/interceptor_list.h', 'src/core/lib/promise/intra_activity_waiter.h', 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', - 'src/core/lib/promise/map_pipe.h', 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', 'src/core/lib/promise/race.h', 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.h', - 'src/core/lib/promise/try_concurrently.h', + 'src/core/lib/promise/trace.h', + 'src/core/lib/promise/try_join.h', 'src/core/lib/promise/try_seq.h', 'src/core/lib/resolver/resolver.h', 'src/core/lib/resolver/resolver_factory.h', diff --git a/grpc.gemspec b/grpc.gemspec index 4da80f8c12c..5cce81b11c8 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1230,6 +1230,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/call_combiner.h ) s.files += %w( src/core/lib/iomgr/cfstream_handle.cc ) s.files += %w( src/core/lib/iomgr/cfstream_handle.h ) + s.files += %w( src/core/lib/iomgr/closure.cc ) s.files += %w( src/core/lib/iomgr/closure.h ) s.files += %w( src/core/lib/iomgr/combiner.cc ) s.files += %w( src/core/lib/iomgr/combiner.h ) @@ -1385,19 +1386,19 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/promise/activity.h ) s.files += %w( src/core/lib/promise/arena_promise.h ) s.files += %w( src/core/lib/promise/context.h ) + s.files += %w( src/core/lib/promise/detail/basic_join.h ) s.files += %w( src/core/lib/promise/detail/basic_seq.h ) s.files += %w( src/core/lib/promise/detail/promise_factory.h ) s.files += %w( src/core/lib/promise/detail/promise_like.h ) s.files += %w( src/core/lib/promise/detail/status.h ) s.files += %w( src/core/lib/promise/detail/switch.h ) s.files += %w( src/core/lib/promise/exec_ctx_wakeup_scheduler.h ) - s.files += %w( src/core/lib/promise/for_each.h ) + s.files += %w( src/core/lib/promise/if.h ) + s.files += %w( src/core/lib/promise/interceptor_list.h ) s.files += %w( src/core/lib/promise/intra_activity_waiter.h ) s.files += %w( src/core/lib/promise/latch.h ) s.files += %w( src/core/lib/promise/loop.h ) s.files += %w( src/core/lib/promise/map.h ) - s.files += %w( src/core/lib/promise/map_pipe.h ) - s.files += %w( src/core/lib/promise/pipe.cc ) s.files += %w( src/core/lib/promise/pipe.h ) s.files += %w( src/core/lib/promise/poll.h ) s.files += %w( src/core/lib/promise/promise.h ) @@ -1405,7 +1406,9 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/promise/seq.h ) s.files += %w( src/core/lib/promise/sleep.cc ) s.files += %w( src/core/lib/promise/sleep.h ) - s.files += %w( src/core/lib/promise/try_concurrently.h ) + s.files += %w( src/core/lib/promise/trace.cc ) + s.files += %w( src/core/lib/promise/trace.h ) + s.files += %w( src/core/lib/promise/try_join.h ) s.files += %w( src/core/lib/promise/try_seq.h ) s.files += %w( src/core/lib/resolver/resolver.cc ) s.files += %w( src/core/lib/resolver/resolver.h ) @@ -1560,6 +1563,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/slice/slice_buffer.cc ) s.files += %w( src/core/lib/slice/slice_buffer.h ) s.files += %w( src/core/lib/slice/slice_internal.h ) + s.files += %w( src/core/lib/slice/slice_refcount.cc ) s.files += %w( src/core/lib/slice/slice_refcount.h ) s.files += %w( src/core/lib/slice/slice_string_helpers.cc ) s.files += %w( src/core/lib/slice/slice_string_helpers.h ) diff --git a/grpc.gyp b/grpc.gyp index aac09f741a8..42a1ac2eb33 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -886,6 +886,7 @@ 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/cfstream_handle.cc', + 'src/core/lib/iomgr/closure.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/dualstack_socket_posix.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -967,8 +968,8 @@ 'src/core/lib/load_balancing/lb_policy_registry.cc', 'src/core/lib/matchers/matchers.cc', 'src/core/lib/promise/activity.cc', - 'src/core/lib/promise/pipe.cc', 'src/core/lib/promise/sleep.cc', + 'src/core/lib/promise/trace.cc', 'src/core/lib/resolver/resolver.cc', 'src/core/lib/resolver/resolver_registry.cc', 'src/core/lib/resolver/server_address.cc', @@ -1046,6 +1047,7 @@ 'src/core/lib/slice/percent_encoding.cc', 'src/core/lib/slice/slice.cc', 'src/core/lib/slice/slice_buffer.cc', + 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', 'src/core/lib/surface/builtins.cc', @@ -1364,6 +1366,7 @@ 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/cfstream_handle.cc', + 'src/core/lib/iomgr/closure.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/dualstack_socket_posix.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -1443,8 +1446,8 @@ 'src/core/lib/load_balancing/lb_policy.cc', 'src/core/lib/load_balancing/lb_policy_registry.cc', 'src/core/lib/promise/activity.cc', - 'src/core/lib/promise/pipe.cc', 'src/core/lib/promise/sleep.cc', + 'src/core/lib/promise/trace.cc', 'src/core/lib/resolver/resolver.cc', 'src/core/lib/resolver/resolver_registry.cc', 'src/core/lib/resolver/server_address.cc', @@ -1491,6 +1494,7 @@ 'src/core/lib/slice/percent_encoding.cc', 'src/core/lib/slice/slice.cc', 'src/core/lib/slice/slice_buffer.cc', + 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', 'src/core/lib/surface/builtins.cc', @@ -1865,6 +1869,7 @@ 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/cfstream_handle.cc', + 'src/core/lib/iomgr/closure.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/dualstack_socket_posix.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -1944,7 +1949,7 @@ 'src/core/lib/load_balancing/lb_policy_registry.cc', 'src/core/lib/matchers/matchers.cc', 'src/core/lib/promise/activity.cc', - 'src/core/lib/promise/pipe.cc', + 'src/core/lib/promise/trace.cc', 'src/core/lib/resolver/resolver.cc', 'src/core/lib/resolver/resolver_registry.cc', 'src/core/lib/resolver/server_address.cc', @@ -1991,6 +1996,7 @@ 'src/core/lib/slice/percent_encoding.cc', 'src/core/lib/slice/slice.cc', 'src/core/lib/slice/slice_buffer.cc', + 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', 'src/core/lib/surface/builtins.cc', diff --git a/package.xml b/package.xml index 3f97548e386..8ffbd092a4f 100644 --- a/package.xml +++ b/package.xml @@ -1212,6 +1212,7 @@ + @@ -1367,19 +1368,19 @@ + - + + - - @@ -1387,7 +1388,9 @@ - + + + @@ -1542,6 +1545,7 @@ + diff --git a/src/core/BUILD b/src/core/BUILD index 4ccff69a079..3b71c5dd4ac 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -380,6 +380,7 @@ grpc_cc_library( hdrs = ["lib/promise/try_concurrently.h"], external_deps = [ "absl/status", + "absl/strings", "absl/types:variant", ], language = "c++", @@ -387,6 +388,7 @@ grpc_cc_library( "lib/promise/map_pipe.h", ], deps = [ + "activity", "construct_destruct", "for_each", "map", @@ -395,8 +397,10 @@ grpc_cc_library( "promise_factory", "promise_like", "promise_status", + "promise_trace", "try_seq", "//:gpr", + "//:grpc_trace", ], ) @@ -411,9 +415,13 @@ grpc_cc_library( "for_each", "map", "pipe", + "poll", "promise_factory", + "promise_trace", "try_seq", + "//:gpr", "//:gpr_platform", + "//:grpc_trace", ], ) @@ -524,6 +532,7 @@ grpc_cc_library( language = "c++", public_hdrs = ["lib/promise/if.h"], deps = [ + "construct_destruct", "poll", "promise_factory", "promise_like", @@ -779,14 +788,18 @@ grpc_cc_library( grpc_cc_library( name = "latch", + external_deps = ["absl/strings"], language = "c++", public_hdrs = [ "lib/promise/latch.h", ], deps = [ + "activity", "intra_activity_waiter", "poll", + "promise_trace", "//:gpr", + "//:grpc_trace", ], ) @@ -811,14 +824,37 @@ grpc_cc_library( ) grpc_cc_library( - name = "pipe", - srcs = [ - "lib/promise/pipe.cc", + name = "interceptor_list", + hdrs = [ + "lib/promise/interceptor_list.h", + ], + external_deps = [ + "absl/strings", + "absl/strings:str_format", + "absl/types:optional", + "absl/types:variant", ], + deps = [ + "arena", + "construct_destruct", + "context", + "poll", + "promise_factory", + "promise_trace", + "//:debug_location", + "//:gpr", + "//:grpc_trace", + ], +) + +grpc_cc_library( + name = "pipe", hdrs = [ "lib/promise/pipe.h", ], external_deps = [ + "absl/base:core_headers", + "absl/memory", "absl/strings", "absl/types:optional", "absl/types:variant", @@ -828,10 +864,32 @@ grpc_cc_library( "activity", "arena", "context", + "if", + "interceptor_list", "intra_activity_waiter", + "map", "poll", + "promise_trace", + "seq", + "//:debug_location", "//:gpr", "//:grpc_trace", + "//:ref_counted_ptr", + ], +) + +grpc_cc_library( + name = "promise_trace", + srcs = [ + "lib/promise/trace.cc", + ], + hdrs = [ + "lib/promise/trace.h", + ], + language = "c++", + deps = [ + "//:gpr_platform", + "//:grpc_trace", ], ) @@ -856,16 +914,20 @@ grpc_cc_library( name = "for_each", external_deps = [ "absl/status", + "absl/strings", "absl/types:variant", ], language = "c++", public_hdrs = ["lib/promise/for_each.h"], deps = [ + "activity", "construct_destruct", "poll", "promise_factory", + "promise_trace", "//:gpr", "//:gpr_platform", + "//:grpc_trace", ], ) @@ -1123,6 +1185,9 @@ grpc_cc_library( grpc_cc_library( name = "slice_refcount", + srcs = [ + "lib/slice/slice_refcount.cc", + ], hdrs = [ "lib/slice/slice_refcount.h", ], @@ -1130,8 +1195,10 @@ grpc_cc_library( "//:include/grpc/slice.h", ], deps = [ + "//:debug_location", "//:event_engine_base_hdrs", "//:gpr", + "//:grpc_trace", ], ) @@ -1155,6 +1222,7 @@ grpc_cc_library( deps = [ "slice_cast", "slice_refcount", + "//:debug_location", "//:event_engine_base_hdrs", "//:gpr", ], @@ -1169,6 +1237,7 @@ grpc_cc_library( "lib/slice/slice_buffer.h", "//:include/grpc/slice_buffer.h", ], + external_deps = ["absl/memory"], deps = [ "slice", "slice_refcount", @@ -1204,9 +1273,13 @@ grpc_cc_library( grpc_cc_library( name = "closure", + srcs = [ + "lib/iomgr/closure.cc", + ], hdrs = [ "lib/iomgr/closure.h", ], + external_deps = ["absl/strings:str_format"], visibility = ["@grpc:alt_grpc_base_legacy"], deps = [ "error", @@ -3514,7 +3587,6 @@ grpc_cc_library( "ext/filters/stateful_session/stateful_session_service_config_parser.h", ], external_deps = [ - "absl/status", "absl/status:statusor", "absl/strings", "absl/types:optional", @@ -3523,7 +3595,6 @@ grpc_cc_library( deps = [ "arena", "arena_promise", - "basic_seq", "channel_args", "channel_fwd", "context", @@ -3531,12 +3602,12 @@ grpc_cc_library( "json", "json_args", "json_object_loader", - "latch", - "seq", + "map", + "pipe", + "poll", "service_config_parser", "slice", "time", - "try_concurrently", "unique_type_name", "validation_errors", "//:config", @@ -3574,26 +3645,29 @@ grpc_cc_library( ], language = "c++", deps = [ + "arena", "arena_promise", "channel_args", "channel_fwd", "channel_init", "channel_stack_type", "closure", + "context", "error", "gpr_atm", "grpc_sockaddr", "json", "json_args", "json_object_loader", - "latch", "lb_policy", "lb_policy_factory", "lb_policy_registry", + "map", + "pipe", + "poll", "pollset_set", "ref_counted", "resolved_address", - "seq", "slice", "slice_refcount", "status_helper", @@ -3617,7 +3691,6 @@ grpc_cc_library( "//:grpc_security_base", "//:grpc_trace", "//:orphanable", - "//:promise", "//:protobuf_duration_upb", "//:protobuf_timestamp_upb", "//:ref_counted_ptr", diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc index 45e875ba2a6..d9e979b7c43 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc @@ -31,9 +31,11 @@ #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/promise/latch.h" -#include "src/core/lib/promise/promise.h" -#include "src/core/lib/promise/seq.h" +#include "src/core/lib/promise/context.h" +#include "src/core/lib/promise/map.h" +#include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/poll.h" +#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -61,16 +63,21 @@ ArenaPromise ClientLoadReportingFilter::MakeCallPromise( client_stats.reset(*client_stats_md); } - auto* server_initial_metadata = call_args.server_initial_metadata; + auto* saw_initial_metadata = GetContext()->New(false); + call_args.server_initial_metadata->InterceptAndMap( + [saw_initial_metadata](ServerMetadataHandle md) { + *saw_initial_metadata = true; + return md; + }); - return Seq(next_promise_factory(std::move(call_args)), - [server_initial_metadata, client_stats = std::move(client_stats)]( + return Map(next_promise_factory(std::move(call_args)), + [saw_initial_metadata, client_stats = std::move(client_stats)]( ServerMetadataHandle trailing_metadata) { if (client_stats != nullptr) { client_stats->AddCallFinished( trailing_metadata->get(GrpcStreamNetworkState()) == GrpcStreamNetworkState::kNotSentOnWire, - NowOrNever(server_initial_metadata->Wait()).has_value()); + *saw_initial_metadata); } return trailing_metadata; }); diff --git a/src/core/ext/filters/http/client/http_client_filter.cc b/src/core/ext/filters/http/client/http_client_filter.cc index 7e299e3f2c7..58f9a709024 100644 --- a/src/core/ext/filters/http/client/http_client_filter.cc +++ b/src/core/ext/filters/http/client/http_client_filter.cc @@ -40,10 +40,11 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/detail/basic_seq.h" #include "src/core/lib/promise/latch.h" -#include "src/core/lib/promise/seq.h" -#include "src/core/lib/promise/try_concurrently.h" +#include "src/core/lib/promise/map.h" +#include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/race.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/percent_encoding.h" #include "src/core/lib/transport/status_conversion.h" @@ -118,24 +119,27 @@ ArenaPromise HttpClientFilter::MakeCallPromise( md->Set(ContentTypeMetadata(), ContentTypeMetadata::kApplicationGrpc); md->Set(UserAgentMetadata(), user_agent_.Ref()); - auto* read_latch = GetContext()->New>(); - auto* write_latch = - std::exchange(call_args.server_initial_metadata, read_latch); - - return TryConcurrently( - Seq(next_promise_factory(std::move(call_args)), - [](ServerMetadataHandle md) -> ServerMetadataHandle { - auto r = CheckServerMetadata(md.get()); - if (!r.ok()) return ServerMetadataFromStatus(r); - return md; - })) - .NecessaryPull(Seq(read_latch->Wait(), - [write_latch](ServerMetadata** md) -> absl::Status { - auto r = *md == nullptr ? absl::OkStatus() - : CheckServerMetadata(*md); - write_latch->Set(*md); - return r; - })); + auto* initial_metadata_err = + GetContext()->New>(); + + call_args.server_initial_metadata->InterceptAndMap( + [initial_metadata_err]( + ServerMetadataHandle md) -> absl::optional { + auto r = CheckServerMetadata(md.get()); + if (!r.ok()) { + initial_metadata_err->Set(ServerMetadataFromStatus(r)); + return absl::nullopt; + } + return std::move(md); + }); + + return Race(Map(next_promise_factory(std::move(call_args)), + [](ServerMetadataHandle md) -> ServerMetadataHandle { + auto r = CheckServerMetadata(md.get()); + if (!r.ok()) return ServerMetadataFromStatus(r); + return md; + }), + initial_metadata_err->Wait()); } HttpClientFilter::HttpClientFilter(HttpSchemeMetadata::ValueType scheme, diff --git a/src/core/ext/filters/http/message_compress/compression_filter.cc b/src/core/ext/filters/http/message_compress/compression_filter.cc index 69a7f0a25f7..5a84aceefe5 100644 --- a/src/core/ext/filters/http/message_compress/compression_filter.cc +++ b/src/core/ext/filters/http/message_compress/compression_filter.cc @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include "absl/meta/type_traits.h" @@ -43,17 +43,16 @@ #include "src/core/lib/compression/compression_internal.h" #include "src/core/lib/compression/message_compress.h" #include "src/core/lib/debug/trace.h" +#include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/detail/promise_like.h" #include "src/core/lib/promise/latch.h" -#include "src/core/lib/promise/map_pipe.h" #include "src/core/lib/promise/pipe.h" -#include "src/core/lib/promise/promise.h" -#include "src/core/lib/promise/seq.h" -#include "src/core/lib/promise/try_concurrently.h" +#include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/race.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/call.h" +#include "src/core/lib/surface/call_trace.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -158,20 +157,19 @@ MessageHandle CompressionFilter::CompressMessage( } absl::StatusOr CompressionFilter::DecompressMessage( - MessageHandle message, grpc_compression_algorithm algorithm, - absl::optional max_recv_message_length) const { + MessageHandle message, DecompressArgs args) const { if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { gpr_log(GPR_ERROR, "DecompressMessage: len=%" PRIdPTR " max=%d alg=%d", - message->payload()->Length(), max_recv_message_length.value_or(-1), - algorithm); + message->payload()->Length(), + args.max_recv_message_length.value_or(-1), args.algorithm); } // Check max message length. - if (max_recv_message_length.has_value() && + if (args.max_recv_message_length.has_value() && message->payload()->Length() > - static_cast(*max_recv_message_length)) { + static_cast(*args.max_recv_message_length)) { return absl::ResourceExhaustedError(absl::StrFormat( "Received message larger than max (%u vs. %d)", - message->payload()->Length(), *max_recv_message_length)); + message->payload()->Length(), *args.max_recv_message_length)); } // Check if decompression is enabled (if not, we can just pass the message // up). @@ -181,11 +179,11 @@ absl::StatusOr CompressionFilter::DecompressMessage( } // Try to decompress the payload. SliceBuffer decompressed_slices; - if (grpc_msg_decompress(algorithm, message->payload()->c_slice_buffer(), + if (grpc_msg_decompress(args.algorithm, message->payload()->c_slice_buffer(), decompressed_slices.c_slice_buffer()) == 0) { return absl::InternalError( absl::StrCat("Unexpected error decompressing data for algorithm ", - CompressionAlgorithmAsString(algorithm))); + CompressionAlgorithmAsString(args.algorithm))); } // Swap the decompressed slices into the message. message->payload()->Swap(&decompressed_slices); @@ -194,123 +192,114 @@ absl::StatusOr CompressionFilter::DecompressMessage( return std::move(message); } -class CompressionFilter::DecompressLoop { - public: - explicit DecompressLoop(CompressionFilter* filter, CallArgs& call_args) - : filter_(filter), - mapper_(PipeMapper::Intercept( - *call_args.incoming_messages)) {} - - // Once we have a compression algorithm we can construct the decompression - // loop. - // Returns a promise that resolves to MessageHandle. - auto TakeAndRun(grpc_compression_algorithm algorithm) { - // Configure max receive size. - auto max_recv_message_length = filter_->max_recv_size_; - const MessageSizeParsedConfig* limits = - MessageSizeParsedConfig::GetFromCallContext( - GetContext(), - filter_->message_size_service_config_parser_index_); - if (limits != nullptr && limits->max_recv_size().has_value() && - (!max_recv_message_length.has_value() || - *limits->max_recv_size() < *max_recv_message_length)) { - max_recv_message_length = *limits->max_recv_size(); - } - // Interject decompression into the message loop. - return mapper_.TakeAndRun([algorithm, max_recv_message_length, - filter = filter_](MessageHandle message) { - return filter->DecompressMessage(std::move(message), algorithm, - max_recv_message_length); - }); +grpc_compression_algorithm CompressionFilter::HandleOutgoingMetadata( + grpc_metadata_batch& outgoing_metadata) { + const auto algorithm = outgoing_metadata.Take(GrpcInternalEncodingRequest()) + .value_or(default_compression_algorithm()); + // Convey supported compression algorithms. + outgoing_metadata.Set(GrpcAcceptEncodingMetadata(), + enabled_compression_algorithms()); + if (algorithm != GRPC_COMPRESS_NONE) { + outgoing_metadata.Set(GrpcEncodingMetadata(), algorithm); } + return algorithm; +} - private: - CompressionFilter* filter_; - PipeMapper mapper_; -}; - -class CompressionFilter::CompressLoop { - public: - explicit CompressLoop(CompressionFilter* filter, CallArgs& call_args) - : filter_(filter), - mapper_(PipeMapper::Intercept( - *call_args.outgoing_messages)) {} - - // Once we're ready to send initial metadata we can construct the compression - // loop. - // Returns a promise that resolves to MessageHandle. - auto TakeAndRun(grpc_metadata_batch& outgoing_metadata) { - const auto algorithm = - outgoing_metadata.Take(GrpcInternalEncodingRequest()) - .value_or(filter_->default_compression_algorithm()); - // Convey supported compression algorithms. - outgoing_metadata.Set(GrpcAcceptEncodingMetadata(), - filter_->enabled_compression_algorithms()); - if (algorithm != GRPC_COMPRESS_NONE) { - outgoing_metadata.Set(GrpcEncodingMetadata(), algorithm); - } - // Interject compression into the message loop. - return mapper_.TakeAndRun([filter = filter_, algorithm](MessageHandle m) { - return filter->CompressMessage(std::move(m), algorithm); - }); +CompressionFilter::DecompressArgs CompressionFilter::HandleIncomingMetadata( + const grpc_metadata_batch& incoming_metadata) { + // Configure max receive size. + auto max_recv_message_length = max_recv_size_; + const MessageSizeParsedConfig* limits = + MessageSizeParsedConfig::GetFromCallContext( + GetContext(), + message_size_service_config_parser_index_); + if (limits != nullptr && limits->max_recv_size().has_value() && + (!max_recv_message_length.has_value() || + *limits->max_recv_size() < *max_recv_message_length)) { + max_recv_message_length = *limits->max_recv_size(); } - - private: - CompressionFilter* filter_; - PipeMapper mapper_; -}; + return DecompressArgs{incoming_metadata.get(GrpcEncodingMetadata()) + .value_or(GRPC_COMPRESS_NONE), + max_recv_message_length}; +} ArenaPromise ClientCompressionFilter::MakeCallPromise( CallArgs call_args, NextPromiseFactory next_promise_factory) { - auto compress_loop = CompressLoop(this, call_args) - .TakeAndRun(*call_args.client_initial_metadata); - DecompressLoop decompress_loop(this, call_args); - auto* server_initial_metadata = call_args.server_initial_metadata; - // Concurrently: - // - call the next filter - // - wait for initial metadata from the server and then commence decompression - // - compress outgoing messages - return TryConcurrently(next_promise_factory(std::move(call_args))) - .NecessaryPull(Seq(server_initial_metadata->Wait(), - [decompress_loop = std::move(decompress_loop)]( - ServerMetadata** server_initial_metadata) mutable - -> ArenaPromise { - if (*server_initial_metadata == nullptr) { - return ImmediateOkStatus(); - } - return decompress_loop.TakeAndRun( - (*server_initial_metadata) - ->get(GrpcEncodingMetadata()) - .value_or(GRPC_COMPRESS_NONE)); - })) - .Push(std::move(compress_loop)); + auto compression_algorithm = + HandleOutgoingMetadata(*call_args.client_initial_metadata); + call_args.client_to_server_messages->InterceptAndMap( + [compression_algorithm, + this](MessageHandle message) -> absl::optional { + return CompressMessage(std::move(message), compression_algorithm); + }); + auto* decompress_args = GetContext()->New( + DecompressArgs{GRPC_COMPRESS_NONE, absl::nullopt}); + auto* decompress_err = + GetContext()->New>(); + call_args.server_initial_metadata->InterceptAndMap( + [decompress_args, this](ServerMetadataHandle server_initial_metadata) + -> absl::optional { + if (server_initial_metadata == nullptr) return absl::nullopt; + *decompress_args = HandleIncomingMetadata(*server_initial_metadata); + return std::move(server_initial_metadata); + }); + call_args.server_to_client_messages->InterceptAndMap( + [decompress_err, decompress_args, + this](MessageHandle message) -> absl::optional { + auto r = DecompressMessage(std::move(message), *decompress_args); + if (!r.ok()) { + decompress_err->Set(ServerMetadataFromStatus(r.status())); + return absl::nullopt; + } + return std::move(*r); + }); + // Run the next filter, and race it with getting an error from decompression. + return Race(next_promise_factory(std::move(call_args)), + decompress_err->Wait()); } ArenaPromise ServerCompressionFilter::MakeCallPromise( CallArgs call_args, NextPromiseFactory next_promise_factory) { - CompressLoop compress_loop(this, call_args); - auto decompress_loop = DecompressLoop(this, call_args) - .TakeAndRun(call_args.client_initial_metadata - ->get(GrpcEncodingMetadata()) - .value_or(GRPC_COMPRESS_NONE)); - auto* read_latch = GetContext()->New>(); - auto* write_latch = - std::exchange(call_args.server_initial_metadata, read_latch); + auto decompress_args = + HandleIncomingMetadata(*call_args.client_initial_metadata); + auto* decompress_err = + GetContext()->New>(); + call_args.client_to_server_messages->InterceptAndMap( + [decompress_err, decompress_args, + this](MessageHandle message) -> absl::optional { + auto r = DecompressMessage(std::move(message), decompress_args); + gpr_log(GPR_DEBUG, "DecompressMessage returned %s", + r.status().ToString().c_str()); + if (!r.ok()) { + decompress_err->Set(ServerMetadataFromStatus(r.status())); + return absl::nullopt; + } + return std::move(*r); + }); + auto* compression_algorithm = + GetContext()->New(); + call_args.server_initial_metadata->InterceptAndMap( + [this, compression_algorithm](ServerMetadataHandle md) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[compression] Write metadata", + Activity::current()->DebugTag().c_str()); + } + // Find the compression algorithm. + *compression_algorithm = HandleOutgoingMetadata(*md); + return md; + }); + call_args.server_to_client_messages->InterceptAndMap( + [compression_algorithm, + this](MessageHandle message) -> absl::optional { + return CompressMessage(std::move(message), *compression_algorithm); + }); // Concurrently: // - call the next filter // - decompress incoming messages // - wait for initial metadata to be sent, and then commence compression of // outgoing messages - return TryConcurrently(next_promise_factory(std::move(call_args))) - .Pull(std::move(decompress_loop)) - .Push(Seq(read_latch->Wait(), - [write_latch, compress_loop = std::move(compress_loop)]( - ServerMetadata** md) mutable { - // Find the compression algorithm. - auto loop = compress_loop.TakeAndRun(**md); - write_latch->Set(*md); - return loop; - })); + return Race(next_promise_factory(std::move(call_args)), + decompress_err->Wait()); } } // namespace grpc_core diff --git a/src/core/ext/filters/http/message_compress/compression_filter.h b/src/core/ext/filters/http/message_compress/compression_filter.h index 8a17bebe0dc..caf49c03336 100644 --- a/src/core/ext/filters/http/message_compress/compression_filter.h +++ b/src/core/ext/filters/http/message_compress/compression_filter.h @@ -34,6 +34,7 @@ #include "src/core/lib/channel/promise_based_filter.h" #include "src/core/lib/compression/compression_internal.h" #include "src/core/lib/promise/arena_promise.h" +#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" namespace grpc_core { @@ -62,10 +63,12 @@ namespace grpc_core { class CompressionFilter : public ChannelFilter { protected: - explicit CompressionFilter(const ChannelArgs& args); + struct DecompressArgs { + grpc_compression_algorithm algorithm; + absl::optional max_recv_message_length; + }; - class CompressLoop; - class DecompressLoop; + explicit CompressionFilter(const ChannelArgs& args); grpc_compression_algorithm default_compression_algorithm() const { return default_compression_algorithm_; @@ -75,15 +78,19 @@ class CompressionFilter : public ChannelFilter { return enabled_compression_algorithms_; } - private: + grpc_compression_algorithm HandleOutgoingMetadata( + grpc_metadata_batch& outgoing_metadata); + DecompressArgs HandleIncomingMetadata( + const grpc_metadata_batch& incoming_metadata); + // Compress one message synchronously. MessageHandle CompressMessage(MessageHandle message, grpc_compression_algorithm algorithm) const; // Decompress one message synchronously. - absl::StatusOr DecompressMessage( - MessageHandle message, grpc_compression_algorithm algorithm, - absl::optional max_recv_message_length) const; + absl::StatusOr DecompressMessage(MessageHandle message, + DecompressArgs args) const; + private: // Max receive message length, if set. absl::optional max_recv_size_; size_t message_size_service_config_parser_index_; diff --git a/src/core/ext/filters/http/server/http_server_filter.cc b/src/core/ext/filters/http/server/http_server_filter.cc index 23062d5c007..8ec0e54df0b 100644 --- a/src/core/ext/filters/http/server/http_server_filter.cc +++ b/src/core/ext/filters/http/server/http_server_filter.cc @@ -22,6 +22,7 @@ #include #include +#include #include #include "absl/base/attributes.h" @@ -30,18 +31,19 @@ #include "absl/types/optional.h" #include +#include #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/detail/basic_seq.h" -#include "src/core/lib/promise/latch.h" +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/promise/activity.h" +#include "src/core/lib/promise/map.h" +#include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/poll.h" #include "src/core/lib/promise/promise.h" -#include "src/core/lib/promise/seq.h" -#include "src/core/lib/promise/try_concurrently.h" -#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/percent_encoding.h" #include "src/core/lib/slice/slice.h" +#include "src/core/lib/surface/call_trace.h" #include "src/core/lib/transport/metadata_batch.h" namespace grpc_core { @@ -129,24 +131,23 @@ ArenaPromise HttpServerFilter::MakeCallPromise( md->Remove(UserAgentMetadata()); } - auto* read_latch = GetContext()->New>(); - auto* write_latch = - std::exchange(call_args.server_initial_metadata, read_latch); - - return TryConcurrently( - Seq(next_promise_factory(std::move(call_args)), - [](ServerMetadataHandle md) -> ServerMetadataHandle { - FilterOutgoingMetadata(md.get()); - return md; - })) - .Push(Seq(read_latch->Wait(), [write_latch](ServerMetadata** md) { - FilterOutgoingMetadata(*md); - (*md)->Set(HttpStatusMetadata(), 200); - (*md)->Set(ContentTypeMetadata(), - ContentTypeMetadata::kApplicationGrpc); - write_latch->Set(*md); - return absl::OkStatus(); - })); + call_args.server_initial_metadata->InterceptAndMap( + [](ServerMetadataHandle md) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[http-server] Write metadata", + Activity::current()->DebugTag().c_str()); + } + FilterOutgoingMetadata(md.get()); + md->Set(HttpStatusMetadata(), 200); + md->Set(ContentTypeMetadata(), ContentTypeMetadata::kApplicationGrpc); + return md; + }); + + return Map(next_promise_factory(std::move(call_args)), + [](ServerMetadataHandle md) -> ServerMetadataHandle { + FilterOutgoingMetadata(md.get()); + return md; + }); } absl::StatusOr HttpServerFilter::Create( diff --git a/src/core/ext/filters/stateful_session/stateful_session_filter.cc b/src/core/ext/filters/stateful_session/stateful_session_filter.cc index cefed052921..e64975fd9b4 100644 --- a/src/core/ext/filters/stateful_session/stateful_session_filter.cc +++ b/src/core/ext/filters/stateful_session/stateful_session_filter.cc @@ -27,7 +27,6 @@ #include #include -#include "absl/status/status.h" #include "absl/strings/escaping.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" @@ -47,10 +46,9 @@ #include "src/core/lib/gprpp/crash.h" #include "src/core/lib/gprpp/time.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/detail/basic_seq.h" -#include "src/core/lib/promise/latch.h" -#include "src/core/lib/promise/seq.h" -#include "src/core/lib/promise/try_concurrently.h" +#include "src/core/lib/promise/map.h" +#include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/service_config/service_config_call_data.h" #include "src/core/lib/slice/slice.h" @@ -165,32 +163,23 @@ ArenaPromise StatefulSessionFilter::MakeCallPromise( *cookie_value); } // Intercept server initial metadata. - auto* read_latch = GetContext()->New>(); - auto* write_latch = - std::exchange(call_args.server_initial_metadata, read_latch); - return TryConcurrently( - Seq(next_promise_factory(std::move(call_args)), - [cookie_config, cookie_value](ServerMetadataHandle md) { - // If we got a Trailers-Only response, then add the - // cookie to the trailing metadata instead of the - // initial metadata. - if (md->get(GrpcTrailersOnly()).value_or(false)) { - MaybeUpdateServerInitialMetadata(cookie_config, - cookie_value, md.get()); - } - return md; - })) - .NecessaryPull(Seq(read_latch->Wait(), - [write_latch, cookie_config, - cookie_value](ServerMetadata** md) -> absl::Status { - if (*md != nullptr) { - // Add cookie to server initial metadata if needed. - MaybeUpdateServerInitialMetadata( - cookie_config, cookie_value, *md); - } - write_latch->Set(*md); - return absl::OkStatus(); - })); + call_args.server_initial_metadata->InterceptAndMap( + [cookie_config, cookie_value](ServerMetadataHandle md) { + // Add cookie to server initial metadata if needed. + MaybeUpdateServerInitialMetadata(cookie_config, cookie_value, md.get()); + return md; + }); + return Map(next_promise_factory(std::move(call_args)), + [cookie_config, cookie_value](ServerMetadataHandle md) { + // If we got a Trailers-Only response, then add the + // cookie to the trailing metadata instead of the + // initial metadata. + if (md->get(GrpcTrailersOnly()).value_or(false)) { + MaybeUpdateServerInitialMetadata(cookie_config, cookie_value, + md.get()); + } + return md; + }); } absl::optional diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index ed0b6276575..2c54d917d25 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -668,6 +668,10 @@ grpc_chttp2_stream::grpc_chttp2_stream(grpc_chttp2_transport* t, flow_control(&t->flow_control) { if (server_data) { id = static_cast(reinterpret_cast(server_data)); + if (grpc_http_trace.enabled()) { + gpr_log(GPR_DEBUG, "HTTP:%p/%p creating accept stream %d [from %p]", t, + this, id, server_data); + } *t->accepting_stream = this; grpc_chttp2_stream_map_add(&t->stream_map, id, this); post_destructive_reclaimer(t); @@ -1829,6 +1833,12 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t, // Lambda is immediately invoked as a big scoped section that can be // exited out of at any point by returning. [&]() { + if (grpc_http_trace.enabled()) { + gpr_log(GPR_DEBUG, + "maybe_complete_recv_message %p final_metadata_requested=%d " + "seen_error=%d", + s, s->final_metadata_requested, s->seen_error); + } if (s->final_metadata_requested && s->seen_error) { grpc_slice_buffer_reset_and_unref(&s->frame_storage); s->recv_message->reset(); @@ -1839,6 +1849,12 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t, int64_t min_progress_size; auto r = grpc_deframe_unprocessed_incoming_frames( s, &min_progress_size, &**s->recv_message, s->recv_message_flags); + if (grpc_http_trace.enabled()) { + gpr_log(GPR_DEBUG, "Deframe data frame: %s", + grpc_core::PollToString(r, [](absl::Status r) { + return r.ToString(); + }).c_str()); + } if (absl::holds_alternative(r)) { if (s->read_closed) { grpc_slice_buffer_reset_and_unref(&s->frame_storage); @@ -1889,6 +1905,14 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t, void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { grpc_chttp2_maybe_complete_recv_message(t, s); + if (grpc_http_trace.enabled()) { + gpr_log(GPR_DEBUG, + "maybe_complete_recv_trailing_metadata cli=%d s=%p closure=%p " + "read_closed=%d " + "write_closed=%d %" PRIdPTR, + t->is_client, s, s->recv_trailing_metadata_finished, s->read_closed, + s->write_closed, s->frame_storage.length); + } if (s->recv_trailing_metadata_finished != nullptr && s->read_closed && s->write_closed) { if (s->seen_error || !t->is_client) { @@ -2048,6 +2072,14 @@ void grpc_chttp2_fail_pending_writes(grpc_chttp2_transport* t, void grpc_chttp2_mark_stream_closed(grpc_chttp2_transport* t, grpc_chttp2_stream* s, int close_reads, int close_writes, grpc_error_handle error) { + if (grpc_http_trace.enabled()) { + gpr_log( + GPR_DEBUG, "MARK_STREAM_CLOSED: t=%p s=%p(id=%d) %s [%s]", t, s, s->id, + (close_reads && close_writes) + ? "read+write" + : (close_reads ? "read" : (close_writes ? "write" : "nothing??")), + error.ToString().c_str()); + } if (s->read_closed && s->write_closed) { // already closed, but we should still fake the status if needed. grpc_error_handle overall_error = removal_error(error, s, "Stream removed"); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 0724e70cfe8..58b1e02effc 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -389,12 +389,16 @@ struct grpc_chttp2_transport uint32_t incoming_frame_size = 0; uint32_t incoming_stream_id = 0; - // active parser - void* parser_data = nullptr; grpc_chttp2_stream* incoming_stream = nullptr; - grpc_error_handle (*parser)(void* parser_user_data, grpc_chttp2_transport* t, - grpc_chttp2_stream* s, const grpc_slice& slice, - int is_last); + // active parser + struct Parser { + const char* name; + grpc_error_handle (*parser)(void* parser_user_data, + grpc_chttp2_transport* t, grpc_chttp2_stream* s, + const grpc_slice& slice, int is_last); + void* user_data = nullptr; + }; + Parser parser; grpc_chttp2_write_cb* write_cb_pool = nullptr; diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc index 2913543e171..b4d719e5623 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.cc +++ b/src/core/ext/transport/chttp2/transport/parsing.cc @@ -18,7 +18,7 @@ #include -#include +#include #include #include @@ -28,6 +28,7 @@ #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" #include #include @@ -127,6 +128,65 @@ uint32_t grpc_chttp2_min_read_progress_size(grpc_chttp2_transport* t) { GPR_UNREACHABLE_CODE(return 1); } +namespace { +struct KnownFlag { + uint8_t flag; + absl::string_view name; +}; + +std::string MakeFrameTypeString(absl::string_view frame_type, uint8_t flags, + std::initializer_list known_flags) { + std::string result(frame_type); + for (const KnownFlag& known_flag : known_flags) { + if (flags & known_flag.flag) { + absl::StrAppend(&result, ":", known_flag.name); + flags &= ~known_flag.flag; + } + } + if (flags != 0) { + absl::StrAppend(&result, ":UNKNOWN_FLAGS=0x", + absl::Hex(flags, absl::kZeroPad2)); + } + return result; +} + +std::string FrameTypeString(uint8_t frame_type, uint8_t flags) { + switch (frame_type) { + case GRPC_CHTTP2_FRAME_DATA: + return MakeFrameTypeString( + "DATA", flags, {{GRPC_CHTTP2_DATA_FLAG_END_STREAM, "END_STREAM"}}); + case GRPC_CHTTP2_FRAME_HEADER: + return MakeFrameTypeString( + "HEADERS", flags, + {{GRPC_CHTTP2_DATA_FLAG_END_STREAM, "END_STREAM"}, + {GRPC_CHTTP2_DATA_FLAG_END_HEADERS, "END_HEADERS"}, + {GRPC_CHTTP2_FLAG_HAS_PRIORITY, "PRIORITY"}}); + case GRPC_CHTTP2_FRAME_CONTINUATION: + return MakeFrameTypeString( + "HEADERS", flags, + {{GRPC_CHTTP2_DATA_FLAG_END_STREAM, "END_STREAM"}, + {GRPC_CHTTP2_DATA_FLAG_END_HEADERS, "END_HEADERS"}, + {GRPC_CHTTP2_FLAG_HAS_PRIORITY, "PRIORITY"}}); + case GRPC_CHTTP2_FRAME_RST_STREAM: + return MakeFrameTypeString("RST_STREAM", flags, {}); + case GRPC_CHTTP2_FRAME_SETTINGS: + return MakeFrameTypeString("SETTINGS", flags, + {{GRPC_CHTTP2_FLAG_ACK, "ACK"}}); + case GRPC_CHTTP2_FRAME_PING: + return MakeFrameTypeString("PING", flags, + {{GRPC_CHTTP2_FLAG_ACK, "ACK"}}); + case GRPC_CHTTP2_FRAME_GOAWAY: + return MakeFrameTypeString("GOAWAY", flags, {}); + case GRPC_CHTTP2_FRAME_WINDOW_UPDATE: + return MakeFrameTypeString("WINDOW_UPDATE", flags, {}); + default: + return MakeFrameTypeString( + absl::StrCat("UNKNOWN_FRAME_TYPE_", static_cast(frame_type)), + flags, {}); + } +} +} // namespace + grpc_error_handle grpc_chttp2_perform_read(grpc_chttp2_transport* t, const grpc_slice& slice) { const uint8_t* beg = GRPC_SLICE_START_PTR(slice); @@ -250,6 +310,12 @@ grpc_error_handle grpc_chttp2_perform_read(grpc_chttp2_transport* t, case GRPC_DTS_FH_8: GPR_DEBUG_ASSERT(cur < end); t->incoming_stream_id |= (static_cast(*cur)); + if (grpc_http_trace.enabled()) { + gpr_log(GPR_INFO, "INCOMING[%p]: %s len:%d id:0x%08x", t, + FrameTypeString(t->incoming_frame_type, t->incoming_frame_flags) + .c_str(), + t->incoming_frame_size, t->incoming_stream_id); + } t->deframe_state = GRPC_DTS_FRAME; err = init_frame_parser(t); if (!err.ok()) { @@ -404,8 +470,8 @@ static HPackParser::LogInfo hpack_parser_log_info( static grpc_error_handle init_header_skip_frame_parser( grpc_chttp2_transport* t, HPackParser::Priority priority_type) { bool is_eoh = t->expect_continuation_stream_id != 0; - t->parser = grpc_chttp2_header_parser_parse; - t->parser_data = &t->hpack_parser; + t->parser = grpc_chttp2_transport::Parser{ + "header", grpc_chttp2_header_parser_parse, &t->hpack_parser}; t->hpack_parser.BeginFrame( nullptr, t->settings[GRPC_ACKED_SETTINGS] @@ -417,15 +483,17 @@ static grpc_error_handle init_header_skip_frame_parser( static grpc_error_handle init_non_header_skip_frame_parser( grpc_chttp2_transport* t) { - t->parser = skip_parser; + t->parser = + grpc_chttp2_transport::Parser{"skip_parser", skip_parser, nullptr}; return absl::OkStatus(); } void grpc_chttp2_parsing_become_skip_parser(grpc_chttp2_transport* t) { - if (t->parser == grpc_chttp2_header_parser_parse) { + if (t->parser.parser == grpc_chttp2_header_parser_parse) { t->hpack_parser.StopBufferingFrame(); } else { - t->parser = skip_parser; + t->parser = + grpc_chttp2_transport::Parser{"skip_parser", skip_parser, nullptr}; } } @@ -472,9 +540,8 @@ static grpc_error_handle init_data_frame_parser(grpc_chttp2_transport* t) { error_handler: if (status.ok()) { t->incoming_stream = s; - // t->parser = grpc_chttp2_data_parser_parse; - t->parser = grpc_chttp2_data_parser_parse; - t->parser_data = nullptr; + t->parser = grpc_chttp2_transport::Parser{ + "data", grpc_chttp2_data_parser_parse, nullptr}; t->ping_state.last_ping_sent_time = grpc_core::Timestamp::InfPast(); return absl::OkStatus(); } else if (s != nullptr) { @@ -584,8 +651,8 @@ static grpc_error_handle init_header_frame_parser(grpc_chttp2_transport* t, t->incoming_stream = nullptr; return init_header_skip_frame_parser(t, priority_type); } - t->parser = grpc_chttp2_header_parser_parse; - t->parser_data = &t->hpack_parser; + t->parser = grpc_chttp2_transport::Parser{ + "header", grpc_chttp2_header_parser_parse, &t->hpack_parser}; if (t->header_eof) { s->eos_received = true; } @@ -640,12 +707,17 @@ static grpc_error_handle init_window_update_frame_parser( grpc_chttp2_stream* s = t->incoming_stream = grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); if (s == nullptr) { + if (grpc_http_trace.enabled()) { + gpr_log(GPR_ERROR, "Stream %d not found, ignoring WINDOW_UPDATE", + t->incoming_stream_id); + } return init_non_header_skip_frame_parser(t); } s->stats.incoming.framing_bytes += 9; } - t->parser = grpc_chttp2_window_update_parser_parse; - t->parser_data = &t->simple.window_update; + t->parser = grpc_chttp2_transport::Parser{ + "window_update", grpc_chttp2_window_update_parser_parse, + &t->simple.window_update}; return absl::OkStatus(); } @@ -653,8 +725,8 @@ static grpc_error_handle init_ping_parser(grpc_chttp2_transport* t) { grpc_error_handle err = grpc_chttp2_ping_parser_begin_frame( &t->simple.ping, t->incoming_frame_size, t->incoming_frame_flags); if (!err.ok()) return err; - t->parser = grpc_chttp2_ping_parser_parse; - t->parser_data = &t->simple.ping; + t->parser = grpc_chttp2_transport::Parser{ + "ping", grpc_chttp2_ping_parser_parse, &t->simple.ping}; return absl::OkStatus(); } @@ -668,8 +740,8 @@ static grpc_error_handle init_rst_stream_parser(grpc_chttp2_transport* t) { return init_non_header_skip_frame_parser(t); } s->stats.incoming.framing_bytes += 9; - t->parser = grpc_chttp2_rst_stream_parser_parse; - t->parser_data = &t->simple.rst_stream; + t->parser = grpc_chttp2_transport::Parser{ + "rst_stream", grpc_chttp2_rst_stream_parser_parse, &t->simple.rst_stream}; return absl::OkStatus(); } @@ -677,8 +749,8 @@ static grpc_error_handle init_goaway_parser(grpc_chttp2_transport* t) { grpc_error_handle err = grpc_chttp2_goaway_parser_begin_frame( &t->goaway_parser, t->incoming_frame_size, t->incoming_frame_flags); if (!err.ok()) return err; - t->parser = grpc_chttp2_goaway_parser_parse; - t->parser_data = &t->goaway_parser; + t->parser = grpc_chttp2_transport::Parser{ + "goaway", grpc_chttp2_goaway_parser_parse, &t->goaway_parser}; return absl::OkStatus(); } @@ -706,8 +778,8 @@ static grpc_error_handle init_settings_frame_parser(grpc_chttp2_transport* t) { t, nullptr); t->sent_local_settings = false; } - t->parser = grpc_chttp2_settings_parser_parse; - t->parser_data = &t->simple.settings; + t->parser = grpc_chttp2_transport::Parser{ + "settings", grpc_chttp2_settings_parser_parse, &t->simple.settings}; return absl::OkStatus(); } @@ -715,15 +787,24 @@ static grpc_error_handle parse_frame_slice(grpc_chttp2_transport* t, const grpc_slice& slice, int is_last) { grpc_chttp2_stream* s = t->incoming_stream; - grpc_error_handle err = t->parser(t->parser_data, t, s, slice, is_last); + if (grpc_http_trace.enabled()) { + gpr_log(GPR_DEBUG, + "INCOMING[%p;%p]: Parse %" PRIdPTR "b %sframe fragment with %s", t, + s, GRPC_SLICE_LENGTH(slice), is_last ? "last " : "", + t->parser.name); + } + grpc_error_handle err = + t->parser.parser(t->parser.user_data, t, s, slice, is_last); intptr_t unused; if (GPR_LIKELY(err.ok())) { return err; - } else if (grpc_error_get_int(err, grpc_core::StatusIntProperty::kStreamId, - &unused)) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) { - gpr_log(GPR_ERROR, "%s", grpc_core::StatusToString(err).c_str()); - } + } + if (grpc_http_trace.enabled()) { + gpr_log(GPR_ERROR, "INCOMING[%p;%p]: Parse failed with %s", t, s, + err.ToString().c_str()); + } + if (grpc_error_get_int(err, grpc_core::StatusIntProperty::kStreamId, + &unused)) { grpc_chttp2_parsing_become_skip_parser(t); if (s) { s->forced_close_error = err; diff --git a/src/core/lib/channel/channel_stack_builder_impl.cc b/src/core/lib/channel/channel_stack_builder_impl.cc index c57e12258c4..98cb7a0a4c8 100644 --- a/src/core/lib/channel/channel_stack_builder_impl.cc +++ b/src/core/lib/channel/channel_stack_builder_impl.cc @@ -37,6 +37,7 @@ #include "src/core/lib/gpr/useful.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/surface/call_trace.h" +#include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/transport.h" @@ -53,12 +54,24 @@ absl::StatusOr> ChannelStackBuilderImpl::Build() { std::vector stack; const bool is_promising = IsPromising(); + const bool is_client = + grpc_channel_stack_type_is_client(channel_stack_type()); + const bool client_promise_tracing = + is_client && is_promising && grpc_call_trace.enabled(); + const bool server_promise_tracing = + !is_client && is_promising && grpc_call_trace.enabled(); for (const auto* filter : this->stack()) { - if (is_promising && grpc_call_trace.enabled()) { + if (client_promise_tracing) { stack.push_back(PromiseTracingFilterFor(filter)); } stack.push_back(filter); + if (server_promise_tracing) { + stack.push_back(PromiseTracingFilterFor(filter)); + } + } + if (server_promise_tracing) { + stack.pop_back(); // connected_channel must be last => can't be traced } // calculate the size of the channel stack diff --git a/src/core/lib/channel/connected_channel.cc b/src/core/lib/channel/connected_channel.cc index 9d45df4ac80..87bf124e56f 100644 --- a/src/core/lib/channel/connected_channel.cc +++ b/src/core/lib/channel/connected_channel.cc @@ -24,12 +24,15 @@ #include #include +#include +#include #include #include #include #include #include "absl/base/thread_annotations.h" +#include "absl/container/inlined_vector.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -37,6 +40,7 @@ #include "absl/types/variant.h" #include +#include #include #include @@ -59,10 +63,11 @@ #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/latch.h" +#include "src/core/lib/promise/detail/basic_seq.h" #include "src/core/lib/promise/pipe.h" #include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/arena.h" +#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/call_trace.h" @@ -243,37 +248,37 @@ static void connected_channel_get_channel_info( namespace grpc_core { namespace { -class ClientStream : public Orphanable { +class ConnectedChannelStream : public Orphanable { public: - ClientStream(grpc_transport* transport, CallArgs call_args) - : transport_(transport), - stream_(nullptr, StreamDeleter(this)), - server_initial_metadata_latch_(call_args.server_initial_metadata), - client_to_server_messages_(call_args.outgoing_messages), - server_to_client_messages_(call_args.incoming_messages), - client_initial_metadata_(std::move(call_args.client_initial_metadata)) { - call_context_->IncrementRefCount("client_stream"); - GRPC_STREAM_REF_INIT( - &stream_refcount_, 1, - [](void* p, grpc_error_handle) { - static_cast(p)->BeginDestroy(); - }, - this, "client_stream"); - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sInitImpl: intitial_metadata=%s", - Activity::current()->DebugTag().c_str(), - client_initial_metadata_->DebugString().c_str()); - } + grpc_transport* transport() { return transport_; } + grpc_closure* stream_destroyed_closure() { return &stream_destroyed_; } + + void IncrementRefCount(const char* reason) { +#ifndef NDEBUG + grpc_stream_ref(&stream_refcount_, reason); +#else + (void)reason; + grpc_stream_ref(&stream_refcount_); +#endif } - void Orphan() override { + void Unref(const char* reason) { +#ifndef NDEBUG + grpc_stream_unref(&stream_refcount_, reason); +#else + (void)reason; + grpc_stream_unref(&stream_refcount_); +#endif + } + + void Orphan() final { bool finished; { - MutexLock lock(&mu_); + MutexLock lock(mu()); if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sDropStream: %s", + gpr_log(GPR_INFO, "%s[connected] DropStream: %s finished=%s", Activity::current()->DebugTag().c_str(), - ActiveOpsString().c_str()); + ActiveOpsString().c_str(), finished_ ? "true" : "false"); } finished = finished_; } @@ -284,211 +289,139 @@ class ClientStream : public Orphanable { auto* cancel_op = GetContext()->New(); cancel_op->cancel_stream = true; - cancel_op->payload = &batch_payload_; - auto* stream = stream_.get(); + cancel_op->payload = batch_payload(); + auto* s = stream(); cancel_op->on_complete = NewClosure( [this](grpc_error_handle) { Unref("shutdown client stream"); }); - batch_payload_.cancel_stream.cancel_error = absl::CancelledError(); - grpc_transport_perform_stream_op(transport_, stream, cancel_op); + batch_payload()->cancel_stream.cancel_error = absl::CancelledError(); + grpc_transport_perform_stream_op(transport(), s, cancel_op); } Unref("orphan client stream"); } - void IncrementRefCount(const char* reason) { -#ifndef NDEBUG - grpc_stream_ref(&stream_refcount_, reason); -#else - (void)reason; - grpc_stream_ref(&stream_refcount_); -#endif - } - - void Unref(const char* reason) { -#ifndef NDEBUG - grpc_stream_unref(&stream_refcount_, reason); -#else - (void)reason; - grpc_stream_unref(&stream_refcount_); -#endif + protected: + explicit ConnectedChannelStream(grpc_transport* transport) + : transport_(transport), stream_(nullptr, StreamDeleter(this)) { + call_context_->IncrementRefCount("connected_channel_stream"); + GRPC_STREAM_REF_INIT( + &stream_refcount_, 1, + [](void* p, grpc_error_handle) { + static_cast(p)->BeginDestroy(); + }, + this, "client_stream"); } - void BeginDestroy() { - if (stream_ != nullptr) { - stream_.reset(); - } else { - StreamDestroyed(); - } + grpc_stream* stream() { return stream_.get(); } + void SetStream(grpc_stream* stream) { stream_.reset(stream); } + grpc_stream_refcount* stream_refcount() { return &stream_refcount_; } + Mutex* mu() const ABSL_LOCK_RETURNED(mu_) { return &mu_; } + grpc_transport_stream_op_batch_payload* batch_payload() { + return &batch_payload_; } + bool finished() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { return finished_; } + void set_finished() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { finished_ = true; } + virtual std::string ActiveOpsString() const + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0; - Poll PollOnce() { - MutexLock lock(&mu_); - GPR_ASSERT(!finished_); - + void SchedulePush(grpc_transport_stream_op_batch* batch) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sPollConnectedChannel: %s", + gpr_log(GPR_DEBUG, "%s[connected] Push batch to transport: %s", Activity::current()->DebugTag().c_str(), - ActiveOpsString().c_str()); + grpc_transport_stream_op_batch_string(batch).c_str()); } - - auto push_recv_message = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { - recv_message_state_ = PendingReceiveMessage{}; - auto& pending_recv_message = - absl::get(recv_message_state_); - memset(&recv_message_, 0, sizeof(recv_message_)); - recv_message_.payload = &batch_payload_; - recv_message_.on_complete = nullptr; - recv_message_.recv_message = true; - batch_payload_.recv_message.recv_message = &pending_recv_message.payload; - batch_payload_.recv_message.flags = &pending_recv_message.flags; - batch_payload_.recv_message.call_failed_before_recv_message = nullptr; - batch_payload_.recv_message.recv_message_ready = - &recv_message_batch_done_; - IncrementRefCount("recv_message"); - recv_message_waker_ = Activity::current()->MakeOwningWaker(); - push_recv_message_ = true; - SchedulePush(); - }; - - if (!std::exchange(requested_metadata_, true)) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sPollConnectedChannel: requesting metadata", - Activity::current()->DebugTag().c_str()); - } - stream_.reset(static_cast( - GetContext()->Alloc(transport_->vtable->sizeof_stream))); - grpc_transport_init_stream(transport_, stream_.get(), &stream_refcount_, - nullptr, GetContext()); - grpc_transport_set_pops(transport_, stream_.get(), - GetContext()->polling_entity()); - memset(&metadata_, 0, sizeof(metadata_)); - metadata_.send_initial_metadata = true; - metadata_.recv_initial_metadata = true; - metadata_.recv_trailing_metadata = true; - metadata_.payload = &batch_payload_; - metadata_.on_complete = &metadata_batch_done_; - batch_payload_.send_initial_metadata.send_initial_metadata = - client_initial_metadata_.get(); - batch_payload_.send_initial_metadata.peer_string = - GetContext()->peer_string_atm_ptr(); - server_initial_metadata_ = - GetContext()->MakePooled(GetContext()); - batch_payload_.recv_initial_metadata.recv_initial_metadata = - server_initial_metadata_.get(); - batch_payload_.recv_initial_metadata.recv_initial_metadata_ready = - &recv_initial_metadata_ready_; - batch_payload_.recv_initial_metadata.trailing_metadata_available = - nullptr; - batch_payload_.recv_initial_metadata.peer_string = nullptr; - server_trailing_metadata_ = - GetContext()->MakePooled(GetContext()); - batch_payload_.recv_trailing_metadata.recv_trailing_metadata = - server_trailing_metadata_.get(); - batch_payload_.recv_trailing_metadata.collect_stats = - &GetContext()->call_stats()->transport_stream_stats; - batch_payload_.recv_trailing_metadata.recv_trailing_metadata_ready = - &recv_trailing_metadata_ready_; - push_metadata_ = true; - IncrementRefCount("metadata_batch_done"); - IncrementRefCount("initial_metadata_ready"); - IncrementRefCount("trailing_metadata_ready"); - initial_metadata_waker_ = Activity::current()->MakeOwningWaker(); - trailing_metadata_waker_ = Activity::current()->MakeOwningWaker(); - SchedulePush(); + if (push_batches_.empty()) { + IncrementRefCount("push"); + ExecCtx::Run(DEBUG_LOCATION, &push_, absl::OkStatus()); } + push_batches_.push_back(batch); + } + + void PollSendMessage(PipeReceiver* outgoing_messages, + ClientMetadataHandle* client_trailing_metadata) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { if (absl::holds_alternative(send_message_state_)) { message_to_send_.reset(); } if (absl::holds_alternative(send_message_state_)) { message_to_send_.reset(); - send_message_state_ = client_to_server_messages_->Next(); + send_message_state_.emplace>( + outgoing_messages->Next()); } - if (auto* next = absl::get_if::NextType>( + if (auto* next = absl::get_if>( &send_message_state_)) { auto r = (*next)(); if (auto* p = absl::get_if>(&r)) { memset(&send_message_, 0, sizeof(send_message_)); - send_message_.payload = &batch_payload_; + send_message_.payload = batch_payload(); send_message_.on_complete = &send_message_batch_done_; // No value => half close from above. if (p->has_value()) { - message_to_send_ = std::move(**p); + message_to_send_ = std::move(*p); send_message_state_ = SendMessageToTransport{}; send_message_.send_message = true; - batch_payload_.send_message.send_message = - message_to_send_->payload(); - batch_payload_.send_message.flags = message_to_send_->flags(); + batch_payload()->send_message.send_message = + (*message_to_send_)->payload(); + batch_payload()->send_message.flags = (*message_to_send_)->flags(); } else { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sPollConnectedChannel: half close", + gpr_log(GPR_INFO, "%s[connected] PollConnectedChannel: half close", Activity::current()->DebugTag().c_str()); } GPR_ASSERT(!absl::holds_alternative(send_message_state_)); - client_trailing_metadata_ = - GetContext()->MakePooled( - GetContext()); send_message_state_ = Closed{}; send_message_.send_trailing_metadata = true; - batch_payload_.send_trailing_metadata.send_trailing_metadata = - client_trailing_metadata_.get(); - batch_payload_.send_trailing_metadata.sent = nullptr; + if (client_trailing_metadata != nullptr) { + *client_trailing_metadata = + GetContext()->MakePooled( + GetContext()); + batch_payload()->send_trailing_metadata.send_trailing_metadata = + client_trailing_metadata->get(); + batch_payload()->send_trailing_metadata.sent = nullptr; + } else { + return; // Skip rest of function for server + } } IncrementRefCount("send_message"); send_message_waker_ = Activity::current()->MakeOwningWaker(); - push_send_message_ = true; - SchedulePush(); + SchedulePush(&send_message_); } } + } + + void PollRecvMessage(PipeSender*& incoming_messages) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { if (auto* pending = absl::get_if(&recv_message_state_)) { if (pending->received) { if (pending->payload.has_value()) { if (grpc_call_trace.enabled()) { gpr_log(GPR_INFO, - "%sRecvMessageBatchDone: received payload of %" PRIdPTR - " bytes", + "%s[connected] PollRecvMessage: received payload of " + "%" PRIdPTR " bytes", recv_message_waker_.ActivityDebugTag().c_str(), pending->payload->Length()); } - recv_message_state_ = server_to_client_messages_->Push( - GetContext()->MakePooled( + recv_message_state_ = + incoming_messages->Push(GetContext()->MakePooled( std::move(*pending->payload), pending->flags)); } else { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sRecvMessageBatchDone: received no payload", + gpr_log(GPR_INFO, + "%s[connected] PollRecvMessage: received no payload", recv_message_waker_.ActivityDebugTag().c_str()); } recv_message_state_ = Closed{}; - std::exchange(server_to_client_messages_, nullptr)->Close(); + std::exchange(incoming_messages, nullptr)->Close(); } } } - if (server_initial_metadata_state_ == - ServerInitialMetadataState::kReceivedButNotSet) { - server_initial_metadata_state_ = ServerInitialMetadataState::kSet; - server_initial_metadata_latch_->Set(server_initial_metadata_.get()); - } if (absl::holds_alternative(recv_message_state_)) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sPollConnectedChannel: requesting message", + gpr_log(GPR_INFO, "%s[connected] PollRecvMessage: requesting message", Activity::current()->DebugTag().c_str()); } - push_recv_message(); - } - if (server_initial_metadata_state_ == ServerInitialMetadataState::kSet && - !absl::holds_alternative::PushType>( - recv_message_state_) && - !absl::holds_alternative(recv_message_state_) && - std::exchange(queued_trailing_metadata_, false)) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%sPollConnectedChannel: finished request, returning: {%s}; " - "active_ops: %s", - Activity::current()->DebugTag().c_str(), - server_trailing_metadata_->DebugString().c_str(), - ActiveOpsString().c_str()); - } - finished_ = true; - return ServerMetadataHandle(std::move(server_trailing_metadata_)); + PushRecvMessage(); } if (auto* push = absl::get_if::PushType>( &recv_message_state_)) { @@ -498,57 +431,120 @@ class ClientStream : public Orphanable { if (!finished_) { if (grpc_call_trace.enabled()) { gpr_log(GPR_INFO, - "%sPollConnectedChannel: pushed message; requesting next", + "%s[connected] PollRecvMessage: pushed message; " + "requesting next", Activity::current()->DebugTag().c_str()); } - push_recv_message(); + PushRecvMessage(); } else { if (grpc_call_trace.enabled()) { gpr_log(GPR_INFO, - "%sPollConnectedChannel: pushed message and finished; " + "%s[connected] PollRecvMessage: pushed message " + "and finished; " "marking closed", Activity::current()->DebugTag().c_str()); } recv_message_state_ = Closed{}; + std::exchange(incoming_messages, nullptr)->Close(); } } else { if (grpc_call_trace.enabled()) { gpr_log(GPR_INFO, - "%sPollConnectedChannel: failed to push message; marking " + "%s[connected] PollRecvMessage: failed to push " + "message; marking " "closed", Activity::current()->DebugTag().c_str()); } recv_message_state_ = Closed{}; + std::exchange(incoming_messages, nullptr)->Close(); } } } - return Pending{}; } - void RecvInitialMetadataReady(grpc_error_handle error) { - GPR_ASSERT(error == absl::OkStatus()); - { - MutexLock lock(&mu_); - server_initial_metadata_state_ = - ServerInitialMetadataState::kReceivedButNotSet; - initial_metadata_waker_.Wakeup(); + std::string SendMessageString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) { + return Match( + send_message_state_, [](Idle) -> std::string { return "IDLE"; }, + [](Closed) -> std::string { return "CLOSED"; }, + [](const PipeReceiverNextType&) -> std::string { + return "WAITING"; + }, + [](SendMessageToTransport) -> std::string { return "SENDING"; }); + } + + std::string RecvMessageString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) { + return Match( + recv_message_state_, [](Idle) -> std::string { return "IDLE"; }, + [](Closed) -> std::string { return "CLOSED"; }, + [](const PendingReceiveMessage&) -> std::string { return "WAITING"; }, + [](const absl::optional& message) -> std::string { + return absl::StrCat( + "READY:", message.has_value() + ? absl::StrCat((*message)->payload()->Length(), "b") + : "EOS"); + }, + [](const PipeSender::PushType&) -> std::string { + return "PUSHING"; + }); + } + + bool IsPromiseReceiving() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) { + return absl::holds_alternative::PushType>( + recv_message_state_) || + absl::holds_alternative(recv_message_state_); + } + + private: + struct SendMessageToTransport {}; + struct Idle {}; + struct Closed {}; + + class StreamDeleter { + public: + explicit StreamDeleter(ConnectedChannelStream* impl) : impl_(impl) {} + void operator()(grpc_stream* stream) const { + if (stream == nullptr) return; + grpc_transport_destroy_stream(impl_->transport(), stream, + impl_->stream_destroyed_closure()); } - Unref("initial_metadata_ready"); + + private: + ConnectedChannelStream* impl_; + }; + using StreamPtr = std::unique_ptr; + + void StreamDestroyed() { + call_context_->RunInContext([this] { + auto* cc = call_context_; + this->~ConnectedChannelStream(); + cc->Unref("child_stream"); + }); } - void RecvTrailingMetadataReady(grpc_error_handle error) { - GPR_ASSERT(error == absl::OkStatus()); - { - MutexLock lock(&mu_); - queued_trailing_metadata_ = true; - trailing_metadata_waker_.Wakeup(); + void BeginDestroy() { + if (stream_ != nullptr) { + stream_.reset(); + } else { + StreamDestroyed(); } - Unref("trailing_metadata_ready"); } - void MetadataBatchDone(grpc_error_handle error) { - GPR_ASSERT(error == absl::OkStatus()); - Unref("metadata_batch_done"); + // Called from outside the activity to push work down to the transport. + void Push() { + PushBatches push_batches; + { + MutexLock lock(&mu_); + push_batches.swap(push_batches_); + } + for (auto* batch : push_batches) { + if (stream() != nullptr) { + grpc_transport_perform_stream_op(transport(), stream(), batch); + } else { + grpc_transport_stream_op_batch_finish_with_failure_from_transport( + batch, absl::CancelledError()); + } + } + Unref("push"); } void SendMessageBatchDone(grpc_error_handle error) { @@ -571,21 +567,24 @@ class ClientStream : public Orphanable { void RecvMessageBatchDone(grpc_error_handle error) { { - MutexLock lock(&mu_); + MutexLock lock(mu()); if (error != absl::OkStatus()) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sRecvMessageBatchDone: error=%s", + gpr_log(GPR_INFO, "%s[connected] RecvMessageBatchDone: error=%s", recv_message_waker_.ActivityDebugTag().c_str(), StatusToString(error).c_str()); } } else if (absl::holds_alternative(recv_message_state_)) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sRecvMessageBatchDone: already closed, ignoring", + gpr_log(GPR_INFO, + "%s[connected] RecvMessageBatchDone: already closed, " + "ignoring", recv_message_waker_.ActivityDebugTag().c_str()); } } else { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sRecvMessageBatchDone: received message", + gpr_log(GPR_INFO, + "%s[connected] RecvMessageBatchDone: received message", recv_message_waker_.ActivityDebugTag().c_str()); } auto pending = @@ -599,95 +598,248 @@ class ClientStream : public Orphanable { Unref("recv_message"); } - // Called from outside the activity to push work down to the transport. - void Push() { - auto do_push = [this](grpc_transport_stream_op_batch* batch) { - if (stream_ != nullptr) { - grpc_transport_perform_stream_op(transport_, stream_.get(), batch); - } else { - grpc_transport_stream_op_batch_finish_with_failure_from_transport( - batch, absl::CancelledError()); + void PushRecvMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + recv_message_state_ = PendingReceiveMessage{}; + auto& pending_recv_message = + absl::get(recv_message_state_); + memset(&recv_message_, 0, sizeof(recv_message_)); + recv_message_.payload = batch_payload(); + recv_message_.on_complete = nullptr; + recv_message_.recv_message = true; + batch_payload()->recv_message.recv_message = &pending_recv_message.payload; + batch_payload()->recv_message.flags = &pending_recv_message.flags; + batch_payload()->recv_message.call_failed_before_recv_message = nullptr; + batch_payload()->recv_message.recv_message_ready = + &recv_message_batch_done_; + IncrementRefCount("recv_message"); + recv_message_waker_ = Activity::current()->MakeOwningWaker(); + SchedulePush(&recv_message_); + } + + mutable Mutex mu_; + grpc_transport* const transport_; + CallContext* const call_context_{GetContext()}; + grpc_closure stream_destroyed_ = + MakeMemberClosure( + this, DEBUG_LOCATION); + grpc_stream_refcount stream_refcount_; + StreamPtr stream_; + using PushBatches = absl::InlinedVector; + PushBatches push_batches_ ABSL_GUARDED_BY(mu_); + grpc_closure push_ = + MakeMemberClosure( + this, DEBUG_LOCATION); + + NextResult message_to_send_ ABSL_GUARDED_BY(mu_); + absl::variant, + SendMessageToTransport> + send_message_state_ ABSL_GUARDED_BY(mu_); + grpc_transport_stream_op_batch send_message_; + grpc_closure send_message_batch_done_ = + MakeMemberClosure( + this, DEBUG_LOCATION); + + struct PendingReceiveMessage { + absl::optional payload; + uint32_t flags; + bool received = false; + }; + absl::variant::PushType> + recv_message_state_ ABSL_GUARDED_BY(mu_); + grpc_closure recv_message_batch_done_ = + MakeMemberClosure( + this, DEBUG_LOCATION); + grpc_transport_stream_op_batch recv_message_; + + Waker send_message_waker_ ABSL_GUARDED_BY(mu_); + Waker recv_message_waker_ ABSL_GUARDED_BY(mu_); + bool finished_ ABSL_GUARDED_BY(mu_) = false; + + grpc_transport_stream_op_batch_payload batch_payload_{ + GetContext()}; +}; + +class ClientStream : public ConnectedChannelStream { + public: + ClientStream(grpc_transport* transport, CallArgs call_args) + : ConnectedChannelStream(transport), + server_initial_metadata_pipe_(call_args.server_initial_metadata), + client_to_server_messages_(call_args.client_to_server_messages), + server_to_client_messages_(call_args.server_to_client_messages), + client_initial_metadata_(std::move(call_args.client_initial_metadata)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[connected] InitImpl: intitial_metadata=%s", + Activity::current()->DebugTag().c_str(), + client_initial_metadata_->DebugString().c_str()); + } + } + + Poll PollOnce() { + MutexLock lock(mu()); + GPR_ASSERT(!finished()); + + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[connected] PollConnectedChannel: %s", + Activity::current()->DebugTag().c_str(), + ActiveOpsString().c_str()); + } + + if (!std::exchange(requested_metadata_, true)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%s[connected] PollConnectedChannel: requesting metadata", + Activity::current()->DebugTag().c_str()); } - }; - bool push_metadata; - bool push_send_message; - bool push_recv_message; + SetStream(static_cast( + GetContext()->Alloc(transport()->vtable->sizeof_stream))); + grpc_transport_init_stream(transport(), stream(), stream_refcount(), + nullptr, GetContext()); + grpc_transport_set_pops(transport(), stream(), + GetContext()->polling_entity()); + memset(&metadata_, 0, sizeof(metadata_)); + metadata_.send_initial_metadata = true; + metadata_.recv_initial_metadata = true; + metadata_.recv_trailing_metadata = true; + metadata_.payload = batch_payload(); + metadata_.on_complete = &metadata_batch_done_; + batch_payload()->send_initial_metadata.send_initial_metadata = + client_initial_metadata_.get(); + batch_payload()->send_initial_metadata.peer_string = + GetContext()->peer_string_atm_ptr(); + server_initial_metadata_ = + GetContext()->MakePooled(GetContext()); + batch_payload()->recv_initial_metadata.recv_initial_metadata = + server_initial_metadata_.get(); + batch_payload()->recv_initial_metadata.recv_initial_metadata_ready = + &recv_initial_metadata_ready_; + batch_payload()->recv_initial_metadata.trailing_metadata_available = + nullptr; + batch_payload()->recv_initial_metadata.peer_string = nullptr; + server_trailing_metadata_ = + GetContext()->MakePooled(GetContext()); + batch_payload()->recv_trailing_metadata.recv_trailing_metadata = + server_trailing_metadata_.get(); + batch_payload()->recv_trailing_metadata.collect_stats = + &GetContext()->call_stats()->transport_stream_stats; + batch_payload()->recv_trailing_metadata.recv_trailing_metadata_ready = + &recv_trailing_metadata_ready_; + IncrementRefCount("metadata_batch_done"); + IncrementRefCount("initial_metadata_ready"); + IncrementRefCount("trailing_metadata_ready"); + initial_metadata_waker_ = Activity::current()->MakeOwningWaker(); + trailing_metadata_waker_ = Activity::current()->MakeOwningWaker(); + SchedulePush(&metadata_); + } + if (server_initial_metadata_state_ == + ServerInitialMetadataState::kReceivedButNotPushed) { + server_initial_metadata_state_ = ServerInitialMetadataState::kPushing; + server_initial_metadata_push_promise_ = + server_initial_metadata_pipe_->Push( + std::move(server_initial_metadata_)); + } + if (server_initial_metadata_state_ == + ServerInitialMetadataState::kPushing) { + auto r = (*server_initial_metadata_push_promise_)(); + if (absl::holds_alternative(r)) { + server_initial_metadata_state_ = ServerInitialMetadataState::kPushed; + server_initial_metadata_push_promise_.reset(); + } + } + PollSendMessage(client_to_server_messages_, &client_trailing_metadata_); + PollRecvMessage(server_to_client_messages_); + if (server_initial_metadata_state_ == ServerInitialMetadataState::kPushed && + !IsPromiseReceiving() && + std::exchange(queued_trailing_metadata_, false)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%s[connected] PollConnectedChannel: finished request, " + "returning: {%s}; " + "active_ops: %s", + Activity::current()->DebugTag().c_str(), + server_trailing_metadata_->DebugString().c_str(), + ActiveOpsString().c_str()); + } + set_finished(); + return ServerMetadataHandle(std::move(server_trailing_metadata_)); + } + return Pending{}; + } + + void RecvInitialMetadataReady(grpc_error_handle error) { + GPR_ASSERT(error == absl::OkStatus()); { - MutexLock lock(&mu_); - push_metadata = std::exchange(push_metadata_, false); - push_send_message = std::exchange(push_send_message_, false); - push_recv_message = std::exchange(push_recv_message_, false); - scheduled_push_ = false; - } - if (push_metadata) do_push(&metadata_); - if (push_send_message) do_push(&send_message_); - if (push_recv_message) do_push(&recv_message_); - Unref("push"); + MutexLock lock(mu()); + server_initial_metadata_state_ = + ServerInitialMetadataState::kReceivedButNotPushed; + initial_metadata_waker_.Wakeup(); + } + Unref("initial_metadata_ready"); } - void StreamDestroyed() { - call_context_->RunInContext([this] { - auto* cc = call_context_; - this->~ClientStream(); - cc->Unref("child_stream"); - }); + void RecvTrailingMetadataReady(grpc_error_handle error) { + GPR_ASSERT(error == absl::OkStatus()); + { + MutexLock lock(mu()); + queued_trailing_metadata_ = true; + if (grpc_call_trace.enabled()) { + gpr_log(GPR_DEBUG, + "%s[connected] RecvTrailingMetadataReady: " + "queued_trailing_metadata_ " + "set to true; active_ops: %s", + trailing_metadata_waker_.ActivityDebugTag().c_str(), + ActiveOpsString().c_str()); + } + trailing_metadata_waker_.Wakeup(); + } + Unref("trailing_metadata_ready"); } - private: - struct Idle {}; - struct Closed {}; - struct SendMessageToTransport {}; + void MetadataBatchDone(grpc_error_handle error) { + GPR_ASSERT(error == absl::OkStatus()); + Unref("metadata_batch_done"); + } + private: enum class ServerInitialMetadataState : uint8_t { // Initial metadata has not been received from the server. kNotReceived, // Initial metadata has been received from the server via the transport, but - // has not yet been set on the latch to publish it up the call stack. - kReceivedButNotSet, + // has not yet been pushed onto the pipe to publish it up the call stack. + kReceivedButNotPushed, // Initial metadata has been received from the server via the transport and - // has been set on the latch to publish it up the call stack. - kSet, - }; - - class StreamDeleter { - public: - explicit StreamDeleter(ClientStream* impl) : impl_(impl) {} - void operator()(grpc_stream* stream) const { - if (stream == nullptr) return; - grpc_transport_destroy_stream(impl_->transport_, stream, - &impl_->stream_destroyed_); - } - - private: - ClientStream* impl_; + // has been pushed on the pipe to publish it up the call stack. + // It's still in the pipe and has not been removed by the call at the top + // yet. + kPushing, + // Initial metadata has been received from the server via the transport and + // has been pushed on the pipe to publish it up the call stack AND removed + // by the call at the top. + kPushed, }; - using StreamPtr = std::unique_ptr; - void SchedulePush() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { - if (std::exchange(scheduled_push_, true)) return; - IncrementRefCount("push"); - ExecCtx::Run(DEBUG_LOCATION, &push_, absl::OkStatus()); - } - - std::string ActiveOpsString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + std::string ActiveOpsString() const override + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) { std::vector ops; - if (finished_) ops.push_back("FINISHED"); - // Pushes - std::vector pushes; - if (push_metadata_) pushes.push_back("metadata"); - if (push_send_message_) pushes.push_back("send_message"); - if (push_recv_message_) pushes.push_back("recv_message"); - if (!pushes.empty()) { - ops.push_back( - absl::StrCat(scheduled_push_ ? "push:" : "unscheduled-push:", - absl::StrJoin(pushes, ","))); - } else if (scheduled_push_) { - ops.push_back("push:nothing"); + if (finished()) ops.push_back("FINISHED"); + // Outstanding Operations on Transport + std::vector waiting; + if (initial_metadata_waker_ != Waker()) { + waiting.push_back("initial_metadata"); + } + if (trailing_metadata_waker_ != Waker()) { + waiting.push_back("trailing_metadata"); + } + if (!waiting.empty()) { + ops.push_back(absl::StrCat("waiting:", absl::StrJoin(waiting, ","))); } // Results from transport std::vector queued; if (server_initial_metadata_state_ == - ServerInitialMetadataState::kReceivedButNotSet) { + ServerInitialMetadataState::kReceivedButNotPushed) { queued.push_back("initial_metadata"); } if (queued_trailing_metadata_) queued.push_back("trailing_metadata"); @@ -707,94 +859,31 @@ class ClientStream : public Orphanable { return absl::StrJoin(ops, " "); } - std::string SendMessageString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { - return Match( - send_message_state_, [](Idle) -> std::string { return "IDLE"; }, - [](Closed) -> std::string { return "CLOSED"; }, - [](const PipeReceiver::NextType&) -> std::string { - return "WAITING"; - }, - [](SendMessageToTransport) -> std::string { return "SENDING"; }); - } - - std::string RecvMessageString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { - return Match( - recv_message_state_, [](Idle) -> std::string { return "IDLE"; }, - [](Closed) -> std::string { return "CLOSED"; }, - [](const PendingReceiveMessage&) -> std::string { return "WAITING"; }, - [](const absl::optional& message) -> std::string { - return absl::StrCat( - "READY:", message.has_value() - ? absl::StrCat((*message)->payload()->Length(), "b") - : "EOS"); - }, - [](const PipeSender::PushType&) -> std::string { - return "PUSHING"; - }); - } - - Mutex mu_; bool requested_metadata_ = false; - bool push_metadata_ ABSL_GUARDED_BY(mu_) = false; - bool push_send_message_ ABSL_GUARDED_BY(mu_) = false; - bool push_recv_message_ ABSL_GUARDED_BY(mu_) = false; - bool scheduled_push_ ABSL_GUARDED_BY(mu_) = false; ServerInitialMetadataState server_initial_metadata_state_ - ABSL_GUARDED_BY(mu_) = ServerInitialMetadataState::kNotReceived; - bool queued_trailing_metadata_ ABSL_GUARDED_BY(mu_) = false; - bool finished_ ABSL_GUARDED_BY(mu_) = false; - CallContext* const call_context_{GetContext()}; - Waker initial_metadata_waker_ ABSL_GUARDED_BY(mu_); - Waker trailing_metadata_waker_ ABSL_GUARDED_BY(mu_); - Waker send_message_waker_ ABSL_GUARDED_BY(mu_); - Waker recv_message_waker_ ABSL_GUARDED_BY(mu_); - grpc_transport* const transport_; - grpc_stream_refcount stream_refcount_; - StreamPtr stream_; - Latch* server_initial_metadata_latch_; + ABSL_GUARDED_BY(mu()) = ServerInitialMetadataState::kNotReceived; + bool queued_trailing_metadata_ ABSL_GUARDED_BY(mu()) = false; + Waker initial_metadata_waker_ ABSL_GUARDED_BY(mu()); + Waker trailing_metadata_waker_ ABSL_GUARDED_BY(mu()); + PipeSender* server_initial_metadata_pipe_; PipeReceiver* client_to_server_messages_; PipeSender* server_to_client_messages_; - MessageHandle message_to_send_ ABSL_GUARDED_BY(mu_); - absl::variant::NextType, - SendMessageToTransport> - send_message_state_ ABSL_GUARDED_BY(mu_); - struct PendingReceiveMessage { - absl::optional payload; - uint32_t flags; - bool received = false; - }; - absl::variant::PushType> - recv_message_state_ ABSL_GUARDED_BY(mu_); grpc_closure recv_initial_metadata_ready_ = MakeMemberClosure( this, DEBUG_LOCATION); grpc_closure recv_trailing_metadata_ready_ = MakeMemberClosure( this, DEBUG_LOCATION); - grpc_closure push_ = MakeMemberClosure( - this, DEBUG_LOCATION); ClientMetadataHandle client_initial_metadata_; ClientMetadataHandle client_trailing_metadata_; ServerMetadataHandle server_initial_metadata_; ServerMetadataHandle server_trailing_metadata_; + absl::optional::PushType> + server_initial_metadata_push_promise_; grpc_transport_stream_op_batch metadata_; grpc_closure metadata_batch_done_ = MakeMemberClosure( this, DEBUG_LOCATION); - grpc_transport_stream_op_batch send_message_; - grpc_closure send_message_batch_done_ = - MakeMemberClosure( - this, DEBUG_LOCATION); - grpc_closure recv_message_batch_done_ = - MakeMemberClosure( - this, DEBUG_LOCATION); - grpc_transport_stream_op_batch recv_message_; - grpc_transport_stream_op_batch_payload batch_payload_{ - GetContext()}; - grpc_closure stream_destroyed_ = - MakeMemberClosure( - this, DEBUG_LOCATION); }; class ClientConnectedCallPromise { @@ -815,7 +904,8 @@ class ClientConnectedCallPromise { } static ArenaPromise Make(grpc_transport* transport, - CallArgs call_args) { + CallArgs call_args, + NextPromiseFactory) { return ClientConnectedCallPromise(transport, std::move(call_args)); } @@ -825,8 +915,430 @@ class ClientConnectedCallPromise { OrphanablePtr impl_; }; +class ServerStream final : public ConnectedChannelStream { + public: + ServerStream(grpc_transport* transport, + NextPromiseFactory next_promise_factory) + : ConnectedChannelStream(transport) { + SetStream(static_cast( + GetContext()->Alloc(transport->vtable->sizeof_stream))); + grpc_transport_init_stream( + transport, stream(), stream_refcount(), + GetContext()->server_call_context()->server_stream_data(), + GetContext()); + grpc_transport_set_pops(transport, stream(), + GetContext()->polling_entity()); + + // Fetch initial metadata + auto& gim = call_state_.emplace(this); + gim.recv_initial_metadata_ready_waker = + Activity::current()->MakeOwningWaker(); + memset(&gim.recv_initial_metadata, 0, sizeof(gim.recv_initial_metadata)); + gim.recv_initial_metadata.payload = batch_payload(); + gim.recv_initial_metadata.on_complete = nullptr; + gim.recv_initial_metadata.recv_initial_metadata = true; + gim.next_promise_factory = std::move(next_promise_factory); + batch_payload()->recv_initial_metadata.recv_initial_metadata = + gim.client_initial_metadata.get(); + batch_payload()->recv_initial_metadata.recv_initial_metadata_ready = + &gim.recv_initial_metadata_ready; + SchedulePush(&gim.recv_initial_metadata); + + // Fetch trailing metadata (to catch cancellations) + auto& gtm = + client_trailing_metadata_state_.emplace(); + gtm.recv_trailing_metadata_ready = + MakeMemberClosure(this); + memset(>m.recv_trailing_metadata, 0, sizeof(gtm.recv_trailing_metadata)); + gtm.recv_trailing_metadata.payload = batch_payload(); + gtm.recv_trailing_metadata.recv_trailing_metadata = true; + batch_payload()->recv_trailing_metadata.recv_trailing_metadata = + gtm.result.get(); + batch_payload()->recv_trailing_metadata.collect_stats = + &GetContext()->call_stats()->transport_stream_stats; + batch_payload()->recv_trailing_metadata.recv_trailing_metadata_ready = + >m.recv_trailing_metadata_ready; + SchedulePush(>m.recv_trailing_metadata); + gtm.waker = Activity::current()->MakeOwningWaker(); + } + + Poll PollOnce() { + MutexLock lock(mu()); + + auto poll_send_initial_metadata = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED( + mu()) { + if (auto* promise = + absl::get_if>( + &server_initial_metadata_)) { + auto r = (*promise)(); + if (auto* md = absl::get_if>(&r)) { + if (grpc_call_trace.enabled()) { + gpr_log( + GPR_INFO, "%s[connected] got initial metadata %s", + Activity::current()->DebugTag().c_str(), + (md->has_value() ? (**md)->DebugString() : "") + .c_str()); + } + memset(&send_initial_metadata_, 0, sizeof(send_initial_metadata_)); + send_initial_metadata_.send_initial_metadata = true; + send_initial_metadata_.payload = batch_payload(); + send_initial_metadata_.on_complete = &send_initial_metadata_done_; + batch_payload()->send_initial_metadata.send_initial_metadata = + server_initial_metadata_ + .emplace(std::move(**md)) + .get(); + batch_payload()->send_initial_metadata.peer_string = nullptr; + SchedulePush(&send_initial_metadata_); + return true; + } else { + return false; + } + } else { + return true; + } + }; + + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[connected] PollConnectedChannel: %s", + Activity::current()->DebugTag().c_str(), + ActiveOpsString().c_str()); + } + + poll_send_initial_metadata(); + + if (auto* p = absl::get_if( + &client_trailing_metadata_state_)) { + pipes_.client_to_server.sender.Close(); + if (!p->result.ok()) { + // client cancelled, we should cancel too + if (absl::holds_alternative(call_state_) || + absl::holds_alternative(call_state_) || + absl::holds_alternative(call_state_)) { + if (!absl::holds_alternative( + server_initial_metadata_)) { + // pretend we've sent initial metadata to stop that op from + // progressing if it's stuck somewhere above us in the stack + server_initial_metadata_.emplace(); + } + // cancel the call - this status will be returned to the server bottom + // promise + call_state_.emplace( + Complete{ServerMetadataFromStatus(p->result)}); + } + } + } + + if (auto* p = absl::get_if(&call_state_)) { + incoming_messages_ = &pipes_.client_to_server.sender; + auto promise = p->next_promise_factory(CallArgs{ + std::move(p->client_initial_metadata), + &pipes_.server_initial_metadata.sender, + &pipes_.client_to_server.receiver, &pipes_.server_to_client.sender}); + call_state_.emplace( + MessageLoop{&pipes_.server_to_client.receiver, std::move(promise)}); + server_initial_metadata_ + .emplace>( + pipes_.server_initial_metadata.receiver.Next()); + } + if (incoming_messages_ != nullptr) { + PollRecvMessage(incoming_messages_); + } + if (auto* p = absl::get_if(&call_state_)) { + if (absl::holds_alternative( + server_initial_metadata_)) { + PollSendMessage(p->outgoing_messages, nullptr); + } + auto poll = p->promise(); + if (auto* r = absl::get_if(&poll)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[connected] got trailing metadata %s; %s", + Activity::current()->DebugTag().c_str(), + (*r)->DebugString().c_str(), ActiveOpsString().c_str()); + } + auto& completing = call_state_.emplace(); + completing.server_trailing_metadata = std::move(*r); + completing.on_complete = + MakeMemberClosure(this); + completing.waker = Activity::current()->MakeOwningWaker(); + auto& op = completing.send_trailing_metadata; + memset(&op, 0, sizeof(op)); + op.payload = batch_payload(); + op.on_complete = &completing.on_complete; + // If we've gotten initial server metadata, we can send trailing + // metadata. + // Otherwise we need to cancel the call. + // There could be an unlucky ordering, so we poll here to make sure. + if (poll_send_initial_metadata()) { + op.send_trailing_metadata = true; + batch_payload()->send_trailing_metadata.send_trailing_metadata = + completing.server_trailing_metadata.get(); + batch_payload()->send_trailing_metadata.sent = &completing.sent; + } else { + op.cancel_stream = true; + const auto status_code = + completing.server_trailing_metadata->get(GrpcStatusMetadata()) + .value_or(GRPC_STATUS_UNKNOWN); + batch_payload()->cancel_stream.cancel_error = grpc_error_set_int( + absl::Status(static_cast(status_code), + completing.server_trailing_metadata + ->GetOrCreatePointer(GrpcMessageMetadata()) + ->as_string_view()), + StatusIntProperty::kRpcStatus, status_code); + } + SchedulePush(&op); + } + } + if (auto* p = absl::get_if(&call_state_)) { + set_finished(); + return std::move(p->result); + } + return Pending{}; + } + + private: + // Call state: we've asked the transport for initial metadata and are + // waiting for it before proceeding. + struct GettingInitialMetadata { + explicit GettingInitialMetadata(ServerStream* stream) + : recv_initial_metadata_ready( + MakeMemberClosure( + stream)) {} + // The batch we're using to get initial metadata. + grpc_transport_stream_op_batch recv_initial_metadata; + // Waker to re-enter the activity once the transport returns. + Waker recv_initial_metadata_ready_waker; + // Initial metadata storage for the transport. + ClientMetadataHandle client_initial_metadata = + GetContext()->MakePooled(GetContext()); + // Closure for the transport to call when it's ready. + grpc_closure recv_initial_metadata_ready; + // Next promise factory to use once we have initial metadata. + NextPromiseFactory next_promise_factory; + }; + + // Call state: transport has returned initial metadata, we're waiting to + // re-enter the activity to process it. + struct GotInitialMetadata { + ClientMetadataHandle client_initial_metadata; + NextPromiseFactory next_promise_factory; + }; + + // Call state: we're sending/receiving messages and processing the filter + // stack. + struct MessageLoop { + PipeReceiver* outgoing_messages; + ArenaPromise promise; + }; + + // Call state: promise stack has returned trailing metadata, we're sending it + // to the transport to communicate. + struct Completing { + ServerMetadataHandle server_trailing_metadata; + grpc_transport_stream_op_batch send_trailing_metadata; + grpc_closure on_complete; + bool sent = false; + Waker waker; + }; + + // Call state: server metadata has been communicated to the transport and sent + // to the client. + // The metadata will be returned down to the server call to tick the + // cancellation bit or not on the originating batch. + struct Complete { + ServerMetadataHandle result; + }; + + // Trailing metadata state: we've asked the transport for trailing metadata + // and are waiting for it before proceeding. + struct WaitingForTrailingMetadata { + ClientMetadataHandle result = + GetContext()->MakePooled(GetContext()); + grpc_transport_stream_op_batch recv_trailing_metadata; + grpc_closure recv_trailing_metadata_ready; + Waker waker; + }; + + // We've received trailing metadata from the transport - which indicates reads + // are closed. + // We convert to an absl::Status here and use that to drive a decision to + // cancel the call (on error) or not. + struct GotClientHalfClose { + absl::Status result; + }; + + void RecvInitialMetadataReady(absl::Status status) { + MutexLock lock(mu()); + auto& getting = absl::get(call_state_); + auto waker = std::move(getting.recv_initial_metadata_ready_waker); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_DEBUG, "%sGOT INITIAL METADATA: err=%s %s", + waker.ActivityDebugTag().c_str(), status.ToString().c_str(), + getting.client_initial_metadata->DebugString().c_str()); + } + GotInitialMetadata got{std::move(getting.client_initial_metadata), + std::move(getting.next_promise_factory)}; + call_state_.emplace(std::move(got)); + waker.Wakeup(); + } + + void SendTrailingMetadataDone(absl::Status result) { + MutexLock lock(mu()); + auto& completing = absl::get(call_state_); + auto md = std::move(completing.server_trailing_metadata); + auto waker = std::move(completing.waker); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_DEBUG, "%sSEND TRAILING METADATA DONE: err=%s sent=%s %s", + waker.ActivityDebugTag().c_str(), result.ToString().c_str(), + completing.sent ? "true" : "false", md->DebugString().c_str()); + } + md->Set(GrpcStatusFromWire(), completing.sent); + if (!result.ok()) { + md->Clear(); + md->Set(GrpcStatusMetadata(), + static_cast(result.code())); + md->Set(GrpcMessageMetadata(), Slice::FromCopiedString(result.message())); + md->Set(GrpcStatusFromWire(), false); + } + call_state_.emplace(Complete{std::move(md)}); + waker.Wakeup(); + } + + std::string ActiveOpsString() const override + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) { + std::vector ops; + ops.push_back(absl::StrCat( + "call_state:", + Match( + call_state_, + [](const absl::monostate&) { return "absl::monostate"; }, + [](const GettingInitialMetadata&) { return "GETTING"; }, + [](const GotInitialMetadata&) { return "GOT"; }, + [](const MessageLoop&) { return "RUNNING"; }, + [](const Completing&) { return "COMPLETING"; }, + [](const Complete&) { return "COMPLETE"; }))); + ops.push_back( + absl::StrCat("client_trailing_metadata_state:", + Match( + client_trailing_metadata_state_, + [](const absl::monostate&) -> std::string { + return "absl::monostate"; + }, + [](const WaitingForTrailingMetadata&) -> std::string { + return "WAITING"; + }, + [](const GotClientHalfClose& got) -> std::string { + return absl::StrCat("GOT:", got.result.ToString()); + }))); + // Send initial metadata + ops.push_back(absl::StrCat( + "server_initial_metadata_state:", + Match( + server_initial_metadata_, + [](const absl::monostate&) { return "absl::monostate"; }, + [](const PipeReceiverNextType&) { + return "WAITING"; + }, + [](const ServerMetadataHandle&) { return "GOT"; }))); + // Send message + std::string send_message_state = SendMessageString(); + if (send_message_state != "WAITING") { + ops.push_back(absl::StrCat("send_message:", send_message_state)); + } + // Receive message + std::string recv_message_state = RecvMessageString(); + if (recv_message_state != "IDLE") { + ops.push_back(absl::StrCat("recv_message:", recv_message_state)); + } + return absl::StrJoin(ops, " "); + } + + void SendInitialMetadataDone() {} + + void RecvTrailingMetadataReady(absl::Status error) { + MutexLock lock(mu()); + auto& state = + absl::get(client_trailing_metadata_state_); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%sRecvTrailingMetadataReady: error:%s metadata:%s state:%s", + state.waker.ActivityDebugTag().c_str(), error.ToString().c_str(), + state.result->DebugString().c_str(), ActiveOpsString().c_str()); + } + auto waker = std::move(state.waker); + ServerMetadataHandle result = std::move(state.result); + if (error.ok()) { + auto* message = result->get_pointer(GrpcMessageMetadata()); + error = absl::Status( + static_cast( + result->get(GrpcStatusMetadata()).value_or(GRPC_STATUS_UNKNOWN)), + message == nullptr ? "" : message->as_string_view()); + } + client_trailing_metadata_state_.emplace( + GotClientHalfClose{error}); + waker.Wakeup(); + } + + struct Pipes { + Pipe server_to_client; + Pipe client_to_server; + Pipe server_initial_metadata; + }; + + using CallState = + absl::variant; + CallState call_state_ ABSL_GUARDED_BY(mu()) = absl::monostate{}; + using ClientTrailingMetadataState = + absl::variant; + ClientTrailingMetadataState client_trailing_metadata_state_ + ABSL_GUARDED_BY(mu()) = absl::monostate{}; + absl::variant, + ServerMetadataHandle> + ABSL_GUARDED_BY(mu()) server_initial_metadata_ = absl::monostate{}; + PipeSender* incoming_messages_ = nullptr; + grpc_transport_stream_op_batch send_initial_metadata_; + grpc_closure send_initial_metadata_done_ = + MakeMemberClosure( + this); + Pipes pipes_ ABSL_GUARDED_BY(mu()); +}; + +class ServerConnectedCallPromise { + public: + ServerConnectedCallPromise(grpc_transport* transport, + NextPromiseFactory next_promise_factory) + : impl_(GetContext()->New( + transport, std::move(next_promise_factory))) {} + + ServerConnectedCallPromise(const ServerConnectedCallPromise&) = delete; + ServerConnectedCallPromise& operator=(const ServerConnectedCallPromise&) = + delete; + ServerConnectedCallPromise(ServerConnectedCallPromise&& other) noexcept + : impl_(std::exchange(other.impl_, nullptr)) {} + ServerConnectedCallPromise& operator=( + ServerConnectedCallPromise&& other) noexcept { + impl_ = std::move(other.impl_); + return *this; + } + + static ArenaPromise Make(grpc_transport* transport, + CallArgs, + NextPromiseFactory next) { + return ServerConnectedCallPromise(transport, std::move(next)); + } + + Poll operator()() { return impl_->PollOnce(); } + + private: + OrphanablePtr impl_; +}; + template (*make_call_promise)( - grpc_transport*, CallArgs)> + grpc_transport*, CallArgs, NextPromiseFactory)> grpc_channel_filter MakeConnectedFilter() { // Create a vtable that contains both the legacy call methods (for filter // stack based calls) and the new promise based method for creating promise @@ -835,15 +1347,14 @@ grpc_channel_filter MakeConnectedFilter() { // and only if all the filters in the stack are promise based will the call // be promise based. return { - connected_channel_start_transport_stream_op_batch, - make_call_promise == nullptr - ? nullptr - : +[](grpc_channel_element* elem, CallArgs call_args, - NextPromiseFactory) { - grpc_transport* transport = - static_cast(elem->channel_data)->transport; - return make_call_promise(transport, std::move(call_args)); - }, + connected_channel_start_transport_stream_op_batch, + [](grpc_channel_element* elem, CallArgs call_args, + NextPromiseFactory next) { + grpc_transport* transport = + static_cast(elem->channel_data)->transport; + return make_call_promise(transport, std::move(call_args), + std::move(next)); + }, connected_channel_start_transport_op, sizeof(call_data), connected_channel_init_call_elem, @@ -857,7 +1368,7 @@ grpc_channel_filter MakeConnectedFilter() { // and I'm not sure what that is yet. // This is only "safe" because call stacks place no additional data // after the last call element, and the last call element MUST be the - // connected channel. + // connected channel. channel_stack->call_stack_size += grpc_transport_stream_size( static_cast(elem->channel_data)->transport); }, @@ -868,7 +1379,7 @@ grpc_channel_filter MakeConnectedFilter() { } ArenaPromise MakeTransportCallPromise( - grpc_transport* transport, CallArgs call_args) { + grpc_transport* transport, CallArgs call_args, NextPromiseFactory) { return transport->vtable->make_call_promise(transport, std::move(call_args)); } @@ -878,7 +1389,8 @@ const grpc_channel_filter kPromiseBasedTransportFilter = const grpc_channel_filter kClientEmulatedFilter = MakeConnectedFilter(); -const grpc_channel_filter kNoPromiseFilter = MakeConnectedFilter(); +const grpc_channel_filter kServerEmulatedFilter = + MakeConnectedFilter(); } // namespace } // namespace grpc_core @@ -903,8 +1415,8 @@ bool grpc_add_connected_filter(grpc_core::ChannelStackBuilder* builder) { builder->AppendFilter(&grpc_core::kClientEmulatedFilter); } else { // Option 3: the transport does not support promise based calls, and we're - // on the server so we can't construct promise based calls just yet. - builder->AppendFilter(&grpc_core::kNoPromiseFilter); + // on the server so we use the server filter. + builder->AppendFilter(&grpc_core::kServerEmulatedFilter); } return true; } diff --git a/src/core/lib/channel/promise_based_filter.cc b/src/core/lib/channel/promise_based_filter.cc index ef5bc1d2989..fd334201a2e 100644 --- a/src/core/lib/channel/promise_based_filter.cc +++ b/src/core/lib/channel/promise_based_filter.cc @@ -24,6 +24,7 @@ #include "absl/base/attributes.h" #include "absl/functional/function_ref.h" +#include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" @@ -37,6 +38,7 @@ #include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/status_helper.h" #include "src/core/lib/iomgr/error.h" +#include "src/core/lib/promise/detail/basic_seq.h" #include "src/core/lib/slice/slice.h" extern grpc_core::TraceFlag grpc_trace_channel; @@ -73,24 +75,28 @@ absl::Status StatusFromMetadata(const ServerMetadata& md) { /////////////////////////////////////////////////////////////////////////////// // BaseCallData -BaseCallData::BaseCallData(grpc_call_element* elem, - const grpc_call_element_args* args, uint8_t flags) +BaseCallData::BaseCallData( + grpc_call_element* elem, const grpc_call_element_args* args, uint8_t flags, + absl::FunctionRef make_send_interceptor, + absl::FunctionRef make_recv_interceptor) : call_stack_(args->call_stack), elem_(elem), arena_(args->arena), call_combiner_(args->call_combiner), deadline_(args->deadline), context_(args->context), - server_initial_metadata_latch_( + server_initial_metadata_pipe_( flags & kFilterExaminesServerInitialMetadata - ? arena_->New>() + ? arena_->New>(arena_) + : nullptr), + send_message_( + flags & kFilterExaminesOutboundMessages + ? arena_->New(this, make_send_interceptor()) + : nullptr), + receive_message_( + flags & kFilterExaminesInboundMessages + ? arena_->New(this, make_recv_interceptor()) : nullptr), - send_message_(flags & kFilterExaminesOutboundMessages - ? arena_->New(this) - : nullptr), - receive_message_(flags & kFilterExaminesInboundMessages - ? arena_->New(this) - : nullptr), event_engine_( static_cast(elem->channel_data) ->hack_until_per_channel_stack_event_engines_land_get_event_engine()) { @@ -104,8 +110,8 @@ BaseCallData::~BaseCallData() { if (receive_message_ != nullptr) { receive_message_->~ReceiveMessage(); } - if (server_initial_metadata_latch_ != nullptr) { - server_initial_metadata_latch_->~Latch(); + if (server_initial_metadata_pipe_ != nullptr) { + server_initial_metadata_pipe_->~Pipe(); } }); } @@ -197,7 +203,14 @@ void BaseCallData::CapturedBatch::ResumeWith(Flusher* releaser) { auto* batch = std::exchange(batch_, nullptr); GPR_ASSERT(batch != nullptr); uintptr_t& refcnt = *RefCountField(batch); - if (refcnt == 0) return; // refcnt==0 ==> cancelled + if (refcnt == 0) { + // refcnt==0 ==> cancelled + if (grpc_trace_channel.enabled()) { + gpr_log(GPR_INFO, "%sRESUME BATCH REQUEST CANCELLED", + Activity::current()->DebugTag().c_str()); + } + return; + } if (--refcnt == 0) { releaser->Resume(batch); } @@ -220,6 +233,10 @@ void BaseCallData::CapturedBatch::CancelWith(grpc_error_handle error, uintptr_t& refcnt = *RefCountField(batch); if (refcnt == 0) { // refcnt==0 ==> cancelled + if (grpc_trace_channel.enabled()) { + gpr_log(GPR_INFO, "%sCANCEL BATCH REQUEST ALREADY CANCELLED", + Activity::current()->DebugTag().c_str()); + } return; } refcnt = 0; @@ -330,12 +347,13 @@ void BaseCallData::SendMessage::StartOp(CapturedBatch batch) { intercepted_on_complete_ = std::exchange(batch_->on_complete, &on_complete_); } -void BaseCallData::SendMessage::GotPipe(PipeReceiver* receiver) { +template +void BaseCallData::SendMessage::GotPipe(T* pipe_end) { if (grpc_trace_channel.enabled()) { gpr_log(GPR_INFO, "%s SendMessage.GotPipe st=%s", base_->LogTag().c_str(), StateString(state_)); } - GPR_ASSERT(receiver != nullptr); + GPR_ASSERT(pipe_end != nullptr); switch (state_) { case State::kInitial: state_ = State::kIdle; @@ -355,7 +373,7 @@ void BaseCallData::SendMessage::GotPipe(PipeReceiver* receiver) { case State::kCancelledButNotYetPolled: return; } - receiver_ = receiver; + interceptor_->GotPipe(pipe_end); } bool BaseCallData::SendMessage::IsIdle() const { @@ -399,6 +417,7 @@ void BaseCallData::SendMessage::OnComplete(absl::Status status) { case State::kForwardedBatch: completed_status_ = status; state_ = State::kBatchCompleted; + ScopedContext ctx(base_); base_->WakeInsideCombiner(&flusher); break; } @@ -442,7 +461,8 @@ void BaseCallData::SendMessage::Done(const ServerMetadata& metadata, } } -void BaseCallData::SendMessage::WakeInsideCombiner(Flusher* flusher) { +void BaseCallData::SendMessage::WakeInsideCombiner(Flusher* flusher, + bool allow_push_to_pipe) { if (grpc_trace_channel.enabled()) { gpr_log(GPR_INFO, "%s SendMessage.WakeInsideCombiner st=%s%s", base_->LogTag().c_str(), StateString(state_), @@ -454,21 +474,23 @@ void BaseCallData::SendMessage::WakeInsideCombiner(Flusher* flusher) { case State::kInitial: case State::kIdle: case State::kGotBatchNoPipe: - case State::kForwardedBatch: case State::kCancelled: break; case State::kCancelledButNotYetPolled: - pipe_.sender.Close(); + interceptor()->Push()->Close(); state_ = State::kCancelled; break; - case State::kGotBatch: { - state_ = State::kPushedToPipe; - auto message = GetContext()->MakePooled(); - message->payload()->Swap(batch_->payload->send_message.send_message); - message->mutable_flags() = batch_->payload->send_message.flags; - push_ = pipe_.sender.Push(std::move(message)); - next_ = receiver_->Next(); - } + case State::kGotBatch: + if (allow_push_to_pipe) { + state_ = State::kPushedToPipe; + auto message = GetContext()->MakePooled(); + message->payload()->Swap(batch_->payload->send_message.send_message); + message->mutable_flags() = batch_->payload->send_message.flags; + push_ = interceptor()->Push()->Push(std::move(message)); + next_.emplace(interceptor()->Pull()->Next()); + } else { + break; + } ABSL_FALLTHROUGH_INTENDED; case State::kPushedToPipe: { GPR_ASSERT(push_.has_value()); @@ -499,25 +521,25 @@ void BaseCallData::SendMessage::WakeInsideCombiner(Flusher* flusher) { batch_->payload->send_message.flags = (**p)->flags(); state_ = State::kForwardedBatch; batch_.ResumeWith(flusher); - next_result_ = std::move(*p); next_.reset(); + if (!absl::holds_alternative((*push_)())) push_.reset(); } } break; + case State::kForwardedBatch: + if (push_.has_value() && !absl::holds_alternative((*push_)())) { + push_.reset(); + } + break; case State::kBatchCompleted: - next_result_.reset(); - // We've cleared out the NextResult on the pipe from promise to us, but - // there's also the pipe from us to the promise (so that the promise can - // intercept the sent messages). The push promise here is pushing into the - // latter pipe, and so we need to keep polling it until it's done, which - // depending on what happens inside the promise may take some time. - if (absl::holds_alternative((*push_)())) break; + if (push_.has_value() && absl::holds_alternative((*push_)())) { + break; + } if (completed_status_.ok()) { state_ = State::kIdle; Activity::current()->ForceImmediateRepoll(); } else { state_ = State::kCancelled; } - push_.reset(); flusher->AddClosure(intercepted_on_complete_, completed_status_, "batch_completed"); break; @@ -557,6 +579,8 @@ const char* BaseCallData::ReceiveMessage::StateString(State state) { return "COMPLETED_WHILE_PULLED_FROM_PIPE"; case State::kCompletedWhilePushedToPipe: return "COMPLETED_WHILE_PUSHED_TO_PIPE"; + case State::kCompletedWhileBatchCompleted: + return "COMPLETED_WHILE_BATCH_COMPLETED"; } return "UNKNOWN"; } @@ -579,6 +603,7 @@ void BaseCallData::ReceiveMessage::StartOp(CapturedBatch& batch) { case State::kForwardedBatchNoPipe: case State::kBatchCompleted: case State::kBatchCompletedNoPipe: + case State::kCompletedWhileBatchCompleted: case State::kPushedToPipe: case State::kPulledFromPipe: case State::kCompletedWhilePulledFromPipe: @@ -598,7 +623,8 @@ void BaseCallData::ReceiveMessage::StartOp(CapturedBatch& batch) { batch->payload->recv_message.recv_message_ready, &on_complete_); } -void BaseCallData::ReceiveMessage::GotPipe(PipeSender* sender) { +template +void BaseCallData::ReceiveMessage::GotPipe(T* pipe_end) { if (grpc_trace_channel.enabled()) { gpr_log(GPR_INFO, "%s ReceiveMessage.GotPipe st=%s", base_->LogTag().c_str(), StateString(state_)); @@ -621,6 +647,7 @@ void BaseCallData::ReceiveMessage::GotPipe(PipeSender* sender) { case State::kPulledFromPipe: case State::kCompletedWhilePulledFromPipe: case State::kCompletedWhilePushedToPipe: + case State::kCompletedWhileBatchCompleted: case State::kCancelledWhilstForwarding: case State::kCancelledWhilstIdle: case State::kBatchCompletedButCancelled: @@ -628,7 +655,7 @@ void BaseCallData::ReceiveMessage::GotPipe(PipeSender* sender) { case State::kCancelled: return; } - sender_ = sender; + interceptor()->GotPipe(pipe_end); } void BaseCallData::ReceiveMessage::OnComplete(absl::Status status) { @@ -643,6 +670,7 @@ void BaseCallData::ReceiveMessage::OnComplete(absl::Status status) { case State::kPushedToPipe: case State::kPulledFromPipe: case State::kBatchCompleted: + case State::kCompletedWhileBatchCompleted: case State::kBatchCompletedNoPipe: case State::kCancelled: case State::kBatchCompletedButCancelled: @@ -684,6 +712,10 @@ void BaseCallData::ReceiveMessage::Done(const ServerMetadata& metadata, case State::kForwardedBatchNoPipe: state_ = State::kCancelledWhilstForwarding; break; + case State::kCompletedWhileBatchCompleted: + case State::kBatchCompleted: + state_ = State::kCompletedWhileBatchCompleted; + break; case State::kCompletedWhilePulledFromPipe: case State::kCompletedWhilePushedToPipe: case State::kPulledFromPipe: @@ -705,7 +737,6 @@ void BaseCallData::ReceiveMessage::Done(const ServerMetadata& metadata, state_ = State::kCancelled; } } break; - case State::kBatchCompleted: case State::kBatchCompletedNoPipe: case State::kBatchCompletedButCancelled: Crash(absl::StrFormat("ILLEGAL STATE: %s", StateString(state_))); @@ -716,12 +747,15 @@ void BaseCallData::ReceiveMessage::Done(const ServerMetadata& metadata, } } -void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher) { +void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher, + bool allow_push_to_pipe) { if (grpc_trace_channel.enabled()) { gpr_log(GPR_INFO, - "%s ReceiveMessage.WakeInsideCombiner st=%s push?=%s next?=%s", + "%s ReceiveMessage.WakeInsideCombiner st=%s push?=%s next?=%s " + "allow_push_to_pipe=%s", base_->LogTag().c_str(), StateString(state_), - push_.has_value() ? "yes" : "no", next_.has_value() ? "yes" : "no"); + push_.has_value() ? "yes" : "no", next_.has_value() ? "yes" : "no", + allow_push_to_pipe ? "yes" : "no"); } switch (state_) { case State::kInitial: @@ -733,31 +767,38 @@ void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher) { case State::kBatchCompletedNoPipe: break; case State::kCancelledWhilstIdle: - sender_->Close(); + interceptor_->Push()->Close(); state_ = State::kCancelled; break; case State::kBatchCompletedButCancelled: - sender_->Close(); + case State::kCompletedWhileBatchCompleted: + interceptor()->Push()->Close(); state_ = State::kCancelled; flusher->AddClosure(std::exchange(intercepted_on_complete_, nullptr), completed_status_, "recv_message"); break; case State::kBatchCompleted: if (completed_status_.ok() && intercepted_slice_buffer_->has_value()) { - state_ = State::kPushedToPipe; + if (!allow_push_to_pipe) break; + if (state_ == State::kBatchCompleted) { + state_ = State::kPushedToPipe; + } else { + state_ = State::kCompletedWhilePushedToPipe; + } auto message = GetContext()->MakePooled(); message->payload()->Swap(&**intercepted_slice_buffer_); message->mutable_flags() = *intercepted_flags_; - push_ = sender_->Push(std::move(message)); - next_ = pipe_.receiver.Next(); + push_ = interceptor()->Push()->Push(std::move(message)); + next_.emplace(interceptor()->Pull()->Next()); } else { - sender_->Close(); + interceptor()->Push()->Close(); state_ = State::kCancelled; flusher->AddClosure(std::exchange(intercepted_on_complete_, nullptr), completed_status_, "recv_message"); break; } - GPR_ASSERT(state_ == State::kPushedToPipe); + GPR_ASSERT(state_ == State::kPushedToPipe || + state_ == State::kCompletedWhilePushedToPipe); ABSL_FALLTHROUGH_INTENDED; case State::kCompletedWhilePushedToPipe: case State::kPushedToPipe: { @@ -777,12 +818,6 @@ void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher) { GPR_ASSERT(next_.has_value()); auto r_next = (*next_)(); if (auto* p = absl::get_if>(&r_next)) { - if (grpc_trace_channel.enabled()) { - gpr_log(GPR_INFO, - "%s ReceiveMessage.WakeInsideCombiner next complete: %s", - base_->LogTag().c_str(), - p->has_value() ? "got message" : "end of stream"); - } next_.reset(); if (p->has_value()) { *intercepted_slice_buffer_ = std::move(*(**p)->payload()); @@ -796,6 +831,18 @@ void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher) { *intercepted_slice_buffer_ = absl::nullopt; *intercepted_flags_ = 0; state_ = State::kCancelled; + flusher->AddClosure( + std::exchange(intercepted_on_complete_, nullptr), + p->cancelled() ? absl::CancelledError() : absl::OkStatus(), + "recv_message"); + } + if (grpc_trace_channel.enabled()) { + gpr_log(GPR_INFO, + "%s ReceiveMessage.WakeInsideCombiner next complete: %s " + "new_state=%s", + base_->LogTag().c_str(), + p->has_value() ? "got message" : "end of stream", + StateString(state_)); } } } @@ -814,7 +861,7 @@ void BaseCallData::ReceiveMessage::WakeInsideCombiner(Flusher* flusher) { base_->LogTag().c_str()); } if (state_ == State::kCompletedWhilePulledFromPipe) { - sender_->Close(); + interceptor()->Push()->Close(); state_ = State::kCancelled; } else { state_ = State::kIdle; @@ -836,56 +883,76 @@ struct ClientCallData::RecvInitialMetadata final { // Initial state; no op seen kInitial, // No op seen, but we have a latch that would like to modify it when we do - kGotLatch, + kGotPipe, // Responded to trailing metadata prior to getting a recv_initial_metadata kRespondedToTrailingMetadataPriorToHook, // Hooked, no latch yet - kHookedWaitingForLatch, + kHookedWaitingForPipe, // Hooked, latch seen - kHookedAndGotLatch, + kHookedAndGotPipe, // Got the callback, haven't set latch yet - kCompleteWaitingForLatch, + kCompleteWaitingForPipe, // Got the callback and got the latch - kCompleteAndGotLatch, + kCompleteAndGotPipe, // Got the callback and set the latch - kCompleteAndSetLatch, + kCompleteAndPushedToPipe, // Called the original callback kResponded, // Called the original callback with an error: still need to set the latch - kRespondedButNeedToSetLatch, + kRespondedButNeedToClosePipe, }; State state = kInitial; grpc_closure* original_on_ready = nullptr; grpc_closure on_ready; grpc_metadata_batch* metadata = nullptr; - Latch* server_initial_metadata_publisher = nullptr; + PipeSender* server_initial_metadata_publisher = nullptr; + absl::optional::PushType> metadata_push_; + absl::optional> metadata_next_; static const char* StateString(State state) { switch (state) { case kInitial: return "INITIAL"; - case kGotLatch: - return "GOT_LATCH"; + case kGotPipe: + return "GOT_PIPE"; case kRespondedToTrailingMetadataPriorToHook: return "RESPONDED_TO_TRAILING_METADATA_PRIOR_TO_HOOK"; - case kHookedWaitingForLatch: - return "HOOKED_WAITING_FOR_LATCH"; - case kHookedAndGotLatch: - return "HOOKED_AND_GOT_LATCH"; - case kCompleteWaitingForLatch: - return "COMPLETE_WAITING_FOR_LATCH"; - case kCompleteAndGotLatch: - return "COMPLETE_AND_GOT_LATCH"; - case kCompleteAndSetLatch: - return "COMPLETE_AND_SET_LATCH"; + case kHookedWaitingForPipe: + return "HOOKED_WAITING_FOR_PIPE"; + case kHookedAndGotPipe: + return "HOOKED_AND_GOT_PIPE"; + case kCompleteWaitingForPipe: + return "COMPLETE_WAITING_FOR_PIPE"; + case kCompleteAndGotPipe: + return "COMPLETE_AND_GOT_PIPE"; + case kCompleteAndPushedToPipe: + return "COMPLETE_AND_PUSHED_TO_PIPE"; case kResponded: return "RESPONDED"; - case kRespondedButNeedToSetLatch: - return "RESPONDED_BUT_NEED_TO_SET_LATCH"; + case kRespondedButNeedToClosePipe: + return "RESPONDED_BUT_NEED_TO_CLOSE_PIPE"; } return "UNKNOWN"; } + + bool AllowRecvMessage() const { + switch (state) { + case kInitial: + case kGotPipe: + case kHookedWaitingForPipe: + case kHookedAndGotPipe: + case kCompleteWaitingForPipe: + case kCompleteAndGotPipe: + case kCompleteAndPushedToPipe: + case kRespondedToTrailingMetadataPriorToHook: + return false; + case kResponded: + case kRespondedButNeedToClosePipe: + return true; + } + GPR_UNREACHABLE_CODE(return false); + } }; class ClientCallData::PollContext { @@ -903,6 +970,7 @@ class ClientCallData::PollContext { PollContext& operator=(const PollContext&) = delete; void Run() { + GPR_DEBUG_ASSERT(HasContext()); if (grpc_trace_channel.enabled()) { gpr_log(GPR_INFO, "%s ClientCallData.PollContext.Run %s", self_->LogTag().c_str(), self_->DebugString().c_str()); @@ -910,43 +978,69 @@ class ClientCallData::PollContext { GPR_ASSERT(have_scoped_activity_); repoll_ = false; if (self_->send_message() != nullptr) { - self_->send_message()->WakeInsideCombiner(flusher_); + self_->send_message()->WakeInsideCombiner(flusher_, true); } if (self_->receive_message() != nullptr) { - self_->receive_message()->WakeInsideCombiner(flusher_); + self_->receive_message()->WakeInsideCombiner( + flusher_, self_->recv_initial_metadata_ == nullptr + ? true + : self_->recv_initial_metadata_->AllowRecvMessage()); } - if (self_->server_initial_metadata_latch() != nullptr) { + if (self_->server_initial_metadata_pipe() != nullptr) { + if (self_->recv_initial_metadata_->metadata_push_.has_value()) { + if (!absl::holds_alternative( + (*self_->recv_initial_metadata_->metadata_push_)())) { + self_->recv_initial_metadata_->metadata_push_.reset(); + } + } switch (self_->recv_initial_metadata_->state) { case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotLatch: - case RecvInitialMetadata::kHookedWaitingForLatch: - case RecvInitialMetadata::kHookedAndGotLatch: - case RecvInitialMetadata::kCompleteWaitingForLatch: + case RecvInitialMetadata::kGotPipe: + case RecvInitialMetadata::kHookedWaitingForPipe: + case RecvInitialMetadata::kHookedAndGotPipe: + case RecvInitialMetadata::kCompleteWaitingForPipe: case RecvInitialMetadata::kResponded: case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: break; - case RecvInitialMetadata::kRespondedButNeedToSetLatch: - self_->recv_initial_metadata_->server_initial_metadata_publisher->Set( - nullptr); + case RecvInitialMetadata::kRespondedButNeedToClosePipe: + self_->recv_initial_metadata_->server_initial_metadata_publisher + ->Close(); self_->recv_initial_metadata_->state = RecvInitialMetadata::kResponded; break; - case RecvInitialMetadata::kCompleteAndGotLatch: + case RecvInitialMetadata::kCompleteAndGotPipe: self_->recv_initial_metadata_->state = - RecvInitialMetadata::kCompleteAndSetLatch; - self_->recv_initial_metadata_->server_initial_metadata_publisher->Set( - self_->recv_initial_metadata_->metadata); + RecvInitialMetadata::kCompleteAndPushedToPipe; + GPR_ASSERT( + !self_->recv_initial_metadata_->metadata_push_.has_value()); + GPR_ASSERT( + !self_->recv_initial_metadata_->metadata_next_.has_value()); + self_->recv_initial_metadata_->metadata_push_.emplace( + self_->recv_initial_metadata_->server_initial_metadata_publisher + ->Push(ServerMetadataHandle( + self_->recv_initial_metadata_->metadata, + Arena::PooledDeleter(nullptr)))); + repoll_ = true; // ensure Push() gets polled. + self_->recv_initial_metadata_->metadata_next_.emplace( + self_->server_initial_metadata_pipe()->receiver.Next()); ABSL_FALLTHROUGH_INTENDED; - case RecvInitialMetadata::kCompleteAndSetLatch: { - Poll p = - self_->server_initial_metadata_latch()->Wait()(); - if (ServerMetadata*** ppp = absl::get_if(&p)) { - ServerMetadata* md = **ppp; - if (self_->recv_initial_metadata_->metadata != md) { - *self_->recv_initial_metadata_->metadata = std::move(*md); + case RecvInitialMetadata::kCompleteAndPushedToPipe: { + GPR_ASSERT(self_->recv_initial_metadata_->metadata_next_.has_value()); + Poll> p = + (*self_->recv_initial_metadata_->metadata_next_)(); + if (NextResult* nr = + absl::get_if(&p)) { + if (nr->has_value()) { + ServerMetadataHandle md = std::move(nr->value()); + if (self_->recv_initial_metadata_->metadata != md.get()) { + *self_->recv_initial_metadata_->metadata = std::move(*md); + } + } else { + self_->recv_initial_metadata_->metadata->Clear(); } self_->recv_initial_metadata_->state = RecvInitialMetadata::kResponded; + repoll_ = true; flusher_->AddClosure( std::exchange(self_->recv_initial_metadata_->original_on_ready, nullptr), @@ -992,25 +1086,25 @@ class ClientCallData::PollContext { if (self_->recv_initial_metadata_ != nullptr) { switch (self_->recv_initial_metadata_->state) { case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotLatch: + case RecvInitialMetadata::kGotPipe: self_->recv_initial_metadata_->state = RecvInitialMetadata:: kRespondedToTrailingMetadataPriorToHook; break; case RecvInitialMetadata:: kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kRespondedButNeedToSetLatch: + case RecvInitialMetadata::kRespondedButNeedToClosePipe: Crash(absl::StrFormat("ILLEGAL STATE: %s", RecvInitialMetadata::StateString( self_->recv_initial_metadata_ ->state))); // not reachable break; - case RecvInitialMetadata::kHookedWaitingForLatch: - case RecvInitialMetadata::kHookedAndGotLatch: + case RecvInitialMetadata::kHookedWaitingForPipe: + case RecvInitialMetadata::kHookedAndGotPipe: case RecvInitialMetadata::kResponded: - case RecvInitialMetadata::kCompleteAndGotLatch: - case RecvInitialMetadata::kCompleteAndSetLatch: + case RecvInitialMetadata::kCompleteAndGotPipe: + case RecvInitialMetadata::kCompleteAndPushedToPipe: break; - case RecvInitialMetadata::kCompleteWaitingForLatch: + case RecvInitialMetadata::kCompleteWaitingForPipe: self_->recv_initial_metadata_->state = RecvInitialMetadata::kResponded; flusher_->AddClosure( @@ -1027,25 +1121,25 @@ class ClientCallData::PollContext { if (self_->recv_initial_metadata_ != nullptr) { switch (self_->recv_initial_metadata_->state) { case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotLatch: + case RecvInitialMetadata::kGotPipe: self_->recv_initial_metadata_->state = RecvInitialMetadata:: kRespondedToTrailingMetadataPriorToHook; break; - case RecvInitialMetadata::kHookedWaitingForLatch: - case RecvInitialMetadata::kHookedAndGotLatch: + case RecvInitialMetadata::kHookedWaitingForPipe: + case RecvInitialMetadata::kHookedAndGotPipe: case RecvInitialMetadata::kResponded: break; case RecvInitialMetadata:: kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kRespondedButNeedToSetLatch: + case RecvInitialMetadata::kRespondedButNeedToClosePipe: Crash(absl::StrFormat("ILLEGAL STATE: %s", RecvInitialMetadata::StateString( self_->recv_initial_metadata_ ->state))); // not reachable break; - case RecvInitialMetadata::kCompleteWaitingForLatch: - case RecvInitialMetadata::kCompleteAndGotLatch: - case RecvInitialMetadata::kCompleteAndSetLatch: + case RecvInitialMetadata::kCompleteWaitingForPipe: + case RecvInitialMetadata::kCompleteAndGotPipe: + case RecvInitialMetadata::kCompleteAndPushedToPipe: self_->recv_initial_metadata_->state = RecvInitialMetadata::kResponded; flusher_->AddClosure( @@ -1110,6 +1204,7 @@ class ClientCallData::PollContext { auto run = [](void* p, grpc_error_handle) { auto* next_poll = static_cast(p); { + ScopedContext ctx(next_poll->call_data); Flusher flusher(next_poll->call_data); next_poll->call_data->WakeInsideCombiner(&flusher); } @@ -1144,11 +1239,16 @@ class ClientCallData::PollContext { ClientCallData::ClientCallData(grpc_call_element* elem, const grpc_call_element_args* args, uint8_t flags) - : BaseCallData(elem, args, flags) { + : BaseCallData( + elem, args, flags, + [args]() { + return args->arena->New(args->arena); + }, + [args]() { return args->arena->New(args->arena); }) { GRPC_CLOSURE_INIT(&recv_trailing_metadata_ready_, RecvTrailingMetadataReadyCallback, this, grpc_schedule_on_exec_ctx); - if (server_initial_metadata_latch() != nullptr) { + if (server_initial_metadata_pipe() != nullptr) { recv_initial_metadata_ = arena()->New(); } } @@ -1211,7 +1311,7 @@ std::string ClientCallData::DebugString() const { " sent_initial_state=", StateString(send_initial_state_), " recv_trailing_state=", StateString(recv_trailing_state_), " captured={", absl::StrJoin(captured, ","), "}", - server_initial_metadata_latch() == nullptr + server_initial_metadata_pipe() == nullptr ? "" : absl::StrCat(" recv_initial_metadata=", RecvInitialMetadata::StateString( @@ -1253,21 +1353,21 @@ void ClientCallData::StartBatch(grpc_transport_stream_op_batch* b) { switch (recv_initial_metadata_->state) { case RecvInitialMetadata::kInitial: recv_initial_metadata_->state = - RecvInitialMetadata::kHookedWaitingForLatch; + RecvInitialMetadata::kHookedWaitingForPipe; break; - case RecvInitialMetadata::kGotLatch: - recv_initial_metadata_->state = RecvInitialMetadata::kHookedAndGotLatch; + case RecvInitialMetadata::kGotPipe: + recv_initial_metadata_->state = RecvInitialMetadata::kHookedAndGotPipe; break; case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: hook = false; break; - case RecvInitialMetadata::kHookedWaitingForLatch: - case RecvInitialMetadata::kHookedAndGotLatch: - case RecvInitialMetadata::kCompleteWaitingForLatch: - case RecvInitialMetadata::kCompleteAndGotLatch: - case RecvInitialMetadata::kCompleteAndSetLatch: + case RecvInitialMetadata::kHookedWaitingForPipe: + case RecvInitialMetadata::kHookedAndGotPipe: + case RecvInitialMetadata::kCompleteWaitingForPipe: + case RecvInitialMetadata::kCompleteAndGotPipe: + case RecvInitialMetadata::kCompleteAndPushedToPipe: case RecvInitialMetadata::kResponded: - case RecvInitialMetadata::kRespondedButNeedToSetLatch: + case RecvInitialMetadata::kRespondedButNeedToClosePipe: Crash(absl::StrFormat( "ILLEGAL STATE: %s", RecvInitialMetadata::StateString( @@ -1371,9 +1471,9 @@ void ClientCallData::Cancel(grpc_error_handle error, Flusher* flusher) { } if (recv_initial_metadata_ != nullptr) { switch (recv_initial_metadata_->state) { - case RecvInitialMetadata::kCompleteWaitingForLatch: - case RecvInitialMetadata::kCompleteAndGotLatch: - case RecvInitialMetadata::kCompleteAndSetLatch: + case RecvInitialMetadata::kCompleteWaitingForPipe: + case RecvInitialMetadata::kCompleteAndGotPipe: + case RecvInitialMetadata::kCompleteAndPushedToPipe: recv_initial_metadata_->state = RecvInitialMetadata::kResponded; GRPC_CALL_COMBINER_START( call_combiner(), @@ -1381,13 +1481,13 @@ void ClientCallData::Cancel(grpc_error_handle error, Flusher* flusher) { error, "propagate cancellation"); break; case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotLatch: + case RecvInitialMetadata::kGotPipe: case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kHookedWaitingForLatch: - case RecvInitialMetadata::kHookedAndGotLatch: + case RecvInitialMetadata::kHookedWaitingForPipe: + case RecvInitialMetadata::kHookedAndGotPipe: case RecvInitialMetadata::kResponded: break; - case RecvInitialMetadata::kRespondedButNeedToSetLatch: + case RecvInitialMetadata::kRespondedButNeedToClosePipe: Crash(absl::StrFormat( "ILLEGAL STATE: %s", RecvInitialMetadata::StateString(recv_initial_metadata_->state))); @@ -1413,8 +1513,15 @@ void ClientCallData::StartPromise(Flusher* flusher) { promise_ = filter->MakeCallPromise( CallArgs{WrapMetadata(send_initial_metadata_batch_->payload ->send_initial_metadata.send_initial_metadata), - server_initial_metadata_latch(), outgoing_messages_pipe(), - incoming_messages_pipe()}, + server_initial_metadata_pipe() == nullptr + ? nullptr + : &server_initial_metadata_pipe()->sender, + send_message() == nullptr + ? nullptr + : send_message()->interceptor()->original_receiver(), + receive_message() == nullptr + ? nullptr + : receive_message()->interceptor()->original_sender()}, [this](CallArgs call_args) { return MakeNextPromise(std::move(call_args)); }); @@ -1423,28 +1530,30 @@ void ClientCallData::StartPromise(Flusher* flusher) { void ClientCallData::RecvInitialMetadataReady(grpc_error_handle error) { if (grpc_trace_channel.enabled()) { - gpr_log(GPR_INFO, "%s ClientCallData.RecvInitialMetadataReady %s", - LogTag().c_str(), DebugString().c_str()); + gpr_log(GPR_INFO, + "%s ClientCallData.RecvInitialMetadataReady %s error:%s md:%s", + LogTag().c_str(), DebugString().c_str(), error.ToString().c_str(), + recv_initial_metadata_->metadata->DebugString().c_str()); } ScopedContext context(this); Flusher flusher(this); if (!error.ok()) { switch (recv_initial_metadata_->state) { - case RecvInitialMetadata::kHookedWaitingForLatch: + case RecvInitialMetadata::kHookedWaitingForPipe: recv_initial_metadata_->state = RecvInitialMetadata::kResponded; break; - case RecvInitialMetadata::kHookedAndGotLatch: + case RecvInitialMetadata::kHookedAndGotPipe: recv_initial_metadata_->state = - RecvInitialMetadata::kRespondedButNeedToSetLatch; + RecvInitialMetadata::kRespondedButNeedToClosePipe; break; case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotLatch: - case RecvInitialMetadata::kCompleteWaitingForLatch: - case RecvInitialMetadata::kCompleteAndGotLatch: - case RecvInitialMetadata::kCompleteAndSetLatch: + case RecvInitialMetadata::kGotPipe: + case RecvInitialMetadata::kCompleteWaitingForPipe: + case RecvInitialMetadata::kCompleteAndGotPipe: + case RecvInitialMetadata::kCompleteAndPushedToPipe: case RecvInitialMetadata::kResponded: case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kRespondedButNeedToSetLatch: + case RecvInitialMetadata::kRespondedButNeedToClosePipe: Crash(absl::StrFormat( "ILLEGAL STATE: %s", RecvInitialMetadata::StateString( @@ -1461,22 +1570,22 @@ void ClientCallData::RecvInitialMetadataReady(grpc_error_handle error) { cancelled_error_, "propagate cancellation"); } else { switch (recv_initial_metadata_->state) { - case RecvInitialMetadata::kHookedWaitingForLatch: + case RecvInitialMetadata::kHookedWaitingForPipe: recv_initial_metadata_->state = - RecvInitialMetadata::kCompleteWaitingForLatch; + RecvInitialMetadata::kCompleteWaitingForPipe; break; - case RecvInitialMetadata::kHookedAndGotLatch: + case RecvInitialMetadata::kHookedAndGotPipe: recv_initial_metadata_->state = - RecvInitialMetadata::kCompleteAndGotLatch; + RecvInitialMetadata::kCompleteAndGotPipe; break; case RecvInitialMetadata::kInitial: - case RecvInitialMetadata::kGotLatch: - case RecvInitialMetadata::kCompleteWaitingForLatch: - case RecvInitialMetadata::kCompleteAndGotLatch: - case RecvInitialMetadata::kCompleteAndSetLatch: + case RecvInitialMetadata::kGotPipe: + case RecvInitialMetadata::kCompleteWaitingForPipe: + case RecvInitialMetadata::kCompleteAndGotPipe: + case RecvInitialMetadata::kCompleteAndPushedToPipe: case RecvInitialMetadata::kResponded: case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kRespondedButNeedToSetLatch: + case RecvInitialMetadata::kRespondedButNeedToClosePipe: Crash(absl::StrFormat( "ILLEGAL STATE: %s", RecvInitialMetadata::StateString( @@ -1523,24 +1632,24 @@ ArenaPromise ClientCallData::MakeNextPromise( call_args.server_initial_metadata; switch (recv_initial_metadata_->state) { case RecvInitialMetadata::kInitial: - recv_initial_metadata_->state = RecvInitialMetadata::kGotLatch; + recv_initial_metadata_->state = RecvInitialMetadata::kGotPipe; break; - case RecvInitialMetadata::kHookedWaitingForLatch: - recv_initial_metadata_->state = RecvInitialMetadata::kHookedAndGotLatch; + case RecvInitialMetadata::kHookedWaitingForPipe: + recv_initial_metadata_->state = RecvInitialMetadata::kHookedAndGotPipe; poll_ctx_->Repoll(); break; - case RecvInitialMetadata::kCompleteWaitingForLatch: + case RecvInitialMetadata::kCompleteWaitingForPipe: recv_initial_metadata_->state = - RecvInitialMetadata::kCompleteAndGotLatch; + RecvInitialMetadata::kCompleteAndGotPipe; poll_ctx_->Repoll(); break; - case RecvInitialMetadata::kGotLatch: - case RecvInitialMetadata::kHookedAndGotLatch: - case RecvInitialMetadata::kCompleteAndGotLatch: - case RecvInitialMetadata::kCompleteAndSetLatch: + case RecvInitialMetadata::kGotPipe: + case RecvInitialMetadata::kHookedAndGotPipe: + case RecvInitialMetadata::kCompleteAndGotPipe: + case RecvInitialMetadata::kCompleteAndPushedToPipe: case RecvInitialMetadata::kResponded: case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook: - case RecvInitialMetadata::kRespondedButNeedToSetLatch: + case RecvInitialMetadata::kRespondedButNeedToClosePipe: Crash(absl::StrFormat( "ILLEGAL STATE: %s", RecvInitialMetadata::StateString( @@ -1550,14 +1659,14 @@ ArenaPromise ClientCallData::MakeNextPromise( GPR_ASSERT(call_args.server_initial_metadata == nullptr); } if (send_message() != nullptr) { - send_message()->GotPipe(call_args.outgoing_messages); + send_message()->GotPipe(call_args.client_to_server_messages); } else { - GPR_ASSERT(call_args.outgoing_messages == nullptr); + GPR_ASSERT(call_args.client_to_server_messages == nullptr); } if (receive_message() != nullptr) { - receive_message()->GotPipe(call_args.incoming_messages); + receive_message()->GotPipe(call_args.server_to_client_messages); } else { - GPR_ASSERT(call_args.incoming_messages == nullptr); + GPR_ASSERT(call_args.server_to_client_messages == nullptr); } return ArenaPromise( [this]() { return PollTrailingMetadata(); }); @@ -1688,29 +1797,31 @@ void ClientCallData::OnWakeup() { struct ServerCallData::SendInitialMetadata { enum State { kInitial, - kGotLatch, - kQueuedWaitingForLatch, - kQueuedAndGotLatch, - kQueuedAndSetLatch, + kGotPipe, + kQueuedWaitingForPipe, + kQueuedAndGotPipe, + kQueuedAndPushedToPipe, kForwarded, kCancelled, }; State state = kInitial; CapturedBatch batch; - Latch* server_initial_metadata_publisher = nullptr; + PipeSender* server_initial_metadata_publisher = nullptr; + absl::optional::PushType> metadata_push_; + absl::optional> metadata_next_; static const char* StateString(State state) { switch (state) { case kInitial: return "INITIAL"; - case kGotLatch: - return "GOT_LATCH"; - case kQueuedWaitingForLatch: - return "QUEUED_WAITING_FOR_LATCH"; - case kQueuedAndGotLatch: - return "QUEUED_AND_GOT_LATCH"; - case kQueuedAndSetLatch: - return "QUEUED_AND_SET_LATCH"; + case kGotPipe: + return "GOT_PIPE"; + case kQueuedWaitingForPipe: + return "QUEUED_WAITING_FOR_PIPE"; + case kQueuedAndGotPipe: + return "QUEUED_AND_GOT_PIPE"; + case kQueuedAndPushedToPipe: + return "QUEUED_AND_PUSHED_TO_PIPE"; case kForwarded: return "FORWARDED"; case kCancelled: @@ -1745,6 +1856,7 @@ class ServerCallData::PollContext { auto* next_poll = static_cast(p); { Flusher flusher(next_poll->call_data); + ScopedContext context(next_poll->call_data); next_poll->call_data->WakeInsideCombiner(&flusher); } GRPC_CALL_STACK_UNREF(next_poll->call_stack, "re-poll"); @@ -1792,6 +1904,8 @@ const char* ServerCallData::StateString(SendTrailingState state) { return "FORWARDED"; case SendTrailingState::kQueuedBehindSendMessage: return "QUEUED_BEHIND_SEND_MESSAGE"; + case SendTrailingState::kQueuedButHaventClosedSends: + return "QUEUED_BUT_HAVENT_CLOSED_SENDS"; case SendTrailingState::kQueued: return "QUEUED"; case SendTrailingState::kCancelled: @@ -1803,8 +1917,13 @@ const char* ServerCallData::StateString(SendTrailingState state) { ServerCallData::ServerCallData(grpc_call_element* elem, const grpc_call_element_args* args, uint8_t flags) - : BaseCallData(elem, args, flags) { - if (server_initial_metadata_latch() != nullptr) { + : BaseCallData( + elem, args, flags, + [args]() { return args->arena->New(args->arena); }, + [args]() { + return args->arena->New(args->arena); + }) { + if (server_initial_metadata_pipe() != nullptr) { send_initial_metadata_ = arena()->New(); } GRPC_CLOSURE_INIT(&recv_initial_metadata_ready_, @@ -1892,19 +2011,19 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* b) { switch (send_initial_metadata_->state) { case SendInitialMetadata::kInitial: send_initial_metadata_->state = - SendInitialMetadata::kQueuedWaitingForLatch; + SendInitialMetadata::kQueuedWaitingForPipe; break; - case SendInitialMetadata::kGotLatch: - send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotLatch; + case SendInitialMetadata::kGotPipe: + send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotPipe; break; case SendInitialMetadata::kCancelled: batch.CancelWith( cancelled_error_.ok() ? absl::CancelledError() : cancelled_error_, &flusher); break; - case SendInitialMetadata::kQueuedAndGotLatch: - case SendInitialMetadata::kQueuedWaitingForLatch: - case SendInitialMetadata::kQueuedAndSetLatch: + case SendInitialMetadata::kQueuedAndGotPipe: + case SendInitialMetadata::kQueuedWaitingForPipe: + case SendInitialMetadata::kQueuedAndPushedToPipe: case SendInitialMetadata::kForwarded: Crash(absl::StrFormat( "ILLEGAL STATE: %s", @@ -1937,6 +2056,9 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* b) { } if (send_message() != nullptr && !send_message()->IsIdle()) { send_trailing_state_ = SendTrailingState::kQueuedBehindSendMessage; + } else if (send_message() != nullptr) { + send_trailing_state_ = SendTrailingState::kQueuedButHaventClosedSends; + wake = true; } else { send_trailing_state_ = SendTrailingState::kQueued; wake = true; @@ -1944,6 +2066,7 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* b) { break; case SendTrailingState::kQueued: case SendTrailingState::kQueuedBehindSendMessage: + case SendTrailingState::kQueuedButHaventClosedSends: case SendTrailingState::kForwarded: Crash( absl::StrFormat("ILLEGAL STATE: %s", @@ -1963,26 +2086,55 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* b) { // Handle cancellation. void ServerCallData::Completed(grpc_error_handle error, Flusher* flusher) { + if (grpc_trace_channel.enabled()) { + gpr_log( + GPR_DEBUG, + "%sServerCallData::Completed: send_trailing_state=%s " + "send_initial_state=%s error=%s", + LogTag().c_str(), StateString(send_trailing_state_), + send_initial_metadata_ == nullptr + ? "null" + : SendInitialMetadata::StateString(send_initial_metadata_->state), + error.ToString().c_str()); + } // Track the latest reason for cancellation. cancelled_error_ = error; // Stop running the promise. promise_ = ArenaPromise(); - if (send_trailing_state_ == SendTrailingState::kQueued) { - send_trailing_state_ = SendTrailingState::kCancelled; - send_trailing_metadata_batch_.CancelWith(error, flusher); - } else { - send_trailing_state_ = SendTrailingState::kCancelled; + switch (send_trailing_state_) { + case SendTrailingState::kInitial: + case SendTrailingState::kForwarded: + send_trailing_state_ = SendTrailingState::kCancelled; + if (!error.ok()) { + auto* batch = grpc_make_transport_stream_op( + NewClosure([call_combiner = call_combiner()](absl::Status) { + GRPC_CALL_COMBINER_STOP(call_combiner, "done-cancel"); + })); + batch->cancel_stream = true; + batch->payload->cancel_stream.cancel_error = error; + flusher->Resume(batch); + } + break; + case SendTrailingState::kQueued: + send_trailing_state_ = SendTrailingState::kCancelled; + send_trailing_metadata_batch_.CancelWith(error, flusher); + break; + case SendTrailingState::kQueuedBehindSendMessage: + case SendTrailingState::kQueuedButHaventClosedSends: + case SendTrailingState::kCancelled: + send_trailing_state_ = SendTrailingState::kCancelled; + break; } if (send_initial_metadata_ != nullptr) { switch (send_initial_metadata_->state) { case SendInitialMetadata::kInitial: - case SendInitialMetadata::kGotLatch: + case SendInitialMetadata::kGotPipe: case SendInitialMetadata::kForwarded: case SendInitialMetadata::kCancelled: break; - case SendInitialMetadata::kQueuedWaitingForLatch: - case SendInitialMetadata::kQueuedAndGotLatch: - case SendInitialMetadata::kQueuedAndSetLatch: + case SendInitialMetadata::kQueuedWaitingForPipe: + case SendInitialMetadata::kQueuedAndGotPipe: + case SendInitialMetadata::kQueuedAndPushedToPipe: send_initial_metadata_->batch.CancelWith(error, flusher); break; } @@ -2019,19 +2171,19 @@ ArenaPromise ServerCallData::MakeNextPromise( call_args.server_initial_metadata; switch (send_initial_metadata_->state) { case SendInitialMetadata::kInitial: - send_initial_metadata_->state = SendInitialMetadata::kGotLatch; + send_initial_metadata_->state = SendInitialMetadata::kGotPipe; break; - case SendInitialMetadata::kGotLatch: - case SendInitialMetadata::kQueuedAndGotLatch: - case SendInitialMetadata::kQueuedAndSetLatch: + case SendInitialMetadata::kGotPipe: + case SendInitialMetadata::kQueuedAndGotPipe: + case SendInitialMetadata::kQueuedAndPushedToPipe: case SendInitialMetadata::kForwarded: Crash(absl::StrFormat( "ILLEGAL STATE: %s", SendInitialMetadata::StateString( send_initial_metadata_->state))); // not reachable break; - case SendInitialMetadata::kQueuedWaitingForLatch: - send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotLatch; + case SendInitialMetadata::kQueuedWaitingForPipe: + send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotPipe; break; case SendInitialMetadata::kCancelled: break; @@ -2040,14 +2192,14 @@ ArenaPromise ServerCallData::MakeNextPromise( GPR_ASSERT(call_args.server_initial_metadata == nullptr); } if (send_message() != nullptr) { - send_message()->GotPipe(call_args.outgoing_messages); + send_message()->GotPipe(call_args.server_to_client_messages); } else { - GPR_ASSERT(call_args.outgoing_messages == nullptr); + GPR_ASSERT(call_args.server_to_client_messages == nullptr); } if (receive_message() != nullptr) { - receive_message()->GotPipe(call_args.incoming_messages); + receive_message()->GotPipe(call_args.client_to_server_messages); } else { - GPR_ASSERT(call_args.incoming_messages == nullptr); + GPR_ASSERT(call_args.client_to_server_messages == nullptr); } return ArenaPromise( [this]() { return PollTrailingMetadata(); }); @@ -2057,9 +2209,14 @@ ArenaPromise ServerCallData::MakeNextPromise( // All polls: await sending the trailing metadata, then foward it down the // stack. Poll ServerCallData::PollTrailingMetadata() { + if (grpc_trace_channel.enabled()) { + gpr_log(GPR_INFO, "%s PollTrailingMetadata: %s", LogTag().c_str(), + StateString(send_trailing_state_)); + } switch (send_trailing_state_) { case SendTrailingState::kInitial: case SendTrailingState::kQueuedBehindSendMessage: + case SendTrailingState::kQueuedButHaventClosedSends: return Pending{}; case SendTrailingState::kQueued: return WrapMetadata(send_trailing_metadata_batch_->payload @@ -2125,8 +2282,15 @@ void ServerCallData::RecvInitialMetadataReady(grpc_error_handle error) { FakeActivity().Run([this, filter] { promise_ = filter->MakeCallPromise( CallArgs{WrapMetadata(recv_initial_metadata_), - server_initial_metadata_latch(), outgoing_messages_pipe(), - incoming_messages_pipe()}, + server_initial_metadata_pipe() == nullptr + ? nullptr + : &server_initial_metadata_pipe()->sender, + receive_message() == nullptr + ? nullptr + : receive_message()->interceptor()->original_receiver(), + send_message() == nullptr + ? nullptr + : send_message()->interceptor()->original_sender()}, [this](CallArgs call_args) { return MakeNextPromise(std::move(call_args)); }); @@ -2168,46 +2332,110 @@ void ServerCallData::WakeInsideCombiner(Flusher* flusher) { gpr_log(GPR_INFO, "%s: WakeInsideCombiner %s", LogTag().c_str(), DebugString().c_str()); } - if (send_initial_metadata_ != nullptr && - send_initial_metadata_->state == - SendInitialMetadata::kQueuedAndGotLatch) { - send_initial_metadata_->state = SendInitialMetadata::kQueuedAndSetLatch; - send_initial_metadata_->server_initial_metadata_publisher->Set( - send_initial_metadata_->batch->payload->send_initial_metadata - .send_initial_metadata); - } poll_ctx.ClearRepoll(); + if (send_initial_metadata_ != nullptr) { + if (send_initial_metadata_->state == + SendInitialMetadata::kQueuedAndGotPipe) { + send_initial_metadata_->state = + SendInitialMetadata::kQueuedAndPushedToPipe; + GPR_ASSERT(!send_initial_metadata_->metadata_push_.has_value()); + GPR_ASSERT(!send_initial_metadata_->metadata_next_.has_value()); + send_initial_metadata_->metadata_push_.emplace( + send_initial_metadata_->server_initial_metadata_publisher->Push( + ServerMetadataHandle( + send_initial_metadata_->batch->payload->send_initial_metadata + .send_initial_metadata, + Arena::PooledDeleter(nullptr)))); + send_initial_metadata_->metadata_next_.emplace( + server_initial_metadata_pipe()->receiver.Next()); + } + if (send_initial_metadata_->metadata_push_.has_value()) { + if (!absl::holds_alternative( + (*send_initial_metadata_->metadata_push_)())) { + if (grpc_trace_channel.enabled()) { + gpr_log(GPR_INFO, "%s: WakeInsideCombiner: metadata_push done", + LogTag().c_str()); + } + send_initial_metadata_->metadata_push_.reset(); + } else if (grpc_trace_channel.enabled()) { + gpr_log(GPR_INFO, "%s: WakeInsideCombiner: metadata_push pending", + LogTag().c_str()); + } + } + } if (send_message() != nullptr) { - send_message()->WakeInsideCombiner(flusher); + if (send_trailing_state_ == + SendTrailingState::kQueuedButHaventClosedSends) { + send_trailing_state_ = SendTrailingState::kQueued; + send_message()->Done(*send_trailing_metadata_batch_->payload + ->send_trailing_metadata.send_trailing_metadata, + flusher); + } + send_message()->WakeInsideCombiner( + flusher, + send_initial_metadata_ == nullptr || + send_initial_metadata_->state == SendInitialMetadata::kForwarded); + if (grpc_trace_channel.enabled()) { + gpr_log(GPR_DEBUG, + "%s: After send_message WakeInsideCombiner %s is_idle=%s " + "is_forwarded=%s", + LogTag().c_str(), DebugString().c_str(), + send_message()->IsIdle() ? "true" : "false", + send_message()->IsForwarded() ? "true" : "false"); + } if (send_trailing_state_ == SendTrailingState::kQueuedBehindSendMessage && - send_message()->IsIdle()) { + (send_message()->IsIdle() || + (send_trailing_metadata_batch_->send_message && + send_message()->IsForwarded()))) { send_trailing_state_ = SendTrailingState::kQueued; + send_message()->Done(*send_trailing_metadata_batch_->payload + ->send_trailing_metadata.send_trailing_metadata, + flusher); } } if (receive_message() != nullptr) { - receive_message()->WakeInsideCombiner(flusher); + receive_message()->WakeInsideCombiner(flusher, true); } if (promise_.has_value()) { Poll poll; poll = promise_(); if (grpc_trace_channel.enabled()) { - gpr_log(GPR_INFO, "%s: WakeInsideCombiner poll=%s", LogTag().c_str(), - PollToString(poll, [](const ServerMetadataHandle& h) { - return h->DebugString(); - }).c_str()); + gpr_log( + GPR_INFO, + "%s: WakeInsideCombiner poll=%s; send_initial_metadata=%s " + "send_trailing_metadata=%s", + LogTag().c_str(), + PollToString( + poll, + [](const ServerMetadataHandle& h) { return h->DebugString(); }) + .c_str(), + send_initial_metadata_ == nullptr + ? "null" + : SendInitialMetadata::StateString(send_initial_metadata_->state), + StateString(send_trailing_state_)); } if (send_initial_metadata_ != nullptr && send_initial_metadata_->state == - SendInitialMetadata::kQueuedAndSetLatch) { - Poll p = server_initial_metadata_latch()->Wait()(); - if (ServerMetadata*** ppp = absl::get_if(&p)) { - ServerMetadata* md = **ppp; + SendInitialMetadata::kQueuedAndPushedToPipe) { + GPR_ASSERT(send_initial_metadata_->metadata_next_.has_value()); + auto p = (*send_initial_metadata_->metadata_next_)(); + if (grpc_trace_channel.enabled()) { + gpr_log(GPR_INFO, + "%s: WakeInsideCombiner send_initial_metadata poll=%s", + LogTag().c_str(), + PollToString(p, [](const NextResult& h) { + return (*h)->DebugString(); + }).c_str()); + } + if (auto* nr = absl::get_if>(&p)) { + ServerMetadataHandle md = std::move(nr->value()); if (send_initial_metadata_->batch->payload->send_initial_metadata - .send_initial_metadata != md) { + .send_initial_metadata != md.get()) { *send_initial_metadata_->batch->payload->send_initial_metadata .send_initial_metadata = std::move(*md); } send_initial_metadata_->state = SendInitialMetadata::kForwarded; + poll_ctx.Repoll(); send_initial_metadata_->batch.ResumeWith(flusher); } } @@ -2223,6 +2451,7 @@ void ServerCallData::WakeInsideCombiner(Flusher* flusher) { } switch (send_trailing_state_) { case SendTrailingState::kQueuedBehindSendMessage: + case SendTrailingState::kQueuedButHaventClosedSends: case SendTrailingState::kQueued: { if (send_trailing_metadata_batch_->payload->send_trailing_metadata .send_trailing_metadata != md) { diff --git a/src/core/lib/channel/promise_based_filter.h b/src/core/lib/channel/promise_based_filter.h index 57fa2608a87..0c604390b7b 100644 --- a/src/core/lib/channel/promise_based_filter.h +++ b/src/core/lib/channel/promise_based_filter.h @@ -19,15 +19,13 @@ // promise-style. Most of this will be removed once the promises conversion is // completed. -// TODO(ctiller): When removing this file, also reduce the number of *'s on the -// server initial metadata latch. - #include #include #include #include +#include #include #include #include @@ -35,6 +33,7 @@ #include #include "absl/container/inlined_vector.h" +#include "absl/functional/function_ref.h" #include "absl/meta/type_traits.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" @@ -60,7 +59,6 @@ #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/latch.h" #include "src/core/lib/promise/pipe.h" #include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/arena.h" @@ -152,9 +150,27 @@ class InvalidChannelFilter : public ChannelFilter { // Call data shared between all implementations of promise-based filters. class BaseCallData : public Activity, private Wakeable { - public: + protected: + // Hook to allow interception of messages on the send/receive path by + // PipeSender and PipeReceiver, as appropriate according to whether we're + // client or server. + class Interceptor { + public: + virtual PipeSender* Push() = 0; + virtual PipeReceiver* Pull() = 0; + virtual PipeReceiver* original_receiver() = 0; + virtual PipeSender* original_sender() = 0; + virtual void GotPipe(PipeSender*) = 0; + virtual void GotPipe(PipeReceiver*) = 0; + virtual ~Interceptor() = default; + }; + BaseCallData(grpc_call_element* elem, const grpc_call_element_args* args, - uint8_t flags); + uint8_t flags, + absl::FunctionRef make_send_interceptor, + absl::FunctionRef make_recv_interceptor); + + public: ~BaseCallData() override; void set_pollent(grpc_polling_entity* pollent) { @@ -270,6 +286,54 @@ class BaseCallData : public Activity, private Wakeable { return p.release(); } + class ReceiveInterceptor final : public Interceptor { + public: + explicit ReceiveInterceptor(Arena* arena) : pipe_{arena} {} + + PipeReceiver* original_receiver() override { + return &pipe_.receiver; + } + PipeSender* original_sender() override { abort(); } + + void GotPipe(PipeReceiver* receiver) override { + GPR_ASSERT(receiver_ == nullptr); + receiver_ = receiver; + } + + void GotPipe(PipeSender*) override { abort(); } + + PipeSender* Push() override { return &pipe_.sender; } + PipeReceiver* Pull() override { return receiver_; } + + private: + Pipe pipe_; + PipeReceiver* receiver_ = nullptr; + }; + + class SendInterceptor final : public Interceptor { + public: + explicit SendInterceptor(Arena* arena) : pipe_{arena} {} + + PipeReceiver* original_receiver() override { abort(); } + PipeSender* original_sender() override { + return &pipe_.sender; + } + + void GotPipe(PipeReceiver*) override { abort(); } + + void GotPipe(PipeSender* sender) override { + GPR_ASSERT(sender_ == nullptr); + sender_ = sender; + } + + PipeSender* Push() override { return sender_; } + PipeReceiver* Pull() override { return &pipe_.receiver; } + + private: + Pipe pipe_; + PipeSender* sender_ = nullptr; + }; + // State machine for sending messages: handles intercepting send_message ops // and forwarding them through pipes to the promise, then getting the result // down the stack. @@ -277,9 +341,11 @@ class BaseCallData : public Activity, private Wakeable { // these members for filters that don't need to intercept sent messages. class SendMessage { public: - explicit SendMessage(BaseCallData* base) - : base_(base), pipe_(base->arena()) {} - PipeReceiver* outgoing_pipe() { return &pipe_.receiver; } + SendMessage(BaseCallData* base, Interceptor* interceptor) + : base_(base), interceptor_(interceptor) {} + ~SendMessage() { interceptor_->~Interceptor(); } + + Interceptor* interceptor() { return interceptor_; } // Start a send_message op. void StartOp(CapturedBatch batch); @@ -287,16 +353,19 @@ class BaseCallData : public Activity, private Wakeable { // This happens when the promise requests to call the next filter: until // this occurs messages can't be sent as we don't know the pipe that the // promise expects to send on. - void GotPipe(PipeReceiver* receiver); + template + void GotPipe(T* pipe); // Called from client/server polling to do the send message part of the // work. - void WakeInsideCombiner(Flusher* flusher); + void WakeInsideCombiner(Flusher* flusher, bool allow_push_to_pipe); // Call is completed, we have trailing metadata. Close things out. void Done(const ServerMetadata& metadata, Flusher* flusher); // Return true if we have a batch captured (for debug logs) bool HaveCapturedBatch() const { return batch_.is_captured(); } // Return true if we're not actively sending a message. bool IsIdle() const; + // Return true if we've released the message for forwarding down the stack. + bool IsForwarded() const { return state_ == State::kForwardedBatch; } private: enum class State : uint8_t { @@ -331,11 +400,9 @@ class BaseCallData : public Activity, private Wakeable { BaseCallData* const base_; State state_ = State::kInitial; - Pipe pipe_; - PipeReceiver* receiver_ = nullptr; + Interceptor* const interceptor_; absl::optional::PushType> push_; - absl::optional::NextType> next_; - absl::optional> next_result_; + absl::optional> next_; CapturedBatch batch_; grpc_closure* intercepted_on_complete_; grpc_closure on_complete_ = @@ -351,9 +418,11 @@ class BaseCallData : public Activity, private Wakeable { // these members for filters that don't need to intercept sent messages. class ReceiveMessage { public: - explicit ReceiveMessage(BaseCallData* base) - : base_(base), pipe_(base->arena()) {} - PipeSender* incoming_pipe() { return &pipe_.sender; } + ReceiveMessage(BaseCallData* base, Interceptor* interceptor) + : base_(base), interceptor_(interceptor) {} + ~ReceiveMessage() { interceptor_->~Interceptor(); } + + Interceptor* interceptor() { return interceptor_; } // Start a recv_message op. void StartOp(CapturedBatch& batch); @@ -361,10 +430,11 @@ class BaseCallData : public Activity, private Wakeable { // This happens when the promise requests to call the next filter: until // this occurs messages can't be received as we don't know the pipe that the // promise expects to forward them with. - void GotPipe(PipeSender* sender); + template + void GotPipe(T* pipe); // Called from client/server polling to do the receive message part of the // work. - void WakeInsideCombiner(Flusher* flusher); + void WakeInsideCombiner(Flusher* flusher, bool allow_push_to_pipe); // Call is completed, we have trailing metadata. Close things out. void Done(const ServerMetadata& metadata, Flusher* flusher); @@ -408,23 +478,28 @@ class BaseCallData : public Activity, private Wakeable { // On the next poll we'll close things out and forward on completions, // then transition to cancelled. kBatchCompletedButCancelled, - // Completed successfully while we're processing a recv message. + // Completed successfully while we're processing a recv message - see + // kPushedToPipe. kCompletedWhilePushedToPipe, + // Completed successfully while we're processing a recv message - see + // kPulledFromPipe. kCompletedWhilePulledFromPipe, + // Completed successfully while we were waiting to process + // kBatchCompleted. + kCompletedWhileBatchCompleted, }; static const char* StateString(State); void OnComplete(absl::Status status); BaseCallData* const base_; - Pipe pipe_; - PipeSender* sender_; + Interceptor* const interceptor_; State state_ = State::kInitial; uint32_t scratch_flags_; absl::optional* intercepted_slice_buffer_; uint32_t* intercepted_flags_; absl::optional::PushType> push_; - absl::optional::NextType> next_; + absl::optional> next_; absl::Status completed_status_; grpc_closure* intercepted_on_complete_; grpc_closure on_complete_ = @@ -436,15 +511,8 @@ class BaseCallData : public Activity, private Wakeable { CallCombiner* call_combiner() const { return call_combiner_; } Timestamp deadline() const { return deadline_; } grpc_call_stack* call_stack() const { return call_stack_; } - Latch* server_initial_metadata_latch() const { - return server_initial_metadata_latch_; - } - PipeReceiver* outgoing_messages_pipe() const { - return send_message_ == nullptr ? nullptr : send_message_->outgoing_pipe(); - } - PipeSender* incoming_messages_pipe() const { - return receive_message_ == nullptr ? nullptr - : receive_message_->incoming_pipe(); + Pipe* server_initial_metadata_pipe() const { + return server_initial_metadata_pipe_; } SendMessage* send_message() const { return send_message_; } ReceiveMessage* receive_message() const { return receive_message_; } @@ -474,7 +542,7 @@ class BaseCallData : public Activity, private Wakeable { CallFinalization finalization_; grpc_call_context_element* const context_; std::atomic pollent_{nullptr}; - Latch* const server_initial_metadata_latch_; + Pipe* const server_initial_metadata_pipe_; SendMessage* const send_message_; ReceiveMessage* const receive_message_; grpc_event_engine::experimental::EventEngine* event_engine_; @@ -626,6 +694,9 @@ class ServerCallData : public BaseCallData { // metadata. kQueuedBehindSendMessage, // We saw the op, and are waiting for the promise to complete + // to forward it. First however we need to close sends. + kQueuedButHaventClosedSends, + // We saw the op, and are waiting for the promise to complete // to forward it. kQueued, // We've forwarded the op to the next filter. diff --git a/src/core/lib/event_engine/slice.cc b/src/core/lib/event_engine/slice.cc index bcb09c50345..b9266714bb2 100644 --- a/src/core/lib/event_engine/slice.cc +++ b/src/core/lib/event_engine/slice.cc @@ -93,7 +93,7 @@ Slice Slice::FromRefcountAndBytes(grpc_slice_refcount* r, const uint8_t* begin, const uint8_t* end) { grpc_slice out; out.refcount = r; - if (r != grpc_slice_refcount::NoopRefcount()) r->Ref(); + if (r != grpc_slice_refcount::NoopRefcount()) r->Ref({}); out.data.refcounted.bytes = const_cast(begin); out.data.refcounted.length = end - begin; return Slice(out); diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc index b860e1f1c7e..093312e8878 100644 --- a/src/core/lib/experiments/experiments.cc +++ b/src/core/lib/experiments/experiments.cc @@ -48,6 +48,9 @@ const char* const description_promise_based_client_call = "(ie when all filters in a stack are promise based)"; const char* const description_free_large_allocator = "If set, return all free bytes from a \042big\042 allocator"; +const char* const description_promise_based_server_call = + "If set, use the new gRPC promise based call code when it's appropriate " + "(ie when all filters in a stack are promise based)"; const char* const description_transport_supplies_client_latency = "If set, use the transport represented value for client latency in " "opencensus"; @@ -71,6 +74,7 @@ const ExperimentMetadata g_experiment_metadata[] = { {"monitoring_experiment", description_monitoring_experiment, true}, {"promise_based_client_call", description_promise_based_client_call, false}, {"free_large_allocator", description_free_large_allocator, false}, + {"promise_based_server_call", description_promise_based_server_call, false}, {"transport_supplies_client_latency", description_transport_supplies_client_latency, false}, {"event_engine_listener", description_event_engine_listener, false}, diff --git a/src/core/lib/experiments/experiments.h b/src/core/lib/experiments/experiments.h index 5e8adedf3fa..b3a56f6fab8 100644 --- a/src/core/lib/experiments/experiments.h +++ b/src/core/lib/experiments/experiments.h @@ -40,10 +40,13 @@ inline bool IsEventEngineClientEnabled() { return IsExperimentEnabled(7); } inline bool IsMonitoringExperimentEnabled() { return IsExperimentEnabled(8); } inline bool IsPromiseBasedClientCallEnabled() { return IsExperimentEnabled(9); } inline bool IsFreeLargeAllocatorEnabled() { return IsExperimentEnabled(10); } -inline bool IsTransportSuppliesClientLatencyEnabled() { +inline bool IsPromiseBasedServerCallEnabled() { return IsExperimentEnabled(11); } -inline bool IsEventEngineListenerEnabled() { return IsExperimentEnabled(12); } +inline bool IsTransportSuppliesClientLatencyEnabled() { + return IsExperimentEnabled(12); +} +inline bool IsEventEngineListenerEnabled() { return IsExperimentEnabled(13); } struct ExperimentMetadata { const char* name; @@ -51,7 +54,7 @@ struct ExperimentMetadata { bool default_value; }; -constexpr const size_t kNumExperiments = 13; +constexpr const size_t kNumExperiments = 14; extern const ExperimentMetadata g_experiment_metadata[kNumExperiments]; } // namespace grpc_core diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index 8f917b5439e..3bd0827f7dc 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -119,6 +119,14 @@ expiry: 2023/04/01 owner: alishananda@google.com test_tags: [resource_quota_test] +- name: promise_based_server_call + description: + If set, use the new gRPC promise based call code when it's appropriate + (ie when all filters in a stack are promise based) + default: false + expiry: 2023/06/01 + owner: ctiller@google.com + test_tags: ["core_end2end_test"] - name: transport_supplies_client_latency description: If set, use the transport represented value for client latency in opencensus diff --git a/src/core/lib/gprpp/debug_location.h b/src/core/lib/gprpp/debug_location.h index 1b75086f4ec..1a29f495383 100644 --- a/src/core/lib/gprpp/debug_location.h +++ b/src/core/lib/gprpp/debug_location.h @@ -56,9 +56,6 @@ class SourceLocation { // No-op for non-debug builds. // Callers can use the DEBUG_LOCATION macro in either case. #ifndef NDEBUG -// TODO(roth): See if there's a way to automatically populate this, -// similarly to how absl::SourceLocation::current() works, so that -// callers don't need to explicitly pass DEBUG_LOCATION anywhere. class DebugLocation { public: DebugLocation(const char* file = GRPC_DEFAULT_FILE, diff --git a/src/core/lib/iomgr/call_combiner.cc b/src/core/lib/iomgr/call_combiner.cc index 2b2996c2c4f..c48766f781a 100644 --- a/src/core/lib/iomgr/call_combiner.cc +++ b/src/core/lib/iomgr/call_combiner.cc @@ -116,9 +116,9 @@ void CallCombiner::Start(grpc_closure* closure, grpc_error_handle error, DEBUG_ARGS const char* reason) { if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) { gpr_log(GPR_INFO, - "==> CallCombiner::Start() [%p] closure=%p [" DEBUG_FMT_STR + "==> CallCombiner::Start() [%p] closure=%s [" DEBUG_FMT_STR "%s] error=%s", - this, closure DEBUG_FMT_ARGS, reason, + this, closure->DebugString().c_str() DEBUG_FMT_ARGS, reason, StatusToString(error).c_str()); } size_t prev_size = @@ -176,8 +176,8 @@ void CallCombiner::Stop(DEBUG_ARGS const char* reason) { internal::StatusMoveFromHeapPtr(closure->error_data.error); closure->error_data.error = 0; if (GRPC_TRACE_FLAG_ENABLED(grpc_call_combiner_trace)) { - gpr_log(GPR_INFO, " EXECUTING FROM QUEUE: closure=%p error=%s", - closure, StatusToString(error).c_str()); + gpr_log(GPR_INFO, " EXECUTING FROM QUEUE: closure=%s error=%s", + closure->DebugString().c_str(), StatusToString(error).c_str()); } ScheduleClosure(closure, error); break; diff --git a/src/core/lib/iomgr/closure.cc b/src/core/lib/iomgr/closure.cc new file mode 100644 index 00000000000..cf1a184032c --- /dev/null +++ b/src/core/lib/iomgr/closure.cc @@ -0,0 +1,27 @@ +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "src/core/lib/iomgr/closure.h" + +#include "absl/strings/str_format.h" + +std::string grpc_closure::DebugString() const { +#ifdef NDEBUG + return absl::StrFormat("%p", this); +#else + return absl::StrFormat("%p|created=%s:%d", this, file_created, line_created); +#endif +} diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h index 9996f163bcd..adf8ebf2b74 100644 --- a/src/core/lib/iomgr/closure.h +++ b/src/core/lib/iomgr/closure.h @@ -86,6 +86,8 @@ struct grpc_closure { const char* file_initiated; int line_initiated; #endif + + std::string DebugString() const; }; #ifndef NDEBUG diff --git a/src/core/lib/promise/activity.h b/src/core/lib/promise/activity.h index 771dcd39c23..73e9bfd3574 100644 --- a/src/core/lib/promise/activity.h +++ b/src/core/lib/promise/activity.h @@ -102,6 +102,10 @@ class Waker { return wakeable_ == other.wakeable_; } + bool operator!=(const Waker& other) const noexcept { + return wakeable_ != other.wakeable_; + } + std::string ActivityDebugTag() { return wakeable_ == nullptr ? "" : wakeable_->ActivityDebugTag(); } diff --git a/src/core/lib/promise/for_each.h b/src/core/lib/promise/for_each.h index 8b3f2ff82b2..cee3a730b30 100644 --- a/src/core/lib/promise/for_each.h +++ b/src/core/lib/promise/for_each.h @@ -17,17 +17,24 @@ #include +#include + +#include #include #include #include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "absl/types/variant.h" #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/construct_destruct.h" +#include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/detail/promise_factory.h" #include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/trace.h" namespace grpc_core { @@ -101,9 +108,21 @@ class ForEach { ReaderResult result; }; + std::string DebugTag() { + return absl::StrCat(Activity::current()->DebugTag(), " FOR_EACH[0x", + reinterpret_cast(this), "]: "); + } + Poll PollReaderNext() { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, "%s PollReaderNext", DebugTag().c_str()); + } auto r = reader_next_(); if (auto* p = absl::get_if(&r)) { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, "%s PollReaderNext: got has_value=%s", + DebugTag().c_str(), p->has_value() ? "true" : "false"); + } if (p->has_value()) { Destruct(&reader_next_); auto action = action_factory_.Make(std::move(**p)); @@ -118,6 +137,9 @@ class ForEach { } Poll PollAction() { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, "%s PollAction", DebugTag().c_str()); + } auto r = in_action_.promise(); if (auto* p = absl::get_if(&r)) { if (p->ok()) { diff --git a/src/core/lib/promise/if.h b/src/core/lib/promise/if.h index 09a62a8aea0..20b83685315 100644 --- a/src/core/lib/promise/if.h +++ b/src/core/lib/promise/if.h @@ -22,6 +22,7 @@ #include "absl/status/statusor.h" #include "absl/types/variant.h" +#include "src/core/lib/gprpp/construct_destruct.h" #include "src/core/lib/promise/detail/promise_factory.h" #include "src/core/lib/promise/detail/promise_like.h" #include "src/core/lib/promise/poll.h" @@ -116,6 +117,66 @@ class If { }; }; +template +class If { + private: + using TrueFactory = promise_detail::OncePromiseFactory; + using FalseFactory = promise_detail::OncePromiseFactory; + using TruePromise = typename TrueFactory::Promise; + using FalsePromise = typename FalseFactory::Promise; + using Result = + typename PollTraits()())>::Type; + + public: + If(bool condition, T if_true, F if_false) : condition_(condition) { + TrueFactory true_factory(std::move(if_true)); + FalseFactory false_factory(std::move(if_false)); + if (condition_) { + Construct(&if_true_, true_factory.Make()); + } else { + Construct(&if_false_, false_factory.Make()); + } + } + ~If() { + if (condition_) { + Destruct(&if_true_); + } else { + Destruct(&if_false_); + } + } + + If(const If&) = delete; + If& operator=(const If&) = delete; + If(If&& other) noexcept : condition_(other.condition_) { + if (condition_) { + Construct(&if_true_, std::move(other.if_true_)); + } else { + Construct(&if_false_, std::move(other.if_false_)); + } + } + If& operator=(If&& other) noexcept { + if (&other == this) return *this; + Destruct(this); + Construct(this, std::move(other)); + return *this; + } + + Poll operator()() { + if (condition_) { + return if_true_(); + } else { + return if_false_(); + } + } + + private: + bool condition_; + union { + TruePromise if_true_; + FalsePromise if_false_; + }; +}; + } // namespace promise_detail // If promise combinator. diff --git a/src/core/lib/promise/interceptor_list.h b/src/core/lib/promise/interceptor_list.h new file mode 100644 index 00000000000..2e09f574ae5 --- /dev/null +++ b/src/core/lib/promise/interceptor_list.h @@ -0,0 +1,309 @@ +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_SRC_CORE_LIB_PROMISE_INTERCEPTOR_LIST_H +#define GRPC_SRC_CORE_LIB_PROMISE_INTERCEPTOR_LIST_H + +#include + +#include + +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/types/optional.h" +#include "absl/types/variant.h" + +#include + +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/construct_destruct.h" +#include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/promise/context.h" +#include "src/core/lib/promise/detail/promise_factory.h" +#include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/trace.h" +#include "src/core/lib/resource_quota/arena.h" + +namespace grpc_core { + +// Tracks a list of maps of T -> optional via promises. +// When Run, runs each transformation in order, and resolves to the ulimate +// result. +// If a map resolves to nullopt, the chain is terminated and the result is +// nullopt. +// Maps can also have synchronous cleanup functions, which are guaranteed to be +// called at the termination of each run through the chain. +template +class InterceptorList { + private: + // A map of T -> T via promises. + class Map { + public: + explicit Map(DebugLocation from) : from_(from) {} + // Construct a promise to transform x into some other value at memory. + virtual void MakePromise(T x, void* memory) = 0; + // Destroy a promise constructed at memory. + virtual void Destroy(void* memory) = 0; + // Poll a promise constructed at memory. + // Resolves to an optional -- if nullopt it means terminate the chain and + // resolve. + virtual Poll> PollOnce(void* memory) = 0; + virtual ~Map() = default; + + // Update the next pointer stored with this map. + // This is only valid to call once, and only before the map is used. + void SetNext(Map* next) { + GPR_DEBUG_ASSERT(next_ == nullptr); + next_ = next; + } + + // Access the creation location for this map (for debug tracing). + DebugLocation from() const { return from_; } + + // Access the next map in the chain (or nullptr if this is the last map). + Map* next() const { return next_; } + + private: + GPR_NO_UNIQUE_ADDRESS const DebugLocation from_; + Map* next_ = nullptr; + }; + + public: + // The result of Run: a promise that will execute the entire chain. + class RunPromise { + public: + RunPromise(size_t memory_required, Map* factory, absl::optional value) { + if (!value.has_value() || factory == nullptr) { + is_immediately_resolved_ = true; + Construct(&result_, std::move(value)); + } else { + is_immediately_resolved_ = false; + Construct(&async_resolution_, memory_required); + factory->MakePromise(std::move(*value), async_resolution_.space.get()); + async_resolution_.current_factory = factory; + } + } + + ~RunPromise() { + if (is_immediately_resolved_) { + Destruct(&result_); + } else { + if (async_resolution_.current_factory != nullptr) { + async_resolution_.current_factory->Destroy( + async_resolution_.space.get()); + } + Destruct(&async_resolution_); + } + } + + RunPromise(const RunPromise&) = delete; + RunPromise& operator=(const RunPromise&) = delete; + + RunPromise(RunPromise&& other) noexcept + : is_immediately_resolved_(other.is_immediately_resolved_) { + if (is_immediately_resolved_) { + Construct(&result_, std::move(other.result_)); + } else { + Construct(&async_resolution_, std::move(other.async_resolution_)); + } + } + + RunPromise& operator=(RunPromise&& other) noexcept = delete; + + Poll> operator()() { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, "InterceptorList::RunPromise: %s", + DebugString().c_str()); + } + if (is_immediately_resolved_) return std::move(result_); + while (true) { + auto r = async_resolution_.current_factory->PollOnce( + async_resolution_.space.get()); + if (auto* p = absl::get_if(&r)) { + async_resolution_.current_factory->Destroy( + async_resolution_.space.get()); + async_resolution_.current_factory = + async_resolution_.current_factory->next(); + if (async_resolution_.current_factory == nullptr || !p->has_value()) { + return std::move(*p); + } + async_resolution_.current_factory->MakePromise( + std::move(**p), async_resolution_.space.get()); + continue; + } + return Pending{}; + } + } + + private: + std::string DebugString() const { + if (is_immediately_resolved_) { + return absl::StrFormat("Result:has_value:%d", result_.has_value()); + } else { + return absl::StrCat( + "Running:", + async_resolution_.current_factory == nullptr + ? "END" + : ([p = async_resolution_.current_factory->from()]() { + return absl::StrCat(p.file(), ":", p.line()); + })() + .c_str()); + } + } + struct AsyncResolution { + explicit AsyncResolution(size_t max_size) + : space(GetContext()->MakePooledArray(max_size)) {} + AsyncResolution(const AsyncResolution&) = delete; + AsyncResolution& operator=(const AsyncResolution&) = delete; + AsyncResolution(AsyncResolution&& other) noexcept + : current_factory(std::exchange(other.current_factory, nullptr)), + space(std::move(other.space)) {} + Map* current_factory; + Arena::PoolPtr space; + }; + union { + AsyncResolution async_resolution_; + absl::optional result_; + }; + // If true, the result_ union is valid, otherwise async_resolution_ is. + // Indicates whether the promise resolved immediately at construction or if + // additional steps were needed. + bool is_immediately_resolved_; + }; + + InterceptorList() = default; + InterceptorList(const InterceptorList&) = delete; + InterceptorList& operator=(const InterceptorList&) = delete; + ~InterceptorList() { DeleteFactories(); } + + RunPromise Run(absl::optional initial_value) { + return RunPromise(promise_memory_required_, first_map_, + std::move(initial_value)); + } + + // Append a new map to the end of the chain. + template + void AppendMap(Fn fn, DebugLocation from) { + Append(MakeMapToAdd( + std::move(fn), [] {}, from)); + } + + // Prepend a new map to the beginning of the chain. + template + void PrependMap(Fn fn, DebugLocation from) { + Prepend(MakeMapToAdd( + std::move(fn), [] {}, from)); + } + + // Append a new map to the end of the chain, with a cleanup function to be + // called at the end of run promise execution. + template + void AppendMapWithCleanup(Fn fn, CleanupFn cleanup_fn, DebugLocation from) { + Append(MakeMapToAdd(std::move(fn), std::move(cleanup_fn), from)); + } + + // Prepend a new map to the beginning of the chain, with a cleanup function to + // be called at the end of run promise execution. + template + void PrependMapWithCleanup(Fn fn, CleanupFn cleanup_fn, DebugLocation from) { + Prepend(MakeMapToAdd(std::move(fn), std::move(cleanup_fn), from)); + } + + protected: + // Clear the interceptor list + void ResetInterceptorList() { + DeleteFactories(); + first_map_ = nullptr; + last_map_ = nullptr; + promise_memory_required_ = 0; + } + + private: + template + class MapImpl final : public Map { + public: + using PromiseFactory = promise_detail::RepeatedPromiseFactory; + using Promise = typename PromiseFactory::Promise; + + explicit MapImpl(Fn fn, CleanupFn cleanup_fn, DebugLocation from) + : Map(from), fn_(std::move(fn)), cleanup_fn_(std::move(cleanup_fn)) {} + ~MapImpl() override { cleanup_fn_(); } + void MakePromise(T x, void* memory) override { + new (memory) Promise(fn_.Make(std::move(x))); + } + void Destroy(void* memory) override { + static_cast(memory)->~Promise(); + } + Poll> PollOnce(void* memory) override { + return poll_cast>((*static_cast(memory))()); + } + + private: + GPR_NO_UNIQUE_ADDRESS PromiseFactory fn_; + GPR_NO_UNIQUE_ADDRESS CleanupFn cleanup_fn_; + }; + + template + Map* MakeMapToAdd(Fn fn, CleanupFn cleanup_fn, DebugLocation from) { + using FactoryType = MapImpl; + promise_memory_required_ = std::max(promise_memory_required_, + sizeof(typename FactoryType::Promise)); + return GetContext()->New(std::move(fn), + std::move(cleanup_fn), from); + } + + void Append(Map* f) { + if (first_map_ == nullptr) { + first_map_ = f; + last_map_ = f; + } else { + last_map_->SetNext(f); + last_map_ = f; + } + } + + void Prepend(Map* f) { + if (first_map_ == nullptr) { + first_map_ = f; + last_map_ = f; + } else { + f->SetNext(first_map_); + first_map_ = f; + } + } + + void DeleteFactories() { + for (auto* f = first_map_; f != nullptr;) { + auto* next = f->next(); + f->~Map(); + f = next; + } + } + + // The first map in the chain. + Map* first_map_ = nullptr; + // The last map in the chain. + Map* last_map_ = nullptr; + // The amount of memory required to store the largest promise in the chain. + size_t promise_memory_required_ = 0; +}; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_LIB_PROMISE_INTERCEPTOR_LIST_H diff --git a/src/core/lib/promise/intra_activity_waiter.h b/src/core/lib/promise/intra_activity_waiter.h index 73bc93f4100..736ec04ae7d 100644 --- a/src/core/lib/promise/intra_activity_waiter.h +++ b/src/core/lib/promise/intra_activity_waiter.h @@ -17,6 +17,8 @@ #include +#include + #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/poll.h" @@ -40,6 +42,10 @@ class IntraActivityWaiter { } } + std::string DebugString() const { + return waiting_ ? "WAITING" : "NOT_WAITING"; + } + private: bool waiting_ = false; }; diff --git a/src/core/lib/promise/latch.h b/src/core/lib/promise/latch.h index d66b461f46c..c1dca66efc1 100644 --- a/src/core/lib/promise/latch.h +++ b/src/core/lib/promise/latch.h @@ -17,12 +17,21 @@ #include +#include + +#include #include +#include + +#include "absl/strings/str_cat.h" #include +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/intra_activity_waiter.h" #include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/trace.h" namespace grpc_core { @@ -56,9 +65,13 @@ class Latch { #ifndef NDEBUG has_had_waiters_ = true; #endif - return [this]() -> Poll { + return [this]() -> Poll { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_INFO, "%sPollWait %s", DebugTag().c_str(), + StateString().c_str()); + } if (has_value_) { - return &value_; + return std::move(value_); } else { return waiter_.pending(); } @@ -67,6 +80,9 @@ class Latch { // Set the value of the latch. Can only be called once. void Set(T value) { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_INFO, "%sSet %s", DebugTag().c_str(), StateString().c_str()); + } GPR_DEBUG_ASSERT(!has_value_); value_ = std::move(value); has_value_ = true; @@ -74,6 +90,16 @@ class Latch { } private: + std::string DebugTag() { + return absl::StrCat(Activity::current()->DebugTag(), " LATCH[0x", + reinterpret_cast(this), "]: "); + } + + std::string StateString() { + return absl::StrCat("has_value:", has_value_ ? "true" : "false", + " waiter:", waiter_.DebugString()); + } + // The value stored (if has_value_ is true), otherwise some random value, we // don't care. // Why not absl::optional<>? Writing things this way lets us compress diff --git a/src/core/lib/promise/map_pipe.h b/src/core/lib/promise/map_pipe.h index 45df76e32d4..6f7312da047 100644 --- a/src/core/lib/promise/map_pipe.h +++ b/src/core/lib/promise/map_pipe.h @@ -19,10 +19,15 @@ #include "absl/status/status.h" +#include + +#include "src/core/lib/debug/trace.h" #include "src/core/lib/promise/detail/promise_factory.h" #include "src/core/lib/promise/for_each.h" #include "src/core/lib/promise/map.h" #include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/trace.h" #include "src/core/lib/promise/try_seq.h" namespace grpc_core { @@ -40,14 +45,25 @@ auto MapPipe(PipeReceiver src, PipeSender dst, Filter filter_factory) { [filter_factory = promise_detail::RepeatedPromiseFactory( std::move(filter_factory)), dst = std::move(dst)](T t) mutable { - return TrySeq(filter_factory.Make(std::move(t)), [&dst](T t) { - return Map(dst.Push(std::move(t)), [](bool successful_push) { - if (successful_push) { - return absl::OkStatus(); - } - return absl::CancelledError(); - }); - }); + return TrySeq( + [] { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, "MapPipe: start map"); + } + return Empty{}; + }, + filter_factory.Make(std::move(t)), + [&dst](T t) { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, "MapPipe: start push"); + } + return Map(dst.Push(std::move(t)), [](bool successful_push) { + if (successful_push) { + return absl::OkStatus(); + } + return absl::CancelledError(); + }); + }); }); } diff --git a/src/core/lib/promise/pipe.h b/src/core/lib/promise/pipe.h index a1f4ed42b56..8033201fac8 100644 --- a/src/core/lib/promise/pipe.h +++ b/src/core/lib/promise/pipe.h @@ -18,10 +18,15 @@ #include #include +#include +#include #include +#include #include +#include "absl/base/attributes.h" +#include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/types/optional.h" #include "absl/types/variant.h" @@ -29,14 +34,19 @@ #include #include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/context.h" +#include "src/core/lib/promise/if.h" +#include "src/core/lib/promise/interceptor_list.h" #include "src/core/lib/promise/intra_activity_waiter.h" +#include "src/core/lib/promise/map.h" #include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/seq.h" +#include "src/core/lib/promise/trace.h" #include "src/core/lib/resource_quota/arena.h" -extern grpc_core::DebugOnlyTraceFlag grpc_trace_promise_pipe; - namespace grpc_core { namespace pipe_detail { @@ -56,21 +66,24 @@ struct Pipe; template class NextResult final { public: - explicit NextResult(pipe_detail::Center* center) : center_(center) {} + NextResult() : center_(nullptr) {} + explicit NextResult(RefCountedPtr> center) + : center_(std::move(center)) { + GPR_ASSERT(center_ != nullptr); + } + explicit NextResult(bool cancelled) + : center_(nullptr), cancelled_(cancelled) {} ~NextResult(); NextResult(const NextResult&) = delete; NextResult& operator=(const NextResult&) = delete; - NextResult(NextResult&& other) noexcept - : center_(std::exchange(other.center_, nullptr)) {} - NextResult& operator=(NextResult&& other) noexcept { - center_ = std::exchange(other.center_, nullptr); - return *this; - } + NextResult(NextResult&& other) noexcept = default; + NextResult& operator=(NextResult&& other) noexcept = default; using value_type = T; void reset(); bool has_value() const; + // Only valid if has_value() const T& value() const { GPR_ASSERT(has_value()); return **this; @@ -81,9 +94,12 @@ class NextResult final { } const T& operator*() const; T& operator*(); + // Only valid if !has_value() + bool cancelled() { return cancelled_; } private: - pipe_detail::Center* center_; + RefCountedPtr> center_; + bool cancelled_; }; namespace pipe_detail { @@ -96,71 +112,39 @@ class Next; // Center sits between a sender and a receiver to provide a one-deep buffer of // Ts template -class Center { +class Center : public InterceptorList { public: // Initialize with one send ref (held by PipeSender) and one recv ref (held by // PipeReceiver) Center() { - send_refs_ = 1; - recv_refs_ = 1; + refs_ = 2; value_state_ = ValueState::kEmpty; } - // Add one ref to the send side of this object, and return this. - Center* RefSend() { - if (grpc_trace_promise_pipe.enabled()) { - gpr_log(GPR_INFO, "%s", DebugOpString("RefSend").c_str()); - } - send_refs_++; - GPR_ASSERT(send_refs_ != 0); - return this; - } - - // Add one ref to the recv side of this object, and return this. - Center* RefRecv() { - if (grpc_trace_promise_pipe.enabled()) { - gpr_log(GPR_INFO, "%s", DebugOpString("RefRecv").c_str()); + // Add one ref to this object, and return this. + void IncrementRefCount() { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, "%s", DebugOpString("IncrementRefCount").c_str()); } - recv_refs_++; - GPR_ASSERT(recv_refs_ != 0); - return this; + refs_++; + GPR_DEBUG_ASSERT(refs_ != 0); } - // Drop a send side ref - // If no send refs remain, wake due to send closure - // If no refs remain, destroy this object - void UnrefSend() { - if (grpc_trace_promise_pipe.enabled()) { - gpr_log(GPR_INFO, "%s", DebugOpString("UnrefSend").c_str()); - } - GPR_DEBUG_ASSERT(send_refs_ > 0); - send_refs_--; - if (0 == send_refs_) { - on_full_.Wake(); - on_empty_.Wake(); - if (0 == recv_refs_) { - this->~Center(); - } - } + RefCountedPtr
Ref() { + IncrementRefCount(); + return RefCountedPtr
(this); } - // Drop a recv side ref - // If no recv refs remain, wake due to recv closure + // Drop a ref // If no refs remain, destroy this object - void UnrefRecv() { - if (grpc_trace_promise_pipe.enabled()) { - gpr_log(GPR_INFO, "%s", DebugOpString("UnrefRecv").c_str()); + void Unref() { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, "%s", DebugOpString("Unref").c_str()); } - GPR_DEBUG_ASSERT(recv_refs_ > 0); - recv_refs_--; - if (0 == recv_refs_) { - on_full_.Wake(); - on_empty_.Wake(); - if (0 == send_refs_) { - this->~Center(); - } else if (value_state_ == ValueState::kReady) { - ResetValue(); - } + GPR_DEBUG_ASSERT(refs_ > 0); + refs_--; + if (0 == refs_) { + this->~Center(); } } @@ -169,26 +153,45 @@ class Center { // Return true if the value was pushed. // Return false if the recv end is closed. Poll Push(T* value) { - if (grpc_trace_promise_pipe.enabled()) { + if (grpc_trace_promise_primitives.enabled()) { gpr_log(GPR_INFO, "%s", DebugOpString("Push").c_str()); } - GPR_DEBUG_ASSERT(send_refs_ != 0); - if (recv_refs_ == 0) return false; - if (value_state_ != ValueState::kEmpty) return on_empty_.pending(); - value_state_ = ValueState::kReady; - value_ = std::move(*value); - on_full_.Wake(); - return true; + GPR_DEBUG_ASSERT(refs_ != 0); + switch (value_state_) { + case ValueState::kClosed: + case ValueState::kReadyClosed: + case ValueState::kCancelled: + return false; + case ValueState::kReady: + case ValueState::kAcked: + return on_empty_.pending(); + case ValueState::kEmpty: + value_state_ = ValueState::kReady; + value_ = std::move(*value); + on_full_.Wake(); + return true; + } + GPR_UNREACHABLE_CODE(return false); } Poll PollAck() { - if (grpc_trace_promise_pipe.enabled()) { + if (grpc_trace_promise_primitives.enabled()) { gpr_log(GPR_INFO, "%s", DebugOpString("PollAck").c_str()); } - GPR_DEBUG_ASSERT(send_refs_ != 0); - if (recv_refs_ == 0) return value_state_ == ValueState::kAcked; - if (value_state_ != ValueState::kAcked) return on_empty_.pending(); - value_state_ = ValueState::kEmpty; + GPR_DEBUG_ASSERT(refs_ != 0); + switch (value_state_) { + case ValueState::kClosed: + case ValueState::kReadyClosed: + case ValueState::kCancelled: + return false; + case ValueState::kReady: + case ValueState::kEmpty: + return on_empty_.pending(); + case ValueState::kAcked: + value_state_ = ValueState::kEmpty; + on_empty_.Wake(); + return true; + } return true; } @@ -196,47 +199,107 @@ class Center { // Return Pending if there is no value. // Return the value if one was retrieved. // Return nullopt if the send end is closed and no value had been pushed. - Poll> Next() { - if (grpc_trace_promise_pipe.enabled()) { + Poll> Next() { + if (grpc_trace_promise_primitives.enabled()) { gpr_log(GPR_INFO, "%s", DebugOpString("Next").c_str()); } - GPR_DEBUG_ASSERT(recv_refs_ != 0); - if (value_state_ != ValueState::kReady) { - if (send_refs_ == 0) return NextResult(nullptr); - return on_full_.pending(); + GPR_DEBUG_ASSERT(refs_ != 0); + switch (value_state_) { + case ValueState::kEmpty: + case ValueState::kAcked: + return on_full_.pending(); + case ValueState::kReadyClosed: + this->ResetInterceptorList(); + value_state_ = ValueState::kClosed; + ABSL_FALLTHROUGH_INTENDED; + case ValueState::kReady: + return std::move(value_); + case ValueState::kClosed: + case ValueState::kCancelled: + return absl::nullopt; } - return NextResult(RefRecv()); + GPR_UNREACHABLE_CODE(return absl::nullopt); } void AckNext() { - if (grpc_trace_promise_pipe.enabled()) { + if (grpc_trace_promise_primitives.enabled()) { gpr_log(GPR_INFO, "%s", DebugOpString("AckNext").c_str()); } - GPR_DEBUG_ASSERT(value_state_ == ValueState::kReady); - value_state_ = ValueState::kAcked; - on_empty_.Wake(); - UnrefRecv(); + switch (value_state_) { + case ValueState::kReady: + value_state_ = ValueState::kAcked; + on_empty_.Wake(); + break; + case ValueState::kReadyClosed: + this->ResetInterceptorList(); + value_state_ = ValueState::kClosed; + break; + case ValueState::kClosed: + case ValueState::kCancelled: + break; + case ValueState::kEmpty: + case ValueState::kAcked: + abort(); + } + } + + void MarkClosed() { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_INFO, "%s", DebugOpString("MarkClosed").c_str()); + } + switch (value_state_) { + case ValueState::kEmpty: + case ValueState::kAcked: + this->ResetInterceptorList(); + value_state_ = ValueState::kClosed; + on_full_.Wake(); + break; + case ValueState::kReady: + value_state_ = ValueState::kReadyClosed; + break; + case ValueState::kReadyClosed: + case ValueState::kClosed: + case ValueState::kCancelled: + break; + } } + void MarkCancelled() { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_INFO, "%s", DebugOpString("MarkCancelled").c_str()); + } + switch (value_state_) { + case ValueState::kEmpty: + case ValueState::kAcked: + case ValueState::kReady: + case ValueState::kReadyClosed: + this->ResetInterceptorList(); + value_state_ = ValueState::kCancelled; + on_full_.Wake(); + break; + case ValueState::kClosed: + value_state_ = ValueState::kCancelled; + break; + case ValueState::kCancelled: + break; + } + } + + bool cancelled() { return value_state_ == ValueState::kCancelled; } + T& value() { return value_; } const T& value() const { return value_; } - private: std::string DebugTag() { - return absl::StrCat(Activity::current()->DebugTag(), "PIPE[0x", - reinterpret_cast(this), "]: "); - } - std::string DebugOpString(std::string op) { - return absl::StrCat(DebugTag(), op, " send_refs=", send_refs_, - " recv_refs=", recv_refs_, - " value_state=", ValueStateName(value_state_)); - } - void ResetValue() { - // Fancy dance to move out of value in the off chance that we reclaim some - // memory earlier. - [](T) {}(std::move(value_)); - value_state_ = ValueState::kEmpty; + if (auto* activity = Activity::current()) { + return absl::StrCat(activity->DebugTag(), " PIPE[0x", + reinterpret_cast(this), "]: "); + } else { + return absl::StrCat("PIPE[0x", reinterpret_cast(this), "]: "); + } } + + private: // State of value_. enum class ValueState : uint8_t { // No value is set, it's possible to send. @@ -246,31 +309,52 @@ class Center { // Value has been received and acked, we can unblock senders and transition // to empty. kAcked, + // Pipe is closed successfully, no more values can be sent + kClosed, + // Pipe is closed successfully, no more values can be sent + // (but one value is queued and ready to be received) + kReadyClosed, + // Pipe is closed unsuccessfully, no more values can be sent + kCancelled, }; + + std::string DebugOpString(std::string op) { + return absl::StrCat(DebugTag(), op, " refs=", refs_, + " value_state=", ValueStateName(value_state_), + " on_empty=", on_empty_.DebugString().c_str(), + " on_full=", on_full_.DebugString().c_str()); + } + static const char* ValueStateName(ValueState state) { switch (state) { case ValueState::kEmpty: - return "kEmpty"; + return "Empty"; case ValueState::kReady: - return "kReady"; + return "Ready"; case ValueState::kAcked: - return "kAcked"; + return "Acked"; + case ValueState::kClosed: + return "Closed"; + case ValueState::kReadyClosed: + return "ReadyClosed"; + case ValueState::kCancelled: + return "Cancelled"; } GPR_UNREACHABLE_CODE(return "unknown"); } + T value_; - // Number of sending objects. - // 0 => send is closed. - // 1 ref each for PipeSender and Push. - uint8_t send_refs_ : 2; - // Number of receiving objects. - // 0 => recv is closed. - // 1 ref each for PipeReceiver, Next, and NextResult. - uint8_t recv_refs_ : 2; + // Number of refs + uint8_t refs_; // Current state of the value. - ValueState value_state_ : 2; + ValueState value_state_; IntraActivityWaiter on_empty_; IntraActivityWaiter on_full_; + + // Make failure to destruct show up in ASAN builds. +#ifndef NDEBUG + std::unique_ptr asan_canary_ = absl::make_unique(0); +#endif }; } // namespace pipe_detail @@ -283,23 +367,18 @@ class PipeSender { PipeSender(const PipeSender&) = delete; PipeSender& operator=(const PipeSender&) = delete; - - PipeSender(PipeSender&& other) noexcept : center_(other.center_) { - other.center_ = nullptr; - } - PipeSender& operator=(PipeSender&& other) noexcept { - if (center_ != nullptr) center_->UnrefSend(); - center_ = other.center_; - other.center_ = nullptr; - return *this; - } + PipeSender(PipeSender&& other) noexcept = default; + PipeSender& operator=(PipeSender&& other) noexcept = default; ~PipeSender() { - if (center_ != nullptr) center_->UnrefSend(); + if (center_ != nullptr) center_->MarkClosed(); } void Close() { - if (auto* center = std::exchange(center_, nullptr)) center->UnrefSend(); + if (center_ != nullptr) { + center_->MarkClosed(); + center_.reset(); + } } void Swap(PipeSender* other) { std::swap(center_, other->center_); } @@ -310,32 +389,37 @@ class PipeSender { // receiver is either closed or able to receive another message. PushType Push(T value); + template + void InterceptAndMap(Fn f, DebugLocation from = {}) { + center_->PrependMap(std::move(f), from); + } + + template + void InterceptAndMap(Fn f, OnHalfClose cleanup_fn, DebugLocation from = {}) { + center_->PrependMapWithCleanup(std::move(f), std::move(cleanup_fn), from); + } + private: friend struct Pipe; explicit PipeSender(pipe_detail::Center* center) : center_(center) {} - pipe_detail::Center* center_; + RefCountedPtr> center_; + + // Make failure to destruct show up in ASAN builds. +#ifndef NDEBUG + std::unique_ptr asan_canary_ = absl::make_unique(0); +#endif }; // Receive end of a Pipe. template class PipeReceiver { public: - using NextType = pipe_detail::Next; - PipeReceiver(const PipeReceiver&) = delete; PipeReceiver& operator=(const PipeReceiver&) = delete; - - PipeReceiver(PipeReceiver&& other) noexcept : center_(other.center_) { - other.center_ = nullptr; - } - PipeReceiver& operator=(PipeReceiver&& other) noexcept { - if (center_ != nullptr) center_->UnrefRecv(); - center_ = other.center_; - other.center_ = nullptr; - return *this; - } + PipeReceiver(PipeReceiver&& other) noexcept = default; + PipeReceiver& operator=(PipeReceiver&& other) noexcept = default; ~PipeReceiver() { - if (center_ != nullptr) center_->UnrefRecv(); + if (center_ != nullptr) center_->MarkClosed(); } void Swap(PipeReceiver* other) { std::swap(center_, other->center_); } @@ -345,12 +429,28 @@ class PipeReceiver { // message was received, or no value if the other end of the pipe was closed. // Blocks the promise until the receiver is either closed or a message is // available. - NextType Next(); + auto Next(); + + template + void InterceptAndMap(Fn f, DebugLocation from = {}) { + center_->AppendMap(std::move(f), from); + } + + template + void InterceptAndMapWithHalfClose(Fn f, OnHalfClose cleanup_fn, + DebugLocation from = {}) { + center_->AppendMapWithCleanup(std::move(f), std::move(cleanup_fn), from); + } private: friend struct Pipe; explicit PipeReceiver(pipe_detail::Center* center) : center_(center) {} - pipe_detail::Center* center_; + RefCountedPtr> center_; + + // Make failure to destruct show up in ASAN builds. +#ifndef NDEBUG + std::unique_ptr asan_canary_ = absl::make_unique(0); +#endif }; namespace pipe_detail { @@ -361,42 +461,33 @@ class Push { public: Push(const Push&) = delete; Push& operator=(const Push&) = delete; - Push(Push&& other) noexcept - : center_(other.center_), push_(std::move(other.push_)) { - other.center_ = nullptr; - } - Push& operator=(Push&& other) noexcept { - if (center_ != nullptr) center_->UnrefSend(); - center_ = other.center_; - other.center_ = nullptr; - push_ = std::move(other.push_); - return *this; - } - - ~Push() { - if (center_ != nullptr) center_->UnrefSend(); - } + Push(Push&& other) noexcept = default; + Push& operator=(Push&& other) noexcept = default; Poll operator()() { - if (push_.has_value()) { - auto r = center_->Push(&*push_); + if (center_ == nullptr) return false; + if (auto* p = absl::get_if(&state_)) { + auto r = center_->Push(p); if (auto* ok = absl::get_if(&r)) { - push_.reset(); + state_.template emplace(); if (!*ok) return false; } else { return Pending{}; } } - GPR_DEBUG_ASSERT(!push_.has_value()); + GPR_DEBUG_ASSERT(absl::holds_alternative(state_)); return center_->PollAck(); } private: + struct AwaitingAck {}; + friend class PipeSender; - explicit Push(pipe_detail::Center* center, T push) - : center_(center), push_(std::move(push)) {} - Center* center_; - absl::optional push_; + explicit Push(RefCountedPtr> center, T push) + : center_(std::move(center)), state_(std::move(push)) {} + + RefCountedPtr> center_; + absl::variant state_; }; // Implementation of PipeReceiver::Next promise. @@ -405,46 +496,56 @@ class Next { public: Next(const Next&) = delete; Next& operator=(const Next&) = delete; - Next(Next&& other) noexcept : center_(other.center_) { - other.center_ = nullptr; - } - Next& operator=(Next&& other) noexcept { - if (center_ != nullptr) center_->UnrefRecv(); - center_ = other.center_; - other.center_ = nullptr; - return *this; - } + Next(Next&& other) noexcept = default; + Next& operator=(Next&& other) noexcept = default; - ~Next() { - if (center_ != nullptr) center_->UnrefRecv(); - } - - Poll> operator()() { - auto r = center_->Next(); - if (!absl::holds_alternative(r)) { - std::exchange(center_, nullptr)->UnrefRecv(); - } - return r; - } + Poll> operator()() { return center_->Next(); } private: friend class PipeReceiver; - explicit Next(pipe_detail::Center* center) : center_(center) {} - Center* center_; + explicit Next(RefCountedPtr> center) : center_(std::move(center)) {} + + RefCountedPtr> center_; }; } // namespace pipe_detail template pipe_detail::Push PipeSender::Push(T value) { - return pipe_detail::Push(center_->RefSend(), std::move(value)); + return pipe_detail::Push(center_ == nullptr ? nullptr : center_->Ref(), + std::move(value)); } template -pipe_detail::Next PipeReceiver::Next() { - return pipe_detail::Next(center_->RefRecv()); +auto PipeReceiver::Next() { + return Seq( + pipe_detail::Next(center_->Ref()), + [center = center_->Ref()](absl::optional value) { + bool open = value.has_value(); + bool cancelled = center->cancelled(); + return If( + open, + [center = std::move(center), value = std::move(value)]() mutable { + auto run_interceptors = center->Run(std::move(value)); + return Map(std::move(run_interceptors), + [center = std::move(center)]( + absl::optional value) mutable { + if (value.has_value()) { + center->value() = std::move(*value); + return NextResult(std::move(center)); + } else { + center->MarkCancelled(); + return NextResult(true); + } + }); + }, + [cancelled]() { return NextResult(cancelled); }); + }); } +template +using PipeReceiverNextType = decltype(std::declval>().Next()); + template bool NextResult::has_value() const { return center_ != nullptr; @@ -467,7 +568,10 @@ NextResult::~NextResult() { template void NextResult::reset() { - if (auto* p = std::exchange(center_, nullptr)) p->AckNext(); + if (center_ != nullptr) { + center_->AckNext(); + center_.reset(); + } } // A Pipe is an intra-Activity communications channel that transmits T's from diff --git a/src/core/lib/promise/pipe.cc b/src/core/lib/promise/trace.cc similarity index 82% rename from src/core/lib/promise/pipe.cc rename to src/core/lib/promise/trace.cc index cbb408bea19..21b430526df 100644 --- a/src/core/lib/promise/pipe.cc +++ b/src/core/lib/promise/trace.cc @@ -14,6 +14,7 @@ #include -#include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/trace.h" -grpc_core::DebugOnlyTraceFlag grpc_trace_promise_pipe(false, "promise_pipe"); +grpc_core::DebugOnlyTraceFlag grpc_trace_promise_primitives( + false, "promise_primitives"); diff --git a/src/core/lib/promise/trace.h b/src/core/lib/promise/trace.h new file mode 100644 index 00000000000..de66138fec6 --- /dev/null +++ b/src/core/lib/promise/trace.h @@ -0,0 +1,24 @@ +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_SRC_CORE_LIB_PROMISE_TRACE_H +#define GRPC_SRC_CORE_LIB_PROMISE_TRACE_H + +#include + +#include "src/core/lib/debug/trace.h" + +extern grpc_core::DebugOnlyTraceFlag grpc_trace_promise_primitives; + +#endif // GRPC_SRC_CORE_LIB_PROMISE_TRACE_H diff --git a/src/core/lib/promise/try_concurrently.h b/src/core/lib/promise/try_concurrently.h index b05c3089b8e..06ca3739bed 100644 --- a/src/core/lib/promise/try_concurrently.h +++ b/src/core/lib/promise/try_concurrently.h @@ -20,17 +20,22 @@ #include #include +#include #include #include +#include "absl/strings/str_cat.h" #include "absl/types/variant.h" #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/construct_destruct.h" +#include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/detail/promise_like.h" #include "src/core/lib/promise/detail/status.h" #include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/trace.h" namespace grpc_core { @@ -185,9 +190,18 @@ class TryConcurrently { typename PollTraits>()())>::Type; Poll operator()() { + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, "%sBEGIN POLL: done_bits=%x necessary_bits=%x", + DebugTag().c_str(), done_bits_, NecessaryBits()); + } auto r = pre_main_.template Run(done_bits_); if (auto* status = absl::get_if(&r)) { GPR_DEBUG_ASSERT(!IsStatusOk(*status)); + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, + "%sFAIL POLL PRE-MAIN: done_bits=%x necessary_bits=%x", + DebugTag().c_str(), done_bits_, NecessaryBits()); + } return std::move(*status); } if ((done_bits_ & 1) == 0) { @@ -201,8 +215,17 @@ class TryConcurrently { r = post_main_.template Run(done_bits_); if (auto* status = absl::get_if(&r)) { GPR_DEBUG_ASSERT(!IsStatusOk(*status)); + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, + "%sFAIL POLL POST-MAIN: done_bits=%x necessary_bits=%x", + DebugTag().c_str(), done_bits_, NecessaryBits()); + } return std::move(*status); } + if (grpc_trace_promise_primitives.enabled()) { + gpr_log(GPR_DEBUG, "%sEND POLL: done_bits=%x necessary_bits=%x", + DebugTag().c_str(), done_bits_, NecessaryBits()); + } if ((done_bits_ & NecessaryBits()) == NecessaryBits()) { return std::move(result_); } @@ -234,6 +257,11 @@ class TryConcurrently { // that is all promises that are not the main one. constexpr uint8_t HelperBits() { return AllBits() ^ 1; } + std::string DebugTag() { + return absl::StrCat(Activity::current()->DebugTag(), " TRY_CONCURRENTLY[0x", + reinterpret_cast(this), "]: "); + } + // done_bits signifies which operations have completed. // Bit 0 is set if main_ has completed. // The next higher bits correspond one per pre-main promise. diff --git a/src/core/lib/resource_quota/arena.h b/src/core/lib/resource_quota/arena.h index 9b67d1ffba9..09d559664b0 100644 --- a/src/core/lib/resource_quota/arena.h +++ b/src/core/lib/resource_quota/arena.h @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -48,6 +49,11 @@ namespace grpc_core { namespace arena_detail { +struct PoolAndSize { + size_t alloc_size; + size_t pool_index; +}; + template struct PoolIndexForSize; @@ -81,10 +87,37 @@ constexpr size_t AllocationSizeFromObjectSize( return PoolIndexForSize::kSize; } +template +struct ChoosePoolForAllocationSizeImpl; + +template +struct ChoosePoolForAllocationSizeImpl { + static PoolAndSize Fn(size_t n) { + if (n <= kBucketSize) return {kBucketSize, kIndex}; + return ChoosePoolForAllocationSizeImpl::Fn(n); + } +}; + +template +struct ChoosePoolForAllocationSizeImpl { + static PoolAndSize Fn(size_t n) { + return PoolAndSize{n, std::numeric_limits::max()}; + } +}; + +template +PoolAndSize ChoosePoolForAllocationSize( + size_t n, absl::integer_sequence) { + return ChoosePoolForAllocationSizeImpl<0, kBucketSizes...>::Fn(n); +} + } // namespace arena_detail class Arena { using PoolSizes = absl::integer_sequence; + struct FreePoolNode { + FreePoolNode* next; + }; public: // Create an arena, with \a initial_size bytes in the first allocated buffer. @@ -133,7 +166,8 @@ class Arena { class PooledDeleter { public: - explicit PooledDeleter(Arena* arena) : arena_(arena) {} + explicit PooledDeleter(std::atomic* free_list) + : free_list_(free_list) {} PooledDeleter() = default; template void operator()(T* p) { @@ -142,24 +176,57 @@ class Arena { // by setting the arena to nullptr. // This is a transitional hack and should be removed once promise based // filter is removed. - if (arena_ != nullptr) arena_->DeletePooled(p); + if (free_list_ != nullptr) { + p->~T(); + FreePooled(p, free_list_); + } } + bool has_freelist() const { return free_list_ != nullptr; } + private: - Arena* arena_; + std::atomic* free_list_; }; template using PoolPtr = std::unique_ptr; + // Make a unique_ptr to T that is allocated from the arena. + // When the pointer is released, the memory may be reused for other + // MakePooled(.*) calls. + // CAUTION: The amount of memory allocated is rounded up to the nearest + // value in Arena::PoolSizes, and so this may pessimize total + // arena size. template PoolPtr MakePooled(Args&&... args) { + auto* free_list = + &pools_[arena_detail::PoolFromObjectSize(PoolSizes())]; return PoolPtr( new (AllocPooled( arena_detail::AllocationSizeFromObjectSize(PoolSizes()), - &pools_[arena_detail::PoolFromObjectSize(PoolSizes())])) - T(std::forward(args)...), - PooledDeleter(this)); + free_list)) T(std::forward(args)...), + PooledDeleter(free_list)); + } + + // Make a unique_ptr to an array of T that is allocated from the arena. + // When the pointer is released, the memory may be reused for other + // MakePooled(.*) calls. + // One can use MakePooledArray to allocate a buffer of bytes. + // CAUTION: The amount of memory allocated is rounded up to the nearest + // value in Arena::PoolSizes, and so this may pessimize total + // arena size. + template + PoolPtr MakePooledArray(size_t n) { + auto where = + arena_detail::ChoosePoolForAllocationSize(n * sizeof(T), PoolSizes()); + if (where.pool_index == std::numeric_limits::max()) { + return PoolPtr(new (Alloc(where.alloc_size)) T[n], + PooledDeleter(nullptr)); + } else { + return PoolPtr( + new (AllocPooled(where.alloc_size, &pools_[where.pool_index])) T[n], + PooledDeleter(&pools_[where.pool_index])); + } } private: @@ -202,17 +269,6 @@ class Arena { void* AllocZone(size_t size); - template - void DeletePooled(T* p) { - p->~T(); - FreePooled( - p, &pools_[arena_detail::PoolFromObjectSize(PoolSizes())]); - } - - struct FreePoolNode { - FreePoolNode* next; - }; - void* AllocPooled(size_t alloc_size, std::atomic* head); static void FreePooled(void* p, std::atomic* head); diff --git a/src/core/lib/slice/slice.cc b/src/core/lib/slice/slice.cc index 177ca456ec5..51ee3a83644 100644 --- a/src/core/lib/slice/slice.cc +++ b/src/core/lib/slice/slice.cc @@ -283,7 +283,7 @@ grpc_slice grpc_slice_sub(grpc_slice source, size_t begin, size_t end) { subset = grpc_slice_sub_no_ref(source, begin, end); // Bump the refcount if (subset.refcount != grpc_slice_refcount::NoopRefcount()) { - subset.refcount->Ref(); + subset.refcount->Ref({}); } } return subset; @@ -332,7 +332,7 @@ grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice* source, size_t split, tail.refcount = source->refcount; // Bump the refcount if (tail.refcount != grpc_slice_refcount::NoopRefcount()) { - tail.refcount->Ref(); + tail.refcount->Ref({}); } break; } @@ -378,7 +378,7 @@ grpc_slice grpc_slice_split_head(grpc_slice* source, size_t split) { head.refcount = source->refcount; // Bump the refcount if (head.refcount != grpc_slice_refcount::NoopRefcount()) { - head.refcount->Ref(); + head.refcount->Ref({}); } // Point into the source array head.data.refcounted.bytes = source->data.refcounted.bytes; diff --git a/src/core/lib/slice/slice.h b/src/core/lib/slice/slice.h index 17047008eca..2640ee80644 100644 --- a/src/core/lib/slice/slice.h +++ b/src/core/lib/slice/slice.h @@ -31,6 +31,7 @@ #include #include "src/core/lib/gpr/string.h" +#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_refcount.h" @@ -54,16 +55,17 @@ namespace grpc_core { -inline const grpc_slice& CSliceRef(const grpc_slice& slice) { +inline const grpc_slice& CSliceRef(const grpc_slice& slice, + DebugLocation loc = {}) { if (reinterpret_cast(slice.refcount) > 1) { - slice.refcount->Ref(); + slice.refcount->Ref(loc); } return slice; } -inline void CSliceUnref(const grpc_slice& slice) { +inline void CSliceUnref(const grpc_slice& slice, DebugLocation loc = {}) { if (reinterpret_cast(slice.refcount) > 1) { - slice.refcount->Unref(); + slice.refcount->Unref(loc); } } @@ -391,10 +393,11 @@ class GPR_MSVC_EMPTY_BASE_CLASS_WORKAROUND Slice Slice Copy() const { return Slice(grpc_slice_copy(c_slice())); } static Slice FromRefcountAndBytes(grpc_slice_refcount* r, - const uint8_t* begin, const uint8_t* end) { + const uint8_t* begin, const uint8_t* end, + DebugLocation location = {}) { grpc_slice out; out.refcount = r; - if (r != grpc_slice_refcount::NoopRefcount()) r->Ref(); + if (r != grpc_slice_refcount::NoopRefcount()) r->Ref(location); out.data.refcounted.bytes = const_cast(begin); out.data.refcounted.length = end - begin; return Slice(out); diff --git a/src/core/lib/slice/slice_buffer.h b/src/core/lib/slice/slice_buffer.h index b87a91e1306..45d41031380 100644 --- a/src/core/lib/slice/slice_buffer.h +++ b/src/core/lib/slice/slice_buffer.h @@ -20,8 +20,11 @@ #include #include +#include #include +#include "absl/memory/memory.h" + #include #include @@ -151,6 +154,11 @@ class SliceBuffer { private: /// The backing raw slice buffer. grpc_slice_buffer slice_buffer_; + +// Make failure to destruct show up in ASAN builds. +#ifndef NDEBUG + std::unique_ptr asan_canary_ = absl::make_unique(0); +#endif }; } // namespace grpc_core diff --git a/src/core/lib/slice/slice_refcount.cc b/src/core/lib/slice/slice_refcount.cc new file mode 100644 index 00000000000..0a05f0a333a --- /dev/null +++ b/src/core/lib/slice/slice_refcount.cc @@ -0,0 +1,20 @@ +// Copyright 2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "src/core/lib/slice/slice_refcount.h" + +grpc_core::DebugOnlyTraceFlag grpc_slice_refcount_trace(false, + "slice_refcount"); diff --git a/src/core/lib/slice/slice_refcount.h b/src/core/lib/slice/slice_refcount.h index 5a862542d45..50fdb6efd9a 100644 --- a/src/core/lib/slice/slice_refcount.h +++ b/src/core/lib/slice/slice_refcount.h @@ -17,10 +17,18 @@ #include +#include #include #include +#include + +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/debug_location.h" + +extern grpc_core::DebugOnlyTraceFlag grpc_slice_refcount_trace; + // grpc_slice_refcount : A reference count for grpc_slice. struct grpc_slice_refcount { public: @@ -40,9 +48,21 @@ struct grpc_slice_refcount { explicit grpc_slice_refcount(DestroyerFn destroyer_fn) : destroyer_fn_(destroyer_fn) {} - void Ref() { ref_.fetch_add(1, std::memory_order_relaxed); } - void Unref() { - if (ref_.fetch_sub(1, std::memory_order_acq_rel) == 1) { + void Ref(grpc_core::DebugLocation location) { + auto prev_refs = ref_.fetch_add(1, std::memory_order_relaxed); + if (grpc_slice_refcount_trace.enabled()) { + gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_INFO, + "REF %p %" PRIdPTR "->%" PRIdPTR, this, prev_refs, prev_refs + 1); + } + } + void Unref(grpc_core::DebugLocation location) { + auto prev_refs = ref_.fetch_sub(1, std::memory_order_acq_rel); + if (grpc_slice_refcount_trace.enabled()) { + gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_INFO, + "UNREF %p %" PRIdPTR "->%" PRIdPTR, this, prev_refs, + prev_refs - 1); + } + if (prev_refs == 1) { destroyer_fn_(this); } } diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 06437057a67..6f7b771e1c8 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -84,7 +84,7 @@ #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/latch.h" +#include "src/core/lib/promise/detail/basic_seq.h" #include "src/core/lib/promise/pipe.h" #include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/arena.h" @@ -103,7 +103,7 @@ grpc_core::TraceFlag grpc_call_error_trace(false, "call_error"); grpc_core::TraceFlag grpc_compression_trace(false, "compression"); grpc_core::TraceFlag grpc_call_trace(false, "call"); -grpc_core::TraceFlag grpc_call_refcount_trace(false, "call_refcount"); +grpc_core::DebugOnlyTraceFlag grpc_call_refcount_trace(false, "call_refcount"); namespace grpc_core { @@ -452,6 +452,31 @@ class FilterStackCall final : public Call { static constexpr gpr_atm kRecvNone = 0; static constexpr gpr_atm kRecvInitialMetadataFirst = 1; + enum class PendingOp { + kRecvMessage, + kRecvInitialMetadata, + kRecvTrailingMetadata, + kSends + }; + static intptr_t PendingOpMask(PendingOp op) { + return static_cast(1) << static_cast(op); + } + static std::string PendingOpString(intptr_t pending_ops) { + std::vector pending_op_strings; + if (pending_ops & PendingOpMask(PendingOp::kRecvMessage)) { + pending_op_strings.push_back("kRecvMessage"); + } + if (pending_ops & PendingOpMask(PendingOp::kRecvInitialMetadata)) { + pending_op_strings.push_back("kRecvInitialMetadata"); + } + if (pending_ops & PendingOpMask(PendingOp::kRecvTrailingMetadata)) { + pending_op_strings.push_back("kRecvTrailingMetadata"); + } + if (pending_ops & PendingOpMask(PendingOp::kSends)) { + pending_op_strings.push_back("kSends"); + } + return absl::StrCat("{", absl::StrJoin(pending_op_strings, ","), "}"); + } struct BatchControl { FilterStackCall* call_ = nullptr; grpc_transport_stream_op_batch op_; @@ -476,17 +501,26 @@ class FilterStackCall final : public Call { } completion_data_; grpc_closure start_batch_; grpc_closure finish_batch_; - std::atomic steps_to_complete_{0}; + std::atomic ops_pending_{0}; AtomicError batch_error_; - void set_num_steps_to_complete(uintptr_t steps) { - steps_to_complete_.store(steps, std::memory_order_release); + void set_pending_ops(uintptr_t ops) { + ops_pending_.store(ops, std::memory_order_release); } - bool completed_batch_step() { - return steps_to_complete_.fetch_sub(1, std::memory_order_acq_rel) == 1; + bool completed_batch_step(PendingOp op) { + auto mask = PendingOpMask(op); + auto r = ops_pending_.fetch_sub(mask, std::memory_order_acq_rel); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_DEBUG, "BATCH:%p COMPLETE:%s REMAINING:%s (tag:%p)", this, + PendingOpString(mask).c_str(), + PendingOpString(r & ~mask).c_str(), + completion_data_.notify_tag.tag); + } + GPR_ASSERT((r & mask) != 0); + return r == mask; } void PostCompletion(); - void FinishStep(); + void FinishStep(PendingOp op); void ProcessDataAfterMetadata(); void ReceivingStreamReady(grpc_error_handle error); void ValidateFilteredMetadata(); @@ -1105,6 +1139,10 @@ FilterStackCall::BatchControl* FilterStackCall::ReuseOrAllocateBatchControl( void FilterStackCall::BatchControl::PostCompletion() { FilterStackCall* call = call_; grpc_error_handle error = batch_error_.get(); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_DEBUG, "tag:%p batch_error=%s", completion_data_.notify_tag.tag, + error.ToString().c_str()); + } if (op_.send_initial_metadata) { call->send_initial_metadata_.Clear(); @@ -1134,14 +1172,12 @@ void FilterStackCall::BatchControl::PostCompletion() { batch_error_.set(absl::OkStatus()); if (completion_data_.notify_tag.is_closure) { - // unrefs error call_ = nullptr; Closure::Run(DEBUG_LOCATION, static_cast(completion_data_.notify_tag.tag), error); call->InternalUnref("completion"); } else { - // unrefs error grpc_cq_end_op( call->cq_, completion_data_.notify_tag.tag, error, [](void* user_data, grpc_cq_completion* /*storage*/) { @@ -1154,8 +1190,8 @@ void FilterStackCall::BatchControl::PostCompletion() { } } -void FilterStackCall::BatchControl::FinishStep() { - if (GPR_UNLIKELY(completed_batch_step())) { +void FilterStackCall::BatchControl::FinishStep(PendingOp op) { + if (GPR_UNLIKELY(completed_batch_step(op))) { PostCompletion(); } } @@ -1165,7 +1201,7 @@ void FilterStackCall::BatchControl::ProcessDataAfterMetadata() { if (!call->receiving_slice_buffer_.has_value()) { *call->receiving_buffer_ = nullptr; call->receiving_message_ = false; - FinishStep(); + FinishStep(PendingOp::kRecvMessage); } else { call->test_only_last_message_flags_ = call->receiving_stream_flags_; if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) && @@ -1180,12 +1216,20 @@ void FilterStackCall::BatchControl::ProcessDataAfterMetadata() { &(*call->receiving_buffer_)->data.raw.slice_buffer); call->receiving_message_ = false; call->receiving_slice_buffer_.reset(); - FinishStep(); + FinishStep(PendingOp::kRecvMessage); } } void FilterStackCall::BatchControl::ReceivingStreamReady( grpc_error_handle error) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_DEBUG, + "tag:%p ReceivingStreamReady error=%s " + "receiving_slice_buffer.has_value=%d recv_state=%" PRIdPTR, + completion_data_.notify_tag.tag, error.ToString().c_str(), + call_->receiving_slice_buffer_.has_value(), + gpr_atm_no_barrier_load(&call_->recv_state_)); + } FilterStackCall* call = call_; if (!error.ok()) { call->receiving_slice_buffer_.reset(); @@ -1303,7 +1347,7 @@ void FilterStackCall::BatchControl::ReceivingInitialMetadataReady( Closure::Run(DEBUG_LOCATION, saved_rsr_closure, error); } - FinishStep(); + FinishStep(PendingOp::kRecvInitialMetadata); } void FilterStackCall::BatchControl::ReceivingTrailingMetadataReady( @@ -1312,7 +1356,7 @@ void FilterStackCall::BatchControl::ReceivingTrailingMetadataReady( "recv_trailing_metadata_ready"); grpc_metadata_batch* md = &call_->recv_trailing_metadata_; call_->RecvTrailingFilter(md, error); - FinishStep(); + FinishStep(PendingOp::kRecvTrailingMetadata); } void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) { @@ -1323,7 +1367,7 @@ void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) { if (!error.ok()) { call_->CancelWithError(error); } - FinishStep(); + FinishStep(PendingOp::kSends); } namespace { @@ -1350,12 +1394,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, size_t i; const grpc_op* op; BatchControl* bctl; - bool has_send_ops = false; - int num_recv_ops = 0; grpc_call_error error = GRPC_CALL_OK; grpc_transport_stream_op_batch* stream_op; grpc_transport_stream_op_batch_payload* stream_op_payload; uint32_t seen_ops = 0; + intptr_t pending_ops = 0; for (i = 0; i < nops; i++) { if (seen_ops & (1u << ops[i].op)) { @@ -1472,7 +1515,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, stream_op_payload->send_initial_metadata.peer_string = peer_string_atm_ptr(); } - has_send_ops = true; + pending_ops |= PendingOpMask(PendingOp::kSends); break; } case GRPC_OP_SEND_MESSAGE: { @@ -1504,7 +1547,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, send_slice_buffer_.c_slice_buffer()); stream_op_payload->send_message.flags = flags; stream_op_payload->send_message.send_message = &send_slice_buffer_; - has_send_ops = true; + pending_ops |= PendingOpMask(PendingOp::kSends); break; } case GRPC_OP_SEND_CLOSE_FROM_CLIENT: { @@ -1525,7 +1568,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, sent_final_op_ = true; stream_op_payload->send_trailing_metadata.send_trailing_metadata = &send_trailing_metadata_; - has_send_ops = true; + pending_ops |= PendingOpMask(PendingOp::kSends); break; } case GRPC_OP_SEND_STATUS_FROM_SERVER: { @@ -1589,7 +1632,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, &send_trailing_metadata_; stream_op_payload->send_trailing_metadata.sent = &sent_server_trailing_metadata_; - has_send_ops = true; + pending_ops |= PendingOpMask(PendingOp::kSends); break; } case GRPC_OP_RECV_INITIAL_METADATA: { @@ -1624,7 +1667,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, stream_op_payload->recv_initial_metadata.peer_string = peer_string_atm_ptr(); } - ++num_recv_ops; + pending_ops |= PendingOpMask(PendingOp::kRecvInitialMetadata); break; } case GRPC_OP_RECV_MESSAGE: { @@ -1660,7 +1703,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, bctl, grpc_schedule_on_exec_ctx); stream_op_payload->recv_message.recv_message_ready = &receiving_stream_ready_; - ++num_recv_ops; + pending_ops |= PendingOpMask(PendingOp::kRecvMessage); break; } case GRPC_OP_RECV_STATUS_ON_CLIENT: { @@ -1699,7 +1742,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, bctl, grpc_schedule_on_exec_ctx); stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready = &receiving_trailing_metadata_ready_; - ++num_recv_ops; + pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata); break; } case GRPC_OP_RECV_CLOSE_ON_SERVER: { @@ -1732,7 +1775,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, bctl, grpc_schedule_on_exec_ctx); stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready = &receiving_trailing_metadata_ready_; - ++num_recv_ops; + pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata); break; } } @@ -1742,9 +1785,9 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, if (!is_notify_tag_closure) { GPR_ASSERT(grpc_cq_begin_op(cq_, notify_tag)); } - bctl->set_num_steps_to_complete((has_send_ops ? 1 : 0) + num_recv_ops); + bctl->set_pending_ops(pending_ops); - if (has_send_ops) { + if (pending_ops & PendingOpMask(PendingOp::kSends)) { GRPC_CLOSURE_INIT( &bctl->finish_batch_, [](void* bctl, grpc_error_handle error) { @@ -1754,6 +1797,13 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, stream_op->on_complete = &bctl->finish_batch_; } + if (grpc_call_trace.enabled()) { + gpr_log(GPR_DEBUG, "BATCH:%p START:%s BATCH:%s (tag:%p)", bctl, + PendingOpString(pending_ops).c_str(), + grpc_transport_stream_op_batch_string(stream_op).c_str(), + bctl->completion_data_.notify_tag.tag); + } + ExecuteBatch(stream_op, &bctl->start_batch_); done: @@ -1827,21 +1877,52 @@ class PromiseBasedCall : public Call, public grpc_event_engine::experimental::EventEngine:: Closure /* for deadlines */ { public: - PromiseBasedCall(Arena* arena, const grpc_call_create_args& args); + PromiseBasedCall(Arena* arena, uint32_t initial_external_refs, + const grpc_call_create_args& args); void ContextSet(grpc_context_index elem, void* value, void (*destroy)(void* value)) override; void* ContextGet(grpc_context_index elem) const override; void SetCompletionQueue(grpc_completion_queue* cq) override; + void SetCompletionQueueLocked(grpc_completion_queue* cq) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + void CancelWithError(absl::Status error) final ABSL_LOCKS_EXCLUDED(mu_) { + MutexLock lock(&mu_); + CancelWithErrorLocked(std::move(error)); + } + virtual void CancelWithErrorLocked(absl::Status error) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0; + bool Completed() final ABSL_LOCKS_EXCLUDED(mu_) { + MutexLock lock(&mu_); + return completed_; + } + + void Orphan() final { + MutexLock lock(&mu_); + if (!completed_) { + CancelWithErrorLocked(absl::CancelledError("Call orphaned")); + } + } // Implementation of call refcounting: move this to DualRefCounted once we // don't need to maintain FilterStackCall compatibility void ExternalRef() final { - refs_.fetch_add(MakeRefPair(1, 0), std::memory_order_relaxed); + const uint64_t prev_ref_pair = + refs_.fetch_add(MakeRefPair(1, 0), std::memory_order_relaxed); + if (grpc_call_refcount_trace.enabled()) { + gpr_log(GPR_DEBUG, "%s EXTERNAL_REF: %d:%d->%d:%d", DebugTag().c_str(), + GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair), + GetStrongRefs(prev_ref_pair) + 1, GetWeakRefs(prev_ref_pair)); + } } void ExternalUnref() final { const uint64_t prev_ref_pair = refs_.fetch_add(MakeRefPair(-1, 1), std::memory_order_acq_rel); + if (grpc_call_refcount_trace.enabled()) { + gpr_log(GPR_DEBUG, "%s EXTERNAL_UNREF: %d:%d->%d:%d", DebugTag().c_str(), + GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair), + GetStrongRefs(prev_ref_pair) - 1, GetWeakRefs(prev_ref_pair) + 1); + } const uint32_t strong_refs = GetStrongRefs(prev_ref_pair); if (GPR_UNLIKELY(strong_refs == 1)) { Orphan(); @@ -1849,12 +1930,22 @@ class PromiseBasedCall : public Call, // Now drop the weak ref. InternalUnref("external_ref"); } - void InternalRef(const char*) final { - refs_.fetch_add(MakeRefPair(0, 1), std::memory_order_relaxed); + void InternalRef(const char* reason) final { + uint64_t n = refs_.fetch_add(MakeRefPair(0, 1), std::memory_order_relaxed); + if (grpc_call_refcount_trace.enabled()) { + gpr_log(GPR_DEBUG, "%s REF: %s %d:%d->%d:%d", DebugTag().c_str(), reason, + GetStrongRefs(n), GetWeakRefs(n), GetStrongRefs(n), + GetWeakRefs(n) + 1); + } } - void InternalUnref(const char*) final { + void InternalUnref(const char* reason) final { const uint64_t prev_ref_pair = refs_.fetch_sub(MakeRefPair(0, 1), std::memory_order_acq_rel); + if (grpc_call_refcount_trace.enabled()) { + gpr_log(GPR_DEBUG, "%s UNREF: %s %d:%d->%d:%d", DebugTag().c_str(), + reason, GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair), + GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair) - 1); + } if (GPR_UNLIKELY(prev_ref_pair == MakeRefPair(0, 1))) { DeleteThis(); } @@ -1954,9 +2045,12 @@ class PromiseBasedCall : public Call, void UpdateDeadline(Timestamp deadline); void ResetDeadline(); + // Implementation of EventEngine::Closure, called when deadline expires void Run() override; + virtual ServerCallContext* server_call_context() { return nullptr; } + protected: class ScopedContext : public ScopedActivity, @@ -1994,11 +2088,6 @@ class PromiseBasedCall : public Call, uint8_t TakeIndex() { return std::exchange(index_, kNullIndex); } bool has_value() const { return index_ != kNullIndex; } - std::string ToString() const { - return index_ == kNullIndex ? "null" - : std::to_string(static_cast(index_)); - } - private: enum : uint8_t { kNullIndex = 0xff }; uint8_t index_; @@ -2016,22 +2105,27 @@ class PromiseBasedCall : public Call, // The following correspond with the batch operations from above kReceiveInitialMetadata, kReceiveStatusOnClient, + kReceiveCloseOnServer = kReceiveStatusOnClient, kSendMessage, kReceiveMessage, + kSendStatusFromServer, + kSendCloseFromClient = kSendStatusFromServer, }; - static constexpr const char* PendingOpString(PendingOp reason) { + const char* PendingOpString(PendingOp reason) const { switch (reason) { case PendingOp::kStartingBatch: return "StartingBatch"; case PendingOp::kReceiveInitialMetadata: return "ReceiveInitialMetadata"; case PendingOp::kReceiveStatusOnClient: - return "ReceiveStatusOnClient"; + return is_client() ? "ReceiveStatusOnClient" : "ReceiveCloseOnServer"; case PendingOp::kSendMessage: return "SendMessage"; case PendingOp::kReceiveMessage: return "ReceiveMessage"; + case PendingOp::kSendStatusFromServer: + return is_client() ? "SendCloseFromClient" : "SendStatusFromServer"; } return "Unknown"; } @@ -2041,7 +2135,6 @@ class PromiseBasedCall : public Call, } Mutex* mu() const ABSL_LOCK_RETURNED(mu_) { return &mu_; } - // Begin work on a completion, recording the tag/closure to notify. // Use the op selected in \a ops to determine the index to allocate into. // Starts the "StartingBatch" PendingOp immediately. @@ -2051,12 +2144,21 @@ class PromiseBasedCall : public Call, // Add one pending op to the completion, and return it. Completion AddOpToCompletion(const Completion& completion, PendingOp reason) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + // Stringify a completion + std::string CompletionString(const Completion& completion) const { + return completion.has_value() + ? absl::StrFormat( + "%d:tag=%p", static_cast(completion.index()), + completion_info_[completion.index()].pending.tag) + : "no-completion"; + } // Finish one op on the completion. Must have been previously been added. // The completion as a whole finishes when all pending ops finish. void FinishOpOnCompletion(Completion* completion, PendingOp reason) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Mark the completion as failed. Does not finish it. - void FailCompletion(const Completion& completion); + void FailCompletion(const Completion& completion, + SourceLocation source_location = {}); // Run the promise polling loop until it stalls. void Update() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Update the promise state once. @@ -2087,6 +2189,56 @@ class PromiseBasedCall : public Call, finalization_.Run(&final_info); } + std::string PresentAndCompletionText(const char* caption, bool has, + const Completion& completion) const { + if (has) { + if (completion.has_value()) { + return absl::StrCat(caption, ":", CompletionString(completion), " "); + } else { + return absl::StrCat(caption, + ":!!BUG:operation is present, no completion!! "); + } + } else { + if (!completion.has_value()) { + return ""; + } else { + return absl::StrCat(caption, ":no-op:", CompletionString(completion), + " "); + } + } + } + + std::string PollStateDebugString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return absl::StrCat(PresentAndCompletionText("outstanding_send", + outstanding_send_.has_value(), + send_message_completion_) + .c_str(), + PresentAndCompletionText("outstanding_recv", + outstanding_recv_.has_value(), + recv_message_completion_) + .c_str()); + } + + void StartRecvMessage(const grpc_op& op, const Completion& completion, + PipeReceiver* receiver) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + void PollRecvMessage(grpc_compression_algorithm compression_algorithm) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + void CancelRecvMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + void StartSendMessage(const grpc_op& op, const Completion& completion, + PipeSender* sender) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + bool PollSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + void CancelSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + + bool completed() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return completed_; + } + void set_completed() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { completed_ = true; } + bool is_sending() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return outstanding_send_.has_value(); + } + private: union CompletionInfo { struct Pending { @@ -2186,7 +2338,7 @@ class PromiseBasedCall : public Call, } mutable Mutex mu_; - std::atomic refs_{MakeRefPair(1, 0)}; + std::atomic refs_; CallContext call_context_{this}; bool keep_polling_ ABSL_GUARDED_BY(mu()) = false; @@ -2200,6 +2352,14 @@ class PromiseBasedCall : public Call, // Current deadline. Timestamp deadline_ = Timestamp::InfFuture(); grpc_event_engine::experimental::EventEngine::TaskHandle deadline_task_; + absl::optional::PushType> outstanding_send_ + ABSL_GUARDED_BY(mu_); + absl::optional> outstanding_recv_ + ABSL_GUARDED_BY(mu_); + grpc_byte_buffer** recv_message_ ABSL_GUARDED_BY(mu_) = nullptr; + Completion send_message_completion_ ABSL_GUARDED_BY(mu_); + Completion recv_message_completion_ ABSL_GUARDED_BY(mu_); + bool completed_ ABSL_GUARDED_BY(mu_) = false; }; template @@ -2215,10 +2375,11 @@ grpc_error_handle MakePromiseBasedCall(grpc_call_create_args* args, return absl::OkStatus(); } -PromiseBasedCall::PromiseBasedCall(Arena* arena, +PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs, const grpc_call_create_args& args) : Call(arena, args.server_transport_data == nullptr, args.send_deadline, args.channel->Ref()), + refs_(MakeRefPair(initial_external_refs, 0)), cq_(args.cq) { if (args.cq != nullptr) { GPR_ASSERT(args.pollset_set_alternative == nullptr && @@ -2278,8 +2439,8 @@ PromiseBasedCall::Completion PromiseBasedCall::StartCompletion( void* tag, bool is_closure, const grpc_op* ops) { Completion c(BatchSlotForOp(ops[0].op)); if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sStartCompletion %s tag=%p", DebugTag().c_str(), - c.ToString().c_str(), tag); + gpr_log(GPR_INFO, "%s[call] StartCompletion %s tag=%p", DebugTag().c_str(), + CompletionString(c).c_str(), tag); } if (!is_closure) { grpc_cq_begin_op(cq(), tag); @@ -2292,9 +2453,10 @@ PromiseBasedCall::Completion PromiseBasedCall::StartCompletion( PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion( const Completion& completion, PendingOp reason) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sAddOpToCompletion %s %s", DebugTag().c_str(), - completion.ToString().c_str(), PendingOpString(reason)); + gpr_log(GPR_INFO, "%s[call] AddOpToCompletion %s %s", DebugTag().c_str(), + CompletionString(completion).c_str(), PendingOpString(reason)); } + GPR_ASSERT(completion.has_value()); auto& pending_op_bits = completion_info_[completion.index()].pending.pending_op_bits; GPR_ASSERT((pending_op_bits & PendingOpBit(reason)) == 0); @@ -2302,10 +2464,12 @@ PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion( return Completion(completion.index()); } -void PromiseBasedCall::FailCompletion(const Completion& completion) { +void PromiseBasedCall::FailCompletion(const Completion& completion, + SourceLocation location) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sFailCompletion %s", DebugTag().c_str(), - completion.ToString().c_str()); + gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_ERROR, + "%s[call] FailCompletion %s", DebugTag().c_str(), + CompletionString(completion).c_str()); } completion_info_[completion.index()].pending.success = false; } @@ -2324,8 +2488,9 @@ void PromiseBasedCall::FinishOpOnCompletion(Completion* completion, } } gpr_log( - GPR_INFO, "%sFinishOpOnCompletion %s %s %s", DebugTag().c_str(), - completion->ToString().c_str(), PendingOpString(reason), + GPR_INFO, "%s[call] FinishOpOnCompletion tag:%p %s %s %s", + DebugTag().c_str(), completion_info_[completion->index()].pending.tag, + CompletionString(*completion).c_str(), PendingOpString(reason), (pending.empty() ? (success ? std::string("done") : std::string("failed")) : absl::StrFormat("pending_ops={%s}", absl::StrJoin(pending, ","))) @@ -2360,6 +2525,10 @@ void PromiseBasedCall::ForceImmediateRepoll() { keep_polling_ = true; } void PromiseBasedCall::SetCompletionQueue(grpc_completion_queue* cq) { MutexLock lock(&mu_); + SetCompletionQueueLocked(cq); +} + +void PromiseBasedCall::SetCompletionQueueLocked(grpc_completion_queue* cq) { cq_ = cq; GRPC_CQ_INTERNAL_REF(cq, "bind"); call_context_.pollent_ = @@ -2392,6 +2561,121 @@ void PromiseBasedCall::Run() { InternalUnref("deadline"); } +void PromiseBasedCall::StartSendMessage(const grpc_op& op, + const Completion& completion, + PipeSender* sender) { + GPR_ASSERT(!outstanding_send_.has_value()); + if (!completed_) { + send_message_completion_ = + AddOpToCompletion(completion, PendingOp::kSendMessage); + SliceBuffer send; + grpc_slice_buffer_swap( + &op.data.send_message.send_message->data.raw.slice_buffer, + send.c_slice_buffer()); + outstanding_send_.emplace(sender->Push( + GetContext()->MakePooled(std::move(send), op.flags))); + } else { + FailCompletion(completion); + } +} + +bool PromiseBasedCall::PollSendMessage() { + if (!outstanding_send_.has_value()) return true; + Poll r = (*outstanding_send_)(); + if (const bool* result = absl::get_if(&r)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_DEBUG, "%sPollSendMessage completes %s", DebugTag().c_str(), + *result ? "successfully" : "with failure"); + } + if (!*result) { + FailCompletion(send_message_completion_); + return false; + } + FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage); + outstanding_send_.reset(); + } + return true; +} + +void PromiseBasedCall::CancelSendMessage() { + if (!outstanding_send_.has_value()) return; + FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage); + outstanding_send_.reset(); +} + +void PromiseBasedCall::StartRecvMessage(const grpc_op& op, + const Completion& completion, + PipeReceiver* receiver) { + GPR_ASSERT(!outstanding_recv_.has_value()); + recv_message_ = op.data.recv_message.recv_message; + recv_message_completion_ = + AddOpToCompletion(completion, PendingOp::kReceiveMessage); + outstanding_recv_.emplace(receiver->Next()); +} + +void PromiseBasedCall::PollRecvMessage( + grpc_compression_algorithm incoming_compression_algorithm) { + if (!outstanding_recv_.has_value()) return; + Poll> r = (*outstanding_recv_)(); + if (auto* result = absl::get_if>(&r)) { + outstanding_recv_.reset(); + if (result->has_value()) { + MessageHandle& message = **result; + if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) && + (incoming_compression_algorithm != GRPC_COMPRESS_NONE)) { + *recv_message_ = grpc_raw_compressed_byte_buffer_create( + nullptr, 0, incoming_compression_algorithm); + } else { + *recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0); + } + grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(), + &(*recv_message_)->data.raw.slice_buffer); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%s[call] PollRecvMessage: outstanding_recv finishes: received " + "%" PRIdPTR " byte message", + DebugTag().c_str(), + (*recv_message_)->data.raw.slice_buffer.length); + } + } else if (result->cancelled()) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%s[call] PollRecvMessage: outstanding_recv finishes: received " + "end-of-stream with error", + DebugTag().c_str()); + } + FailCompletion(recv_message_completion_); + *recv_message_ = nullptr; + } else { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%s[call] PollRecvMessage: outstanding_recv finishes: received " + "end-of-stream", + DebugTag().c_str()); + } + *recv_message_ = nullptr; + } + FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage); + } else if (completed_) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%s[call] UpdateOnce: outstanding_recv finishes: promise has " + "completed without queuing a message, forcing end-of-stream", + DebugTag().c_str()); + } + outstanding_recv_.reset(); + *recv_message_ = nullptr; + FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage); + } +} + +void PromiseBasedCall::CancelRecvMessage() { + if (!outstanding_recv_.has_value()) return; + *recv_message_ = nullptr; + outstanding_recv_.reset(); + FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage); +} + /////////////////////////////////////////////////////////////////////////////// // CallContext @@ -2405,17 +2689,42 @@ void CallContext::IncrementRefCount(const char* reason) { void CallContext::Unref(const char* reason) { call_->InternalUnref(reason); } +gpr_atm* CallContext::peer_string_atm_ptr() { + return call_->peer_string_atm_ptr(); +} + void CallContext::UpdateDeadline(Timestamp deadline) { call_->UpdateDeadline(deadline); } +ServerCallContext* CallContext::server_call_context() { + return call_->server_call_context(); +} + +/////////////////////////////////////////////////////////////////////////////// +// PublishMetadataArray + +namespace { +void PublishMetadataArray(grpc_metadata_batch* md, grpc_metadata_array* array) { + const auto md_count = md->count(); + if (md_count > array->capacity) { + array->capacity = + std::max(array->capacity + md->count(), array->capacity * 3 / 2); + array->metadata = static_cast( + gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity)); + } + PublishToAppEncoder encoder(array); + md->Encode(&encoder); +} +} // namespace + /////////////////////////////////////////////////////////////////////////////// // ClientPromiseBasedCall class ClientPromiseBasedCall final : public PromiseBasedCall { public: ClientPromiseBasedCall(Arena* arena, grpc_call_create_args* args) - : PromiseBasedCall(arena, *args) { + : PromiseBasedCall(arena, 1, *args) { global_stats().IncrementClientCallsCreated(); ScopedContext context(this); send_initial_metadata_ = @@ -2443,16 +2752,12 @@ class ClientPromiseBasedCall final : public PromiseBasedCall { // before context. auto c2s = std::move(client_to_server_messages_); auto s2c = std::move(server_to_client_messages_); + auto sim = std::move(server_initial_metadata_); } absl::string_view GetServerAuthority() const override { abort(); } - void CancelWithError(grpc_error_handle error) override; - bool Completed() override; - void Orphan() override { - MutexLock lock(mu()); - ScopedContext ctx(this); - if (!completed_) Finish(ServerMetadataFromStatus(absl::CancelledError())); - } + void CancelWithErrorLocked(grpc_error_handle error) override + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); bool is_trailers_only() const override { MutexLock lock(mu()); return is_trailers_only_; @@ -2482,9 +2787,6 @@ class ClientPromiseBasedCall final : public PromiseBasedCall { // Start the underlying promise. void StartPromise(ClientMetadataHandle client_initial_metadata) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); - // Publish some metadata out to the application. - static void PublishMetadataArray(grpc_metadata_array* array, - ServerMetadata* md); // Publish status out to the application. void PublishStatus( grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args, @@ -2495,29 +2797,23 @@ class ClientPromiseBasedCall final : public PromiseBasedCall { ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); ArenaPromise promise_ ABSL_GUARDED_BY(mu()); - Latch server_initial_metadata_ ABSL_GUARDED_BY(mu()); + Pipe server_initial_metadata_ ABSL_GUARDED_BY(mu()){ + arena()}; Pipe client_to_server_messages_ ABSL_GUARDED_BY(mu()){arena()}; Pipe server_to_client_messages_ ABSL_GUARDED_BY(mu()){arena()}; ClientMetadataHandle send_initial_metadata_; grpc_metadata_array* recv_initial_metadata_ ABSL_GUARDED_BY(mu()) = nullptr; - grpc_byte_buffer** recv_message_ ABSL_GUARDED_BY(mu()) = nullptr; absl::variant recv_status_on_client_ ABSL_GUARDED_BY(mu()); - absl::optional::PushType> outstanding_send_ - ABSL_GUARDED_BY(mu()); - absl::optional::NextType> outstanding_recv_ - ABSL_GUARDED_BY(mu()); - absl::optional> + absl::optional> server_initial_metadata_ready_; absl::optional incoming_compression_algorithm_; Completion recv_initial_metadata_completion_ ABSL_GUARDED_BY(mu()); Completion recv_status_on_client_completion_ ABSL_GUARDED_BY(mu()); - Completion send_message_completion_ ABSL_GUARDED_BY(mu()); - Completion recv_message_completion_ ABSL_GUARDED_BY(mu()); - bool completed_ ABSL_GUARDED_BY(mu()) = false; + Completion close_send_completion_ ABSL_GUARDED_BY(mu()); bool is_trailers_only_ ABSL_GUARDED_BY(mu()); }; @@ -2526,14 +2822,13 @@ void ClientPromiseBasedCall::StartPromise( GPR_ASSERT(!promise_.has_value()); promise_ = channel()->channel_stack()->MakeClientCallPromise(CallArgs{ std::move(client_initial_metadata), - &server_initial_metadata_, + &server_initial_metadata_.sender, &client_to_server_messages_.receiver, &server_to_client_messages_.sender, }); } -void ClientPromiseBasedCall::CancelWithError(grpc_error_handle error) { - MutexLock lock(mu()); +void ClientPromiseBasedCall::CancelWithErrorLocked(grpc_error_handle error) { ScopedContext context(this); Finish(ServerMetadataFromStatus(grpc_error_to_absl_status(error))); } @@ -2583,7 +2878,7 @@ void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops, // compression not implemented GPR_ASSERT( !op.data.send_initial_metadata.maybe_compression_level.is_set); - if (!completed_) { + if (!completed()) { CToMetadata(op.data.send_initial_metadata.metadata, op.data.send_initial_metadata.count, send_initial_metadata_.get()); @@ -2593,7 +2888,8 @@ void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops, case GRPC_OP_RECV_INITIAL_METADATA: { recv_initial_metadata_ = op.data.recv_initial_metadata.recv_initial_metadata; - server_initial_metadata_ready_.emplace(server_initial_metadata_.Wait()); + server_initial_metadata_ready_.emplace( + server_initial_metadata_.receiver.Next()); recv_initial_metadata_completion_ = AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata); } break; @@ -2608,32 +2904,17 @@ void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops, recv_status_on_client_ = op.data.recv_status_on_client; } } break; - case GRPC_OP_SEND_MESSAGE: { - GPR_ASSERT(!outstanding_send_.has_value()); - if (!completed_) { - send_message_completion_ = - AddOpToCompletion(completion, PendingOp::kSendMessage); - SliceBuffer send; - grpc_slice_buffer_swap( - &op.data.send_message.send_message->data.raw.slice_buffer, - send.c_slice_buffer()); - outstanding_send_.emplace(client_to_server_messages_.sender.Push( - GetContext()->MakePooled(std::move(send), - op.flags))); - } else { - FailCompletion(completion); - } - } break; - case GRPC_OP_RECV_MESSAGE: { - GPR_ASSERT(!outstanding_recv_.has_value()); - recv_message_ = op.data.recv_message.recv_message; - recv_message_completion_ = - AddOpToCompletion(completion, PendingOp::kReceiveMessage); - outstanding_recv_.emplace(server_to_client_messages_.receiver.Next()); - } break; - case GRPC_OP_SEND_CLOSE_FROM_CLIENT: { - client_to_server_messages_.sender.Close(); - } break; + case GRPC_OP_SEND_MESSAGE: + StartSendMessage(op, completion, &client_to_server_messages_.sender); + break; + case GRPC_OP_RECV_MESSAGE: + StartRecvMessage(op, completion, &server_to_client_messages_.receiver); + break; + case GRPC_OP_SEND_CLOSE_FROM_CLIENT: + close_send_completion_ = + AddOpToCompletion(completion, PendingOp::kSendCloseFromClient); + GPR_ASSERT(close_send_completion_.has_value()); + break; case GRPC_OP_SEND_STATUS_FROM_SERVER: case GRPC_OP_RECV_CLOSE_ON_SERVER: abort(); // unreachable @@ -2668,78 +2949,48 @@ void ClientPromiseBasedCall::PublishInitialMetadata(ServerMetadata* metadata) { metadata->Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE); server_initial_metadata_ready_.reset(); GPR_ASSERT(recv_initial_metadata_ != nullptr); - PublishMetadataArray(std::exchange(recv_initial_metadata_, nullptr), - metadata); + PublishMetadataArray(metadata, + std::exchange(recv_initial_metadata_, nullptr)); FinishOpOnCompletion(&recv_initial_metadata_completion_, PendingOp::kReceiveInitialMetadata); } void ClientPromiseBasedCall::UpdateOnce() { if (grpc_call_trace.enabled()) { - auto present_and_completion_text = - [](const char* caption, bool has, - const Completion& completion) -> std::string { - if (has) { - if (completion.has_value()) { - return absl::StrCat(caption, ":", - static_cast(completion.index()), " "); - } else { - return absl::StrCat(caption, - ":!!BUG:operation is present, no completion!! "); - } - } else { - if (!completion.has_value()) { - return ""; - } else { - return absl::StrCat( - caption, ":no-op:", static_cast(completion.index()), " "); - } - } - }; - gpr_log( - GPR_INFO, "%sUpdateOnce: %s%s%shas_promise=%s", DebugTag().c_str(), - present_and_completion_text("server_initial_metadata_ready", - server_initial_metadata_ready_.has_value(), - recv_initial_metadata_completion_) - .c_str(), - present_and_completion_text("outstanding_send", - outstanding_send_.has_value(), - send_message_completion_) - .c_str(), - present_and_completion_text("outstanding_recv", - outstanding_recv_.has_value(), - recv_message_completion_) - .c_str(), - promise_.has_value() ? "true" : "false"); - } - if (send_message_completion_.has_value()) { - FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage); + gpr_log(GPR_INFO, "%s[call] UpdateOnce: %s%shas_promise=%s", + DebugTag().c_str(), + PresentAndCompletionText("server_initial_metadata_ready", + server_initial_metadata_ready_.has_value(), + recv_initial_metadata_completion_) + .c_str(), + PollStateDebugString().c_str(), + promise_.has_value() ? "true" : "false"); } if (server_initial_metadata_ready_.has_value()) { - Poll r = (*server_initial_metadata_ready_)(); - if (ServerMetadata*** server_initial_metadata = - absl::get_if(&r)) { - PublishInitialMetadata(**server_initial_metadata); - } else if (completed_) { + Poll> r = + (*server_initial_metadata_ready_)(); + if (auto* server_initial_metadata = + absl::get_if>(&r)) { + PublishInitialMetadata(server_initial_metadata->value().get()); + } else if (completed()) { ServerMetadata no_metadata{GetContext()}; PublishInitialMetadata(&no_metadata); } } - if (outstanding_send_.has_value()) { - Poll r = (*outstanding_send_)(); - if (const bool* result = absl::get_if(&r)) { - outstanding_send_.reset(); - if (!*result) { - FailCompletion(send_message_completion_); - Finish(ServerMetadataFromStatus(absl::Status( - absl::StatusCode::kInternal, "Failed to send message to server"))); - } - } + if (!PollSendMessage()) { + Finish(ServerMetadataFromStatus(absl::Status( + absl::StatusCode::kInternal, "Failed to send message to server"))); + } + if (!is_sending() && close_send_completion_.has_value()) { + client_to_server_messages_.sender.Close(); + FinishOpOnCompletion(&close_send_completion_, + PendingOp::kSendCloseFromClient); } if (promise_.has_value()) { Poll r = promise_(); if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sUpdateOnce: promise returns %s", DebugTag().c_str(), + gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s", + DebugTag().c_str(), PollToString(r, [](const ServerMetadataHandle& h) { return h->DebugString(); }).c_str()); @@ -2749,72 +3000,33 @@ void ClientPromiseBasedCall::UpdateOnce() { Finish(std::move(*result)); } } - if (incoming_compression_algorithm_.has_value() && - outstanding_recv_.has_value()) { - Poll> r = (*outstanding_recv_)(); - if (auto* result = absl::get_if>(&r)) { - outstanding_recv_.reset(); - if (result->has_value()) { - MessageHandle& message = **result; - if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) && - (incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) { - *recv_message_ = grpc_raw_compressed_byte_buffer_create( - nullptr, 0, *incoming_compression_algorithm_); - } else { - *recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0); - } - grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(), - &(*recv_message_)->data.raw.slice_buffer); - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%sUpdateOnce: outstanding_recv finishes: received %" PRIdPTR - " byte message", - DebugTag().c_str(), - (*recv_message_)->data.raw.slice_buffer.length); - } - } else { - if (grpc_call_trace.enabled()) { - gpr_log( - GPR_INFO, - "%sUpdateOnce: outstanding_recv finishes: received end-of-stream", - DebugTag().c_str()); - } - *recv_message_ = nullptr; - } - FinishOpOnCompletion(&recv_message_completion_, - PendingOp::kReceiveMessage); - } else if (completed_) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, - "%sUpdateOnce: outstanding_recv finishes: promise has " - "completed without queuing a message, forcing end-of-stream", - DebugTag().c_str()); - } - outstanding_recv_.reset(); - *recv_message_ = nullptr; - FinishOpOnCompletion(&recv_message_completion_, - PendingOp::kReceiveMessage); - } + if (incoming_compression_algorithm_.has_value()) { + PollRecvMessage(*incoming_compression_algorithm_); } } void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) { if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%sFinish: %s", DebugTag().c_str(), + gpr_log(GPR_INFO, "%s[call] Finish: %s", DebugTag().c_str(), trailing_metadata->DebugString().c_str()); } promise_ = ArenaPromise(); ResetDeadline(); - completed_ = true; + set_completed(); if (recv_initial_metadata_ != nullptr) { ForceImmediateRepoll(); } const bool pending_initial_metadata = server_initial_metadata_ready_.has_value(); + if (!pending_initial_metadata) { + server_initial_metadata_ready_.emplace( + server_initial_metadata_.receiver.Next()); + } + Poll> r = + (*server_initial_metadata_ready_)(); server_initial_metadata_ready_.reset(); - Poll r = server_initial_metadata_.Wait()(); - if (auto* result = absl::get_if(&r)) { - if (pending_initial_metadata) PublishInitialMetadata(**result); + if (auto* result = absl::get_if>(&r)) { + if (pending_initial_metadata) PublishInitialMetadata(result->value().get()); is_trailers_only_ = false; } else { if (pending_initial_metadata) { @@ -2886,7 +3098,7 @@ void ClientPromiseBasedCall::PublishStatus( *op_args.error_string = gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str()); } - PublishMetadataArray(op_args.trailing_metadata, trailing_metadata.get()); + PublishMetadataArray(trailing_metadata.get(), op_args.trailing_metadata); // Clear state saying we have a RECV_STATUS_ON_CLIENT outstanding // (so we don't call through twice) recv_status_on_client_ = absl::monostate(); @@ -2894,26 +3106,404 @@ void ClientPromiseBasedCall::PublishStatus( PendingOp::kReceiveStatusOnClient); } -void ClientPromiseBasedCall::PublishMetadataArray(grpc_metadata_array* array, - ServerMetadata* md) { - const auto md_count = md->count(); - if (md_count > array->capacity) { - array->capacity = - std::max(array->capacity + md->count(), array->capacity * 3 / 2); - array->metadata = static_cast( - gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity)); +/////////////////////////////////////////////////////////////////////////////// +// ServerPromiseBasedCall + +class ServerPromiseBasedCall final : public PromiseBasedCall { + public: + ServerPromiseBasedCall(Arena* arena, grpc_call_create_args* args); + + void CancelWithErrorLocked(grpc_error_handle) override + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); + grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag, + bool is_notify_tag_closure) override; + bool failed_before_recv_message() const override { abort(); } + bool is_trailers_only() const override { abort(); } + absl::string_view GetServerAuthority() const override { return ""; } + + // Polling order for the server promise stack: + // + // │ ┌───────────────────────────────────────┐ + // │ │ ServerPromiseBasedCall::UpdateOnce ├──► Lifetime management, + // │ ├───────────────────────────────────────┤ signal call end to app + // │ │ ConnectedChannel ├─┐ + // │ ├───────────────────────────────────────┤ └► Interactions with the + // │ │ ... closest to transport filter │ transport - send/recv msgs + // │ ├───────────────────────────────────────┤ and metadata, call phase + // │ │ ... │ ordering + // │ ├───────────────────────────────────────┤ + // │ │ ... closest to app filter │ ┌► Request matching, initial + // │ ├───────────────────────────────────────┤ │ setup, publishing call to + // │ │ Server::ChannelData::MakeCallPromise ├─┘ application + // │ ├───────────────────────────────────────┤ + // │ │ ServerPromiseBasedCall::PollTopOfCall ├──► Application interactions, + // ▼ └───────────────────────────────────────┘ forwarding messages, + // Polling & sending trailing metadata + // instantiation + // order + + void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override; + Poll PollTopOfCall() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); + + std::string DebugTag() const override { + return absl::StrFormat("SERVER_CALL[%p]: ", this); } - PublishToAppEncoder encoder(array); - md->Encode(&encoder); + + ServerCallContext* server_call_context() override { return &call_context_; } + + private: + class RecvCloseOpCancelState { + public: + // Request that receiver be filled in per grpc_op_recv_close_on_server. + // Returns true if the request can be fulfilled immediately. + // Returns false if the request will be fulfilled later. + bool ReceiveCloseOnServerOpStarted(int* receiver) { + switch (state_) { + case kUnset: + state_ = reinterpret_cast(receiver); + return false; + case kFinishedWithFailure: + *receiver = 1; + return true; + case kFinishedWithSuccess: + *receiver = 0; + return true; + default: + abort(); // unreachable + } + } + + // Mark the call as having completed. + // Returns true if this finishes a previous RequestReceiveCloseOnServer. + bool CompleteCall(bool success) { + switch (state_) { + case kUnset: + state_ = success ? kFinishedWithSuccess : kFinishedWithFailure; + return false; + case kFinishedWithFailure: + case kFinishedWithSuccess: + abort(); // unreachable + default: + *reinterpret_cast(state_) = success ? 0 : 1; + state_ = success ? kFinishedWithSuccess : kFinishedWithFailure; + return true; + } + } + + std::string ToString() const { + switch (state_) { + case kUnset: + return "Unset"; + case kFinishedWithFailure: + return "FinishedWithFailure"; + case kFinishedWithSuccess: + return "FinishedWithSuccess"; + default: + return absl::StrFormat("WaitingForReceiver(%p)", + reinterpret_cast(state_)); + } + } + + private: + static constexpr uintptr_t kUnset = 0; + static constexpr uintptr_t kFinishedWithFailure = 1; + static constexpr uintptr_t kFinishedWithSuccess = 2; + // Holds one of kUnset, kFinishedWithFailure, or kFinishedWithSuccess + // OR an int* that wants to receive the final status. + uintptr_t state_ = kUnset; + }; + + grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const; + void CommitBatch(const grpc_op* ops, size_t nops, + const Completion& completion) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); + + friend class ServerCallContext; + ServerCallContext call_context_; + Server* const server_; + ArenaPromise promise_ ABSL_GUARDED_BY(mu()); + PipeSender* server_to_client_messages_ ABSL_GUARDED_BY(mu()) = + nullptr; + PipeReceiver* client_to_server_messages_ + ABSL_GUARDED_BY(mu()) = nullptr; + using SendInitialMetadataState = + absl::variant*, + typename PipeSender::PushType>; + SendInitialMetadataState send_initial_metadata_state_ ABSL_GUARDED_BY(mu()) = + absl::monostate{}; + ServerMetadataHandle send_trailing_metadata_ ABSL_GUARDED_BY(mu()); + grpc_compression_algorithm incoming_compression_algorithm_ + ABSL_GUARDED_BY(mu()); + RecvCloseOpCancelState recv_close_op_cancel_state_ ABSL_GUARDED_BY(mu()); + Completion recv_close_completion_ ABSL_GUARDED_BY(mu()); + bool cancel_send_and_receive_ ABSL_GUARDED_BY(mu()) = false; + Completion send_status_from_server_completion_ ABSL_GUARDED_BY(mu()); + ClientMetadataHandle client_initial_metadata_ ABSL_GUARDED_BY(mu()); +}; + +ArenaPromise +ServerCallContext::MakeTopOfServerCallPromise( + CallArgs call_args, grpc_completion_queue* cq, + grpc_metadata_array* publish_initial_metadata, + absl::FunctionRef publish) { + call_->mu()->AssertHeld(); + call_->SetCompletionQueueLocked(cq); + call_->server_to_client_messages_ = call_args.server_to_client_messages; + call_->client_to_server_messages_ = call_args.client_to_server_messages; + call_->send_initial_metadata_state_ = call_args.server_initial_metadata; + call_->incoming_compression_algorithm_ = + call_args.client_initial_metadata->get(GrpcEncodingMetadata()) + .value_or(GRPC_COMPRESS_NONE); + call_->client_initial_metadata_ = + std::move(call_args.client_initial_metadata); + PublishMetadataArray(call_->client_initial_metadata_.get(), + publish_initial_metadata); + call_->ExternalRef(); + publish(call_->c_ptr()); + return [this]() { + call_->mu()->AssertHeld(); + return call_->PollTopOfCall(); + }; } -bool ClientPromiseBasedCall::Completed() { +ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena, + grpc_call_create_args* args) + : PromiseBasedCall(arena, 0, *args), + call_context_(this, args->server_transport_data), + server_(args->server) { + global_stats().IncrementServerCallsCreated(); + channelz::ServerNode* channelz_node = server_->channelz_node(); + if (channelz_node != nullptr) { + channelz_node->RecordCallStarted(); + } MutexLock lock(mu()); - return completed_; + ScopedContext activity_context(this); + promise_ = channel()->channel_stack()->MakeServerCallPromise( + CallArgs{nullptr, nullptr, nullptr, nullptr}); } -gpr_atm* CallContext::peer_string_atm_ptr() { - return call_->peer_string_atm_ptr(); +Poll ServerPromiseBasedCall::PollTopOfCall() { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[call] PollTopOfCall: %s%s%s", DebugTag().c_str(), + cancel_send_and_receive_ ? "force-" : "", + send_trailing_metadata_ != nullptr + ? absl::StrCat("send-metadata:", + send_trailing_metadata_->DebugString()) + .c_str() + : " ", + PollStateDebugString().c_str()); + } + + if (cancel_send_and_receive_) { + CancelSendMessage(); + CancelRecvMessage(); + } + + PollSendMessage(); + PollRecvMessage(incoming_compression_algorithm_); + + if (!is_sending() && send_trailing_metadata_ != nullptr) { + server_to_client_messages_->Close(); + return std::move(send_trailing_metadata_); + } + + return Pending{}; +} + +void ServerPromiseBasedCall::UpdateOnce() { + if (grpc_call_trace.enabled()) { + gpr_log( + GPR_INFO, "%s[call] UpdateOnce: recv_close:%s%s %s%shas_promise=%s", + DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(), + recv_close_completion_.has_value() + ? absl::StrCat(":", CompletionString(recv_close_completion_)) + .c_str() + : "", + send_status_from_server_completion_.has_value() + ? absl::StrCat( + "send_status:", + CompletionString(send_status_from_server_completion_), " ") + .c_str() + : "", + PollStateDebugString().c_str(), + promise_.has_value() ? "true" : "false"); + } + if (auto* p = + absl::get_if::PushType>( + &send_initial_metadata_state_)) { + if (!absl::holds_alternative((*p)())) { + send_initial_metadata_state_ = absl::monostate{}; + } + } + if (promise_.has_value()) { + auto r = promise_(); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s", + DebugTag().c_str(), + PollToString(r, [](const ServerMetadataHandle& h) { + return h->DebugString(); + }).c_str()); + } + if (auto* result = absl::get_if(&r)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[call] UpdateOnce: GotResult %s result:%s", + DebugTag().c_str(), + recv_close_op_cancel_state_.ToString().c_str(), + (*result)->DebugString().c_str()); + } + if (recv_close_op_cancel_state_.CompleteCall( + (*result)->get(GrpcStatusFromWire()).value_or(false))) { + FinishOpOnCompletion(&recv_close_completion_, + PendingOp::kReceiveCloseOnServer); + } + channelz::ServerNode* channelz_node = server_->channelz_node(); + if (channelz_node != nullptr) { + if ((*result) + ->get(GrpcStatusMetadata()) + .value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) { + channelz_node->RecordCallSucceeded(); + } else { + channelz_node->RecordCallFailed(); + } + } + if (send_status_from_server_completion_.has_value()) { + FinishOpOnCompletion(&send_status_from_server_completion_, + PendingOp::kSendStatusFromServer); + } + CancelSendMessage(); + CancelRecvMessage(); + set_completed(); + promise_ = ArenaPromise(); + } + } +} + +grpc_call_error ServerPromiseBasedCall::ValidateBatch(const grpc_op* ops, + size_t nops) const { + BitSet<8> got_ops; + for (size_t op_idx = 0; op_idx < nops; op_idx++) { + const grpc_op& op = ops[op_idx]; + switch (op.op) { + case GRPC_OP_SEND_INITIAL_METADATA: + if (!AreInitialMetadataFlagsValid(op.flags)) { + return GRPC_CALL_ERROR_INVALID_FLAGS; + } + if (!ValidateMetadata(op.data.send_initial_metadata.count, + op.data.send_initial_metadata.metadata)) { + return GRPC_CALL_ERROR_INVALID_METADATA; + } + break; + case GRPC_OP_SEND_MESSAGE: + if (!AreWriteFlagsValid(op.flags)) { + return GRPC_CALL_ERROR_INVALID_FLAGS; + } + break; + case GRPC_OP_RECV_MESSAGE: + case GRPC_OP_RECV_CLOSE_ON_SERVER: + case GRPC_OP_SEND_STATUS_FROM_SERVER: + if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS; + break; + case GRPC_OP_RECV_INITIAL_METADATA: + case GRPC_OP_SEND_CLOSE_FROM_CLIENT: + case GRPC_OP_RECV_STATUS_ON_CLIENT: + return GRPC_CALL_ERROR_NOT_ON_SERVER; + } + if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; + got_ops.set(op.op); + } + return GRPC_CALL_OK; +} + +void ServerPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops, + const Completion& completion) { + for (size_t op_idx = 0; op_idx < nops; op_idx++) { + const grpc_op& op = ops[op_idx]; + switch (op.op) { + case GRPC_OP_SEND_INITIAL_METADATA: { + // compression not implemented + GPR_ASSERT( + !op.data.send_initial_metadata.maybe_compression_level.is_set); + if (!completed()) { + auto metadata = arena()->MakePooled(arena()); + CToMetadata(op.data.send_initial_metadata.metadata, + op.data.send_initial_metadata.count, metadata.get()); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[call] Send initial metadata", + DebugTag().c_str()); + } + auto* pipe = absl::get*>( + send_initial_metadata_state_); + send_initial_metadata_state_ = pipe->Push(std::move(metadata)); + } + } break; + case GRPC_OP_SEND_MESSAGE: + StartSendMessage(op, completion, server_to_client_messages_); + break; + case GRPC_OP_RECV_MESSAGE: + StartRecvMessage(op, completion, client_to_server_messages_); + break; + case GRPC_OP_SEND_STATUS_FROM_SERVER: + send_trailing_metadata_ = arena()->MakePooled(arena()); + CToMetadata(op.data.send_status_from_server.trailing_metadata, + op.data.send_status_from_server.trailing_metadata_count, + send_trailing_metadata_.get()); + send_trailing_metadata_->Set(GrpcStatusMetadata(), + op.data.send_status_from_server.status); + if (auto* details = op.data.send_status_from_server.status_details) { + send_trailing_metadata_->Set(GrpcMessageMetadata(), + Slice(CSliceRef(*details))); + } + send_status_from_server_completion_ = + AddOpToCompletion(completion, PendingOp::kSendStatusFromServer); + break; + case GRPC_OP_RECV_CLOSE_ON_SERVER: + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[call] StartBatch: RecvClose %s", + DebugTag().c_str(), + recv_close_op_cancel_state_.ToString().c_str()); + } + if (!recv_close_op_cancel_state_.ReceiveCloseOnServerOpStarted( + op.data.recv_close_on_server.cancelled)) { + recv_close_completion_ = + AddOpToCompletion(completion, PendingOp::kReceiveCloseOnServer); + } + break; + case GRPC_OP_RECV_STATUS_ON_CLIENT: + case GRPC_OP_SEND_CLOSE_FROM_CLIENT: + case GRPC_OP_RECV_INITIAL_METADATA: + abort(); // unreachable + } + } +} + +grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops, + size_t nops, + void* notify_tag, + bool is_notify_tag_closure) { + MutexLock lock(mu()); + ScopedContext activity_context(this); + if (nops == 0) { + EndOpImmediately(cq(), notify_tag, is_notify_tag_closure); + return GRPC_CALL_OK; + } + const grpc_call_error validation_result = ValidateBatch(ops, nops); + if (validation_result != GRPC_CALL_OK) { + return validation_result; + } + Completion completion = + StartCompletion(notify_tag, is_notify_tag_closure, ops); + CommitBatch(ops, nops, completion); + Update(); + FinishOpOnCompletion(&completion, PendingOp::kStartingBatch); + return GRPC_CALL_OK; +} + +void ServerPromiseBasedCall::CancelWithErrorLocked(absl::Status error) { + if (!promise_.has_value()) return; + cancel_send_and_receive_ = true; + send_trailing_metadata_ = ServerMetadataFromStatus(error, arena()); + ForceWakeup(); } } // namespace grpc_core @@ -2933,11 +3523,14 @@ size_t grpc_call_get_initial_size_estimate() { grpc_error_handle grpc_call_create(grpc_call_create_args* args, grpc_call** out_call) { if (grpc_core::IsPromiseBasedClientCallEnabled() && - args->channel->is_promising()) { - if (args->server_transport_data == nullptr) { - return grpc_core::MakePromiseBasedCall( - args, out_call); - } + args->server_transport_data == nullptr && args->channel->is_promising()) { + return grpc_core::MakePromiseBasedCall( + args, out_call); + } + if (grpc_core::IsPromiseBasedServerCallEnabled() && + args->server_transport_data != nullptr && args->channel->is_promising()) { + return grpc_core::MakePromiseBasedCall( + args, out_call); } return grpc_core::FilterStackCall::Create(args, out_call); } diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index 92974700960..0a77709946d 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -25,6 +25,7 @@ #include #include "absl/functional/any_invocable.h" +#include "absl/functional/function_ref.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -43,12 +44,14 @@ #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/iomgr_fwd.h" #include "src/core/lib/iomgr/polling_entity.h" +#include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/context.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/server.h" +#include "src/core/lib/transport/transport.h" typedef void (*grpc_ioreq_completion_func)(grpc_call* call, int success, void* user_data); @@ -74,6 +77,28 @@ typedef struct grpc_call_create_args { namespace grpc_core { class PromiseBasedCall; +class ServerPromiseBasedCall; + +class ServerCallContext { + public: + ServerCallContext(ServerPromiseBasedCall* call, + const void* server_stream_data) + : call_(call), server_stream_data_(server_stream_data) {} + ArenaPromise MakeTopOfServerCallPromise( + CallArgs call_args, grpc_completion_queue* cq, + grpc_metadata_array* publish_initial_metadata, + absl::FunctionRef publish); + + // Server stream data as supplied by the transport (so we can link the + // transport stream up with the call again). + // TODO(ctiller): legacy API - once we move transports to promises we'll + // create the promise directly and not need to pass around this token. + const void* server_stream_data() { return server_stream_data_; } + + private: + ServerPromiseBasedCall* const call_; + const void* const server_stream_data_; +}; // TODO(ctiller): move more call things into this type class CallContext { @@ -98,6 +123,8 @@ class CallContext { gpr_atm* peer_string_atm_ptr(); grpc_polling_entity* polling_entity() { return &pollent_; } + ServerCallContext* server_call_context(); + private: friend class PromiseBasedCall; // Call final info. @@ -112,6 +139,7 @@ class CallContext { template <> struct ContextType {}; + } // namespace grpc_core // Create a new call based on \a args. diff --git a/src/core/lib/surface/call_trace.cc b/src/core/lib/surface/call_trace.cc index f2798aefbc0..7f5a07e1b52 100644 --- a/src/core/lib/surface/call_trace.cc +++ b/src/core/lib/surface/call_trace.cc @@ -25,6 +25,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/meta/type_traits.h" #include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "absl/types/variant.h" #include @@ -45,7 +46,8 @@ const grpc_channel_filter* PromiseTracingFilterFor( struct DerivedFilter : public grpc_channel_filter { explicit DerivedFilter(const grpc_channel_filter* filter) : grpc_channel_filter{ - /* start_transport_stream_op_batch: */ grpc_call_next_op, + // start_transport_stream_op_batch: + grpc_call_next_op, // make_call_promise: [](grpc_channel_element* elem, CallArgs call_args, NextPromiseFactory next_promise_factory) @@ -54,22 +56,22 @@ const grpc_channel_filter* PromiseTracingFilterFor( static_cast(elem->filter)->filter; gpr_log( GPR_DEBUG, - "%sCreateCallPromise[%s]: client_initial_metadata=%s", + "%s[%s] CreateCallPromise: client_initial_metadata=%s", Activity::current()->DebugTag().c_str(), source_filter->name, call_args.client_initial_metadata->DebugString().c_str()); return [source_filter, child = next_promise_factory( std::move(call_args))]() mutable { - gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: begin", + gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: begin", Activity::current()->DebugTag().c_str(), source_filter->name); auto r = child(); if (auto* p = absl::get_if(&r)) { - gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: done: %s", + gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: done: %s", Activity::current()->DebugTag().c_str(), source_filter->name, (*p)->DebugString().c_str()); } else { - gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: <", + gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: <>", Activity::current()->DebugTag().c_str(), source_filter->name); } @@ -85,16 +87,24 @@ const grpc_channel_filter* PromiseTracingFilterFor( // destroy_call_elem: [](grpc_call_element*, const grpc_call_final_info*, grpc_closure*) {}, - /* sizeof_channel_data: */ 0, // init_channel_elem: + // sizeof_channel_data: + 0, + // init_channel_elem: [](grpc_channel_element*, grpc_channel_element_args*) { return absl::OkStatus(); }, // post_init_channel_elem: [](grpc_channel_stack*, grpc_channel_element*) {}, - /* destroy_channel_elem: */ [](grpc_channel_element*) {}, - grpc_channel_next_get_info, filter->name}, - filter(filter) {} + // destroy_channel_elem: + [](grpc_channel_element*) {}, grpc_channel_next_get_info, + // name: + nullptr}, + filter(filter), + name_str(absl::StrCat(filter->name, ".trace")) { + this->name = name_str.c_str(); + } const grpc_channel_filter* const filter; + const std::string name_str; }; struct Globals { Mutex mu; diff --git a/src/core/lib/surface/lame_client.cc b/src/core/lib/surface/lame_client.cc index 181beb9b4b1..ecbc9eed098 100644 --- a/src/core/lib/surface/lame_client.cc +++ b/src/core/lib/surface/lame_client.cc @@ -76,7 +76,9 @@ ArenaPromise LameClientFilter::MakeCallPromise( CallArgs args, NextPromiseFactory) { // TODO(ctiller): remove if check once promise_based_filter is removed (Close // is still needed) - if (args.incoming_messages != nullptr) args.incoming_messages->Close(); + if (args.server_to_client_messages != nullptr) { + args.server_to_client_messages->Close(); + } return Immediate(ServerMetadataFromStatus(error_)); } diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index 985342265f0..319039accaf 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -24,14 +24,19 @@ #include #include +#include #include #include #include +#include +#include #include #include +#include "absl/cleanup/cleanup.h" #include "absl/status/status.h" #include "absl/types/optional.h" +#include "absl/types/variant.h" #include #include @@ -44,13 +49,26 @@ #include "src/core/lib/channel/channel_trace.h" #include "src/core/lib/channel/channelz.h" #include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/crash.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/match.h" #include "src/core/lib/gprpp/mpscq.h" #include "src/core/lib/gprpp/status_helper.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/pollset_set.h" +#include "src/core/lib/promise/activity.h" +#include "src/core/lib/promise/context.h" +#include "src/core/lib/promise/detail/basic_join.h" +#include "src/core/lib/promise/detail/basic_seq.h" +#include "src/core/lib/promise/map.h" +#include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/promise.h" +#include "src/core/lib/promise/try_join.h" +#include "src/core/lib/promise/try_seq.h" +#include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/call.h" @@ -177,6 +195,20 @@ class Server::RequestMatcherInterface { virtual void RequestCallWithPossiblePublish(size_t request_queue_index, RequestedCall* call) = 0; + struct MatchResult { + size_t cq_idx; + RequestedCall* requested_call; + }; + + // This function is invoked on an incoming promise based RPC. + // The RequestMatcher will try to match it against an application-requested + // RPC if possible or will place it in the pending queue otherwise. To enable + // some measure of fairness between server CQs, the match is done starting at + // the start_request_queue_index parameter in a cyclic order rather than + // always starting at 0. + virtual ArenaPromise> MatchRequest( + size_t start_request_queue_index) = 0; + // This function is invoked on an incoming RPC, represented by the calld // object. The RequestMatcher will try to match it against an // application-requested RPC if possible or will place it in the pending queue @@ -208,9 +240,15 @@ class Server::RealRequestMatcher : public RequestMatcherInterface { void ZombifyPending() override { while (!pending_.empty()) { - CallData* calld = pending_.front(); - calld->SetState(CallData::CallState::ZOMBIED); - calld->KillZombie(); + Match( + pending_.front(), + [](CallData* calld) { + calld->SetState(CallData::CallState::ZOMBIED); + calld->KillZombie(); + }, + [](const std::shared_ptr& w) { + w->Finish(absl::InternalError("Server closed")); + }); pending_.pop(); } } @@ -234,19 +272,19 @@ class Server::RealRequestMatcher : public RequestMatcherInterface { if (requests_per_cq_[request_queue_index].Push(&call->mpscq_node)) { // this was the first queued request: we need to lock and start // matching calls - struct PendingCall { + struct NextPendingCall { RequestedCall* rc = nullptr; - CallData* calld; + PendingCall pending; }; auto pop_next_pending = [this, request_queue_index] { - PendingCall pending_call; + NextPendingCall pending_call; { MutexLock lock(&server_->mu_call_); if (!pending_.empty()) { pending_call.rc = reinterpret_cast( requests_per_cq_[request_queue_index].Pop()); if (pending_call.rc != nullptr) { - pending_call.calld = pending_.front(); + pending_call.pending = std::move(pending_.front()); pending_.pop(); } } @@ -254,14 +292,20 @@ class Server::RealRequestMatcher : public RequestMatcherInterface { return pending_call; }; while (true) { - PendingCall next_pending = pop_next_pending(); + NextPendingCall next_pending = pop_next_pending(); if (next_pending.rc == nullptr) break; - if (!next_pending.calld->MaybeActivate()) { - // Zombied Call - next_pending.calld->KillZombie(); - } else { - next_pending.calld->Publish(request_queue_index, next_pending.rc); - } + auto mr = MatchResult{request_queue_index, next_pending.rc}; + Match( + next_pending.pending, + [mr](CallData* calld) { + if (!calld->MaybeActivate()) { + // Zombied Call + calld->KillZombie(); + } else { + calld->Publish(mr.cq_idx, mr.requested_call); + } + }, + [mr](const std::shared_ptr& w) { w->Finish(mr); }); } } } @@ -306,11 +350,66 @@ class Server::RealRequestMatcher : public RequestMatcherInterface { calld->Publish(cq_idx, rc); } + ArenaPromise> MatchRequest( + size_t start_request_queue_index) override { + for (size_t i = 0; i < requests_per_cq_.size(); i++) { + size_t cq_idx = (start_request_queue_index + i) % requests_per_cq_.size(); + RequestedCall* rc = + reinterpret_cast(requests_per_cq_[cq_idx].TryPop()); + if (rc != nullptr) { + return Immediate(MatchResult{cq_idx, rc}); + } + } + // No cq to take the request found; queue it on the slow list. + // We need to ensure that all the queues are empty. We do this under + // the server mu_call_ lock to ensure that if something is added to + // an empty request queue, it will block until the call is actually + // added to the pending list. + RequestedCall* rc = nullptr; + size_t cq_idx = 0; + size_t loop_count; + { + MutexLock lock(&server_->mu_call_); + for (loop_count = 0; loop_count < requests_per_cq_.size(); loop_count++) { + cq_idx = + (start_request_queue_index + loop_count) % requests_per_cq_.size(); + rc = reinterpret_cast(requests_per_cq_[cq_idx].Pop()); + if (rc != nullptr) { + break; + } + } + if (rc == nullptr) { + auto w = std::make_shared( + Activity::current()->MakeNonOwningWaker()); + pending_.push(w); + return [w]() -> Poll> { + std::unique_ptr> r( + w->result.exchange(nullptr, std::memory_order_acq_rel)); + if (r == nullptr) return Pending{}; + return std::move(*r); + }; + } + } + return Immediate(MatchResult{cq_idx, rc}); + } + Server* server() const override { return server_; } private: Server* const server_; - std::queue pending_; + struct ActivityWaiter { + explicit ActivityWaiter(Waker waker) : waker(std::move(waker)) {} + ~ActivityWaiter() { delete result.load(std::memory_order_acquire); } + void Finish(absl::StatusOr r) { + result.store(new absl::StatusOr(std::move(r)), + std::memory_order_release); + waker.Wakeup(); + } + Waker waker; + std::atomic*> result{nullptr}; + }; + using PendingCall = absl::variant>; + std::queue pending_; std::vector requests_per_cq_; }; @@ -371,7 +470,10 @@ class Server::AllocatingRequestMatcherBatch void MatchOrQueue(size_t /*start_request_queue_index*/, CallData* calld) override { - if (server()->ShutdownRefOnRequest()) { + const bool still_running = server()->ShutdownRefOnRequest(); + auto cleanup_ref = + absl::MakeCleanup([this] { server()->ShutdownUnrefOnRequest(); }); + if (still_running) { BatchCallAllocation call_info = allocator_(); GPR_ASSERT(server()->ValidateServerRequest( cq(), static_cast(call_info.tag), nullptr, @@ -384,7 +486,25 @@ class Server::AllocatingRequestMatcherBatch } else { calld->FailCallCreation(); } - server()->ShutdownUnrefOnRequest(); + } + + ArenaPromise> MatchRequest( + size_t /*start_request_queue_index*/) override { + const bool still_running = server()->ShutdownRefOnRequest(); + auto cleanup_ref = + absl::MakeCleanup([this] { server()->ShutdownUnrefOnRequest(); }); + if (still_running) { + BatchCallAllocation call_info = allocator_(); + GPR_ASSERT(server()->ValidateServerRequest( + cq(), static_cast(call_info.tag), nullptr, + nullptr) == GRPC_CALL_OK); + RequestedCall* rc = new RequestedCall( + static_cast(call_info.tag), call_info.cq, call_info.call, + call_info.initial_metadata, call_info.details); + return Immediate(MatchResult{cq_idx(), rc}); + } else { + return Immediate(absl::InternalError("Server shutdown")); + } } private: @@ -404,6 +524,8 @@ class Server::AllocatingRequestMatcherRegistered void MatchOrQueue(size_t /*start_request_queue_index*/, CallData* calld) override { + auto cleanup_ref = + absl::MakeCleanup([this] { server()->ShutdownUnrefOnRequest(); }); if (server()->ShutdownRefOnRequest()) { RegisteredCallAllocation call_info = allocator_(); GPR_ASSERT(server()->ValidateServerRequest( @@ -418,7 +540,26 @@ class Server::AllocatingRequestMatcherRegistered } else { calld->FailCallCreation(); } - server()->ShutdownUnrefOnRequest(); + } + + ArenaPromise> MatchRequest( + size_t /*start_request_queue_index*/) override { + const bool still_running = server()->ShutdownRefOnRequest(); + auto cleanup_ref = + absl::MakeCleanup([this] { server()->ShutdownUnrefOnRequest(); }); + if (still_running) { + RegisteredCallAllocation call_info = allocator_(); + GPR_ASSERT(server()->ValidateServerRequest( + cq(), call_info.tag, call_info.optional_payload, + registered_method_) == GRPC_CALL_OK); + RequestedCall* rc = + new RequestedCall(call_info.tag, call_info.cq, call_info.call, + call_info.initial_metadata, registered_method_, + call_info.deadline, call_info.optional_payload); + return Immediate(MatchResult{cq_idx(), rc}); + } else { + return Immediate(absl::InternalError("Server shutdown")); + } } private: @@ -493,7 +634,7 @@ class ChannelBroadcaster { const grpc_channel_filter Server::kServerTopFilter = { Server::CallData::StartTransportStreamOpBatch, - nullptr, + Server::ChannelData::MakeCallPromise, grpc_channel_next_op, sizeof(Server::CallData), Server::CallData::InitCallElement, @@ -1100,14 +1241,119 @@ void Server::ChannelData::AcceptStream(void* arg, grpc_transport* /*transport*/, args.send_deadline = Timestamp::InfFuture(); grpc_call* call; grpc_error_handle error = grpc_call_create(&args, &call); - grpc_call_element* elem = - grpc_call_stack_element(grpc_call_get_call_stack(call), 0); - auto* calld = static_cast(elem->call_data); - if (!error.ok()) { - calld->FailCallCreation(); + grpc_call_stack* call_stack = grpc_call_get_call_stack(call); + if (call_stack == nullptr) { // Promise based calls do not have a call stack + GPR_ASSERT(error.ok()); + GPR_ASSERT(IsPromiseBasedServerCallEnabled()); return; - } - calld->Start(elem); + } else { + grpc_call_element* elem = grpc_call_stack_element(call_stack, 0); + auto* calld = static_cast(elem->call_data); + if (!error.ok()) { + calld->FailCallCreation(); + return; + } + calld->Start(elem); + } +} + +ArenaPromise Server::ChannelData::MakeCallPromise( + grpc_channel_element* elem, CallArgs call_args, NextPromiseFactory) { + auto* chand = static_cast(elem->channel_data); + auto* server = chand->server_.get(); + if (server->ShutdownCalled()) { + return [] { + return ServerMetadataFromStatus(absl::InternalError("Server shutdown")); + }; + } + absl::optional path = + call_args.client_initial_metadata->Take(HttpPathMetadata()); + if (!path.has_value()) { + return [] { + return ServerMetadataFromStatus( + absl::InternalError("Missing :path header")); + }; + } + auto host_ptr = + call_args.client_initial_metadata->get_pointer(HttpAuthorityMetadata()); + if (host_ptr == nullptr) { + return [] { + return ServerMetadataFromStatus( + absl::InternalError("Missing :authority header")); + }; + } + // TODO(ctiller): deadline handling + Timestamp deadline = Timestamp::InfFuture(); + // Find request matcher. + RequestMatcherInterface* matcher; + ChannelRegisteredMethod* rm = + chand->GetRegisteredMethod(host_ptr->c_slice(), path->c_slice()); + ArenaPromise>> + maybe_read_first_message([] { return NextResult(); }); + if (rm != nullptr) { + matcher = rm->server_registered_method->matcher.get(); + switch (rm->server_registered_method->payload_handling) { + case GRPC_SRM_PAYLOAD_NONE: + break; + case GRPC_SRM_PAYLOAD_READ_INITIAL_BYTE_BUFFER: + maybe_read_first_message = + Map(call_args.client_to_server_messages->Next(), + [](NextResult msg) + -> absl::StatusOr> { + return std::move(msg); + }); + } + } else { + matcher = server->unregistered_request_matcher_.get(); + } + return TrySeq( + TryJoin(matcher->MatchRequest(chand->cq_idx()), + std::move(maybe_read_first_message)), + [path = std::move(*path), host = std::move(*host_ptr), deadline, server, + call_args = std::move(call_args)]( + std::tuple> + match_result_and_payload) mutable { + auto& mr = std::get<0>(match_result_and_payload); + auto& payload = std::get<1>(match_result_and_payload); + auto* rc = mr.requested_call; + auto* cq_for_new_request = server->cqs_[mr.cq_idx]; + switch (rc->type) { + case RequestedCall::Type::BATCH_CALL: + GPR_ASSERT(!payload.has_value()); + rc->data.batch.details->host = CSliceRef(host.c_slice()); + rc->data.batch.details->method = CSliceRef(path.c_slice()); + rc->data.batch.details->deadline = + deadline.as_timespec(GPR_CLOCK_MONOTONIC); + break; + case RequestedCall::Type::REGISTERED_CALL: + *rc->data.registered.deadline = + deadline.as_timespec(GPR_CLOCK_MONOTONIC); + if (rc->data.registered.optional_payload != nullptr) { + if (payload.has_value()) { + auto* sb = payload.value()->payload()->c_slice_buffer(); + *rc->data.registered.optional_payload = + grpc_raw_byte_buffer_create(sb->slices, sb->count); + } else { + *rc->data.registered.optional_payload = nullptr; + } + } + break; + default: + GPR_UNREACHABLE_CODE(abort()); + } + return GetContext() + ->server_call_context() + ->MakeTopOfServerCallPromise( + std::move(call_args), rc->cq_bound_to_call, + rc->initial_metadata, + [rc, cq_for_new_request](grpc_call* call) { + *rc->call = call; + grpc_cq_end_op(cq_for_new_request, rc->tag, absl::OkStatus(), + Server::DoneRequestEvent, rc, &rc->completion, + true); + }); + }); } void Server::ChannelData::FinishDestroy(void* arg, @@ -1273,6 +1519,7 @@ void Server::CallData::KillZombie() { ExecCtx::Run(DEBUG_LOCATION, &kill_zombie_closure_, absl::OkStatus()); } +// If this changes, change MakeCallPromise too. void Server::CallData::StartNewRpc(grpc_call_element* elem) { auto* chand = static_cast(elem->channel_data); if (server_->ShutdownCalled()) { diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index 35f5cc5a27f..d635da585b7 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -56,6 +56,7 @@ #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/iomgr_fwd.h" +#include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/slice/slice.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/completion_queue.h" @@ -236,6 +237,8 @@ class Server : public InternallyRefCounted, static grpc_error_handle InitChannelElement( grpc_channel_element* elem, grpc_channel_element_args* args); static void DestroyChannelElement(grpc_channel_element* elem); + static ArenaPromise MakeCallPromise( + grpc_channel_element* elem, CallArgs call_args, NextPromiseFactory); private: class ConnectivityWatcher; diff --git a/src/core/lib/transport/transport.cc b/src/core/lib/transport/transport.cc index beff91dd21d..6e2e48ac24c 100644 --- a/src/core/lib/transport/transport.cc +++ b/src/core/lib/transport/transport.cc @@ -32,7 +32,6 @@ #include "src/core/lib/event_engine/default_event_engine.h" #include "src/core/lib/gpr/alloc.h" #include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/promise/context.h" #include "src/core/lib/slice/slice.h" #include "src/core/lib/transport/transport_impl.h" @@ -269,9 +268,9 @@ grpc_transport_stream_op_batch* grpc_make_transport_stream_op( namespace grpc_core { -ServerMetadataHandle ServerMetadataFromStatus(const absl::Status& status) { - auto hdl = - GetContext()->MakePooled(GetContext()); +ServerMetadataHandle ServerMetadataFromStatus(const absl::Status& status, + Arena* arena) { + auto hdl = arena->MakePooled(arena); hdl->Set(GrpcStatusMetadata(), static_cast(status.code())); if (!status.ok()) { hdl->Set(GrpcMessageMetadata(), Slice::FromCopiedString(status.message())); diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 3be543a59ff..bcd151cc6af 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -52,8 +52,8 @@ #include "src/core/lib/iomgr/iomgr_fwd.h" #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/promise/arena_promise.h" +#include "src/core/lib/promise/context.h" #include "src/core/lib/promise/detail/status.h" -#include "src/core/lib/promise/latch.h" #include "src/core/lib/promise/pipe.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice_buffer.h" @@ -120,7 +120,8 @@ inline bool IsStatusOk(const ServerMetadataHandle& m) { GRPC_STATUS_OK; } -ServerMetadataHandle ServerMetadataFromStatus(const absl::Status& status); +ServerMetadataHandle ServerMetadataFromStatus( + const absl::Status& status, Arena* arena = GetContext()); template <> struct StatusCastImpl { @@ -136,6 +137,13 @@ struct StatusCastImpl { } }; +template <> +struct StatusCastImpl { + static ServerMetadataHandle Cast(const absl::Status& m) { + return ServerMetadataFromStatus(m); + } +}; + struct CallArgs { // Initial metadata from the client to the server. // During promise setup this can be manipulated by filters (and then @@ -145,11 +153,11 @@ struct CallArgs { // Set once when it's available. // During promise setup filters can substitute their own latch for this // and consequently intercept the sent value and mutate/observe it. - Latch* server_initial_metadata; + PipeSender* server_initial_metadata; // Messages travelling from the application to the transport. - PipeReceiver* outgoing_messages; + PipeReceiver* client_to_server_messages; // Messages travelling from the transport to the application. - PipeSender* incoming_messages; + PipeSender* server_to_client_messages; }; using NextPromiseFactory = diff --git a/src/cpp/ext/filters/logging/BUILD b/src/cpp/ext/filters/logging/BUILD index dff7ba34dbc..8d7b3440224 100644 --- a/src/cpp/ext/filters/logging/BUILD +++ b/src/cpp/ext/filters/logging/BUILD @@ -55,7 +55,6 @@ grpc_cc_library( ], external_deps = [ "absl/random", - "absl/status", "absl/status:statusor", "absl/strings", "absl/types:optional", @@ -74,22 +73,18 @@ grpc_cc_library( "//:grpc_base", "//:grpc_client_channel", "//:grpc_resolver", - "//:promise", "//:uri_parser", "//src/core:arena", "//src/core:arena_promise", - "//src/core:basic_seq", "//src/core:cancel_callback", "//src/core:channel_args", "//src/core:channel_fwd", "//src/core:channel_stack_type", "//src/core:context", - "//src/core:for_each", - "//src/core:latch", - "//src/core:promise_like", - "//src/core:seq", + "//src/core:map", + "//src/core:pipe", + "//src/core:poll", "//src/core:slice", "//src/core:slice_buffer", - "//src/core:try_concurrently", ], ) diff --git a/src/cpp/ext/filters/logging/logging_filter.cc b/src/cpp/ext/filters/logging/logging_filter.cc index 2e664e92d18..9725d80d38a 100644 --- a/src/cpp/ext/filters/logging/logging_filter.cc +++ b/src/cpp/ext/filters/logging/logging_filter.cc @@ -34,7 +34,6 @@ #include #include "absl/random/random.h" -#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" @@ -58,14 +57,9 @@ #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/cancel_callback.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/detail/basic_seq.h" -#include "src/core/lib/promise/detail/promise_like.h" -#include "src/core/lib/promise/for_each.h" -#include "src/core/lib/promise/latch.h" -#include "src/core/lib/promise/map_pipe.h" -#include "src/core/lib/promise/promise.h" -#include "src/core/lib/promise/seq.h" -#include "src/core/lib/promise/try_concurrently.h" +#include "src/core/lib/promise/map.h" +#include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/poll.h" #include "src/core/lib/resolver/resolver_registry.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" @@ -365,52 +359,28 @@ class ClientLoggingFilter final : public grpc_core::ChannelFilter { } calld->LogClientHeader(/*is_client=*/true, call_args.client_initial_metadata); - auto* server_initial_metadata = call_args.server_initial_metadata; - auto incoming_mapper = - grpc_core::PipeMapper::Intercept( - *call_args.incoming_messages); - grpc_core::PipeMapper outgoing_mapper = - grpc_core::PipeMapper::Intercept( - *call_args.outgoing_messages); + call_args.server_initial_metadata->InterceptAndMap( + [calld](grpc_core::ServerMetadataHandle metadata) { + calld->LogServerHeader(/*is_client=*/true, metadata.get()); + return metadata; + }); + call_args.client_to_server_messages->InterceptAndMapWithHalfClose( + [calld](grpc_core::MessageHandle message) { + calld->LogClientMessage(/*is_client=*/true, message->payload()); + return message; + }, + [calld] { calld->LogClientHalfClose(/*is_client=*/true); }); + call_args.server_to_client_messages->InterceptAndMap( + [calld](grpc_core::MessageHandle message) { + calld->LogServerMessage(/*is_client=*/true, message->payload()); + return message; + }); return grpc_core::OnCancel( - grpc_core::TryConcurrently( - grpc_core::Seq( - next_promise_factory(std::move(call_args)), - [calld](grpc_core::ServerMetadataHandle metadata) mutable - -> grpc_core::ServerMetadataHandle { - calld->LogServerTrailer(/*is_client=*/true, metadata.get()); - return metadata; - })) - .NecessaryPull(grpc_core::Seq( - server_initial_metadata->Wait(), - [calld]( - grpc_core::ServerMetadata** server_initial_metadata) mutable - -> grpc_core::ArenaPromise { - if (server_initial_metadata != nullptr) { - calld->LogServerHeader(/*is_client=*/true, - *server_initial_metadata); - } - return grpc_core::ImmediateOkStatus(); - })) - .NecessaryPull(incoming_mapper.TakeAndRun( - [calld](grpc_core::MessageHandle message) - -> absl::StatusOr { - calld->LogServerMessage(/*is_client=*/true, - message->payload()); - return message; - })) - .NecessaryPush(grpc_core::Seq( - outgoing_mapper.TakeAndRun( - [calld](grpc_core::MessageHandle message) - -> absl::StatusOr { - calld->LogClientMessage(/*is_client=*/true, - message->payload()); - return message; - }), - [calld]() mutable -> grpc_core::ArenaPromise { - calld->LogClientHalfClose(/*is_client=*/true); - return grpc_core::ImmediateOkStatus(); - })), + Map(next_promise_factory(std::move(call_args)), + [calld](grpc_core::ServerMetadataHandle md) { + calld->LogServerTrailer(/*is_client=*/true, md.get()); + return md; + }), [calld]() { calld->LogCancel(/*is_client=*/true); }); } @@ -449,50 +419,28 @@ class ServerLoggingFilter final : public grpc_core::ChannelFilter { } calld->LogClientHeader(/*is_client=*/false, call_args.client_initial_metadata); - auto* server_initial_metadata = call_args.server_initial_metadata; - auto incoming_mapper = - grpc_core::PipeMapper::Intercept( - *call_args.incoming_messages); - grpc_core::PipeMapper outgoing_mapper = - grpc_core::PipeMapper::Intercept( - *call_args.outgoing_messages); + call_args.server_initial_metadata->InterceptAndMap( + [calld](grpc_core::ServerMetadataHandle metadata) { + calld->LogServerHeader(/*is_client=*/false, metadata.get()); + return metadata; + }); + call_args.client_to_server_messages->InterceptAndMapWithHalfClose( + [calld](grpc_core::MessageHandle message) { + calld->LogClientMessage(/*is_client=*/false, message->payload()); + return message; + }, + [calld] { calld->LogClientHalfClose(/*is_client=*/false); }); + call_args.server_to_client_messages->InterceptAndMap( + [calld](grpc_core::MessageHandle message) { + calld->LogServerMessage(/*is_client=*/false, message->payload()); + return message; + }); return grpc_core::OnCancel( - grpc_core::TryConcurrently( - grpc_core::Seq( - next_promise_factory(std::move(call_args)), - [calld](grpc_core::ServerMetadataHandle metadata) mutable - -> grpc_core::ServerMetadataHandle { - calld->LogServerTrailer(/*is_client=*/false, metadata.get()); - return metadata; - })) - .Push(grpc_core::Seq( - server_initial_metadata->Wait(), - [calld]( - grpc_core::ServerMetadata** server_initial_metadata) mutable - -> grpc_core::ArenaPromise { - calld->LogServerHeader(/*is_client=*/false, - *server_initial_metadata); - return grpc_core::ImmediateOkStatus(); - })) - .Push(outgoing_mapper.TakeAndRun( - [calld](grpc_core::MessageHandle message) - -> absl::StatusOr { - calld->LogServerMessage(/*is_client=*/false, - message->payload()); - return message; - })) - .NecessaryPull(grpc_core::Seq( - incoming_mapper.TakeAndRun( - [calld](grpc_core::MessageHandle message) - -> absl::StatusOr { - calld->LogClientMessage(/*is_client=*/false, - message->payload()); - return message; - }), - [calld]() mutable -> grpc_core::ArenaPromise { - calld->LogClientHalfClose(/*is_client=*/false); - return grpc_core::ImmediateOkStatus(); - })), + Map(next_promise_factory(std::move(call_args)), + [calld](grpc_core::ServerMetadataHandle md) { + calld->LogServerTrailer(/*is_client=*/false, md.get()); + return md; + }), [calld]() { calld->LogCancel(/*is_client=*/false); }); } }; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 80fd72a77a0..6eb36ade19b 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -573,6 +573,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/cfstream_handle.cc', + 'src/core/lib/iomgr/closure.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/dualstack_socket_posix.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -654,8 +655,8 @@ CORE_SOURCE_FILES = [ 'src/core/lib/load_balancing/lb_policy_registry.cc', 'src/core/lib/matchers/matchers.cc', 'src/core/lib/promise/activity.cc', - 'src/core/lib/promise/pipe.cc', 'src/core/lib/promise/sleep.cc', + 'src/core/lib/promise/trace.cc', 'src/core/lib/resolver/resolver.cc', 'src/core/lib/resolver/resolver_registry.cc', 'src/core/lib/resolver/server_address.cc', @@ -733,6 +734,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/slice/percent_encoding.cc', 'src/core/lib/slice/slice.cc', 'src/core/lib/slice/slice_buffer.cc', + 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', 'src/core/lib/surface/builtins.cc', diff --git a/test/core/end2end/fixtures/h2_sockpair_with_minstack.cc b/test/core/end2end/fixtures/h2_sockpair_with_minstack.cc index 84c93922c5b..5eb130e7474 100644 --- a/test/core/end2end/fixtures/h2_sockpair_with_minstack.cc +++ b/test/core/end2end/fixtures/h2_sockpair_with_minstack.cc @@ -130,15 +130,16 @@ static void chttp2_init_server_socketpair( auto* fixture_data = static_cast(f->fixture_data); grpc_transport* transport; GPR_ASSERT(!f->server); - f->server = grpc_server_create(server_args, nullptr); + f->server = grpc_server_create(grpc_core::ChannelArgs::FromC(server_args) + .Set(GRPC_ARG_MINIMAL_STACK, true) + .ToC() + .get(), + nullptr); grpc_server_register_completion_queue(f->server, f->cq, nullptr); grpc_server_start(f->server); - auto final_server_args = grpc_core::CoreConfiguration::Get() - .channel_args_preconditioning() - .PreconditionChannelArgs(server_args) - .Set(GRPC_ARG_MINIMAL_STACK, true); - transport = grpc_create_chttp2_transport(final_server_args, - fixture_data->ep.server, false); + transport = grpc_create_chttp2_transport( + grpc_core::Server::FromC(f->server)->channel_args(), + fixture_data->ep.server, false); server_setup_transport(f, transport); } diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index a901ae39fdf..f833fbc33bf 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -261,7 +261,7 @@ END2END_TESTS = { exclude_inproc = True, exclude_minstack = True, ), - "max_connection_age": _test_options(exclude_inproc = True), + "max_connection_age": _test_options(exclude_minstack = True, exclude_inproc = True), "max_connection_idle": _test_options(needs_fullstack = True, proxyable = False), "max_message_length": _test_options(exclude_minstack = True), "negative_deadline": _test_options(exclude_minstack = True), diff --git a/test/core/end2end/tests/cancel_after_accept.cc b/test/core/end2end/tests/cancel_after_accept.cc index f70b599e3a8..c440c154d0e 100644 --- a/test/core/end2end/tests/cancel_after_accept.cc +++ b/test/core/end2end/tests/cancel_after_accept.cc @@ -19,6 +19,8 @@ #include #include +#include + #include #include #include @@ -44,6 +46,7 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, grpc_channel_args* client_args, grpc_channel_args* server_args) { grpc_end2end_test_fixture f; + gpr_log(GPR_INFO, "%s", std::string(80, '*').c_str()); gpr_log(GPR_INFO, "Running test: %s/%s/%s/%s", test_name, config.name, mode.name, use_service_config ? "service_config" : "client_api"); f = config.create_fixture(client_args, server_args); diff --git a/test/core/end2end/tests/grpc_authz.cc b/test/core/end2end/tests/grpc_authz.cc index 4536faa2332..90479dec716 100644 --- a/test/core/end2end/tests/grpc_authz.cc +++ b/test/core/end2end/tests/grpc_authz.cc @@ -48,6 +48,7 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, grpc_channel_args* client_args, grpc_channel_args* server_args) { grpc_end2end_test_fixture f; + gpr_log(GPR_INFO, "%s", std::string(80, '*').c_str()); gpr_log(GPR_INFO, "Running test: %s/%s", test_name, config.name); f = config.create_fixture(client_args, server_args); config.init_server(&f, server_args); diff --git a/test/core/end2end/tests/max_message_length.cc b/test/core/end2end/tests/max_message_length.cc index d8f039d0bee..b778ae42b47 100644 --- a/test/core/end2end/tests/max_message_length.cc +++ b/test/core/end2end/tests/max_message_length.cc @@ -596,6 +596,9 @@ static void test_max_receive_message_length_on_compressed_request( nullptr); GPR_ASSERT(GRPC_CALL_OK == error); + // WARNING!! + // It's believed the following line (and the associated batch) is the only + // test we have for failing a receive operation in a batch. cqv.Expect(tag(102), false); cqv.Expect(tag(103), true); cqv.Expect(tag(1), true); diff --git a/test/core/end2end/tests/no_logging.cc b/test/core/end2end/tests/no_logging.cc index 771e04d748f..cce5786cc86 100644 --- a/test/core/end2end/tests/no_logging.cc +++ b/test/core/end2end/tests/no_logging.cc @@ -33,6 +33,7 @@ #include #include +#include "src/core/lib/gprpp/env.h" #include "test/core/end2end/cq_verifier.h" #include "test/core/end2end/end2end_tests.h" #include "test/core/util/test_config.h" @@ -284,6 +285,7 @@ static void test_no_logging_in_one_request(grpc_end2end_test_config config) { } void no_logging(grpc_end2end_test_config config) { + grpc_core::SetEnv("GRPC_TRACE", ""); gpr_set_log_verbosity(GPR_LOG_SEVERITY_DEBUG); grpc_tracer_set_enabled("all", 0); gpr_set_log_function(log_dispatcher_func); diff --git a/test/core/end2end/tests/server_streaming.cc b/test/core/end2end/tests/server_streaming.cc index f5b4e942918..91f899749ff 100644 --- a/test/core/end2end/tests/server_streaming.cc +++ b/test/core/end2end/tests/server_streaming.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -38,9 +39,11 @@ static void* tag(intptr_t t) { return reinterpret_cast(t); } static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, const char* test_name, grpc_channel_args* client_args, - grpc_channel_args* server_args) { + grpc_channel_args* server_args, + int num_messages) { grpc_end2end_test_fixture f; - gpr_log(GPR_INFO, "Running test: %s/%s", test_name, config.name); + gpr_log(GPR_INFO, "%s\nRunning test: %s/%s/%d", std::string(100, '*').c_str(), + test_name, config.name, num_messages); f = config.create_fixture(client_args, server_args); config.init_server(&f, server_args); config.init_client(&f, client_args); @@ -94,8 +97,8 @@ static void end_test(grpc_end2end_test_fixture* f) { // writing, and expects to get the status after the messages. static void test_server_streaming(grpc_end2end_test_config config, int num_messages) { - grpc_end2end_test_fixture f = - begin_test(config, "test_server_streaming", nullptr, nullptr); + grpc_end2end_test_fixture f = begin_test(config, "test_server_streaming", + nullptr, nullptr, num_messages); grpc_call* c; grpc_call* s; auto cqv = std::make_unique(f.cq); diff --git a/test/core/filters/filter_fuzzer.cc b/test/core/filters/filter_fuzzer.cc index dd8b38b0337..72b2435fd83 100644 --- a/test/core/filters/filter_fuzzer.cc +++ b/test/core/filters/filter_fuzzer.cc @@ -66,7 +66,7 @@ #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/promise/context.h" -#include "src/core/lib/promise/latch.h" +#include "src/core/lib/promise/pipe.h" #include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/resource_quota/memory_quota.h" @@ -436,10 +436,12 @@ class MainLoop { CallArgs call_args, NextPromiseFactory) override { Call* call = static_cast(Activity::current()); if (call->server_initial_metadata_) { - call_args.server_initial_metadata->Set( - call->server_initial_metadata_.get()); + call->server_initial_metadata_push_promise_.emplace( + call_args.server_initial_metadata->Push( + ServerMetadataHandle(call->server_initial_metadata_.get(), + Arena::PooledDeleter(nullptr)))); } else { - call->unset_incoming_server_initial_metadata_latch_ = + call->unpushed_incoming_server_initial_metadata_pipe_ = call_args.server_initial_metadata; } return [call]() -> Poll { @@ -472,10 +474,10 @@ class MainLoop { const filter_fuzzer::Metadata& client_initial_metadata, bool is_client) : main_loop_(main_loop), id_(id) { ScopedContext context(this); - auto* server_initial_metadata = arena_->New>(); + auto* server_initial_metadata = arena_->New>(); CallArgs call_args{std::move(*LoadMetadata(client_initial_metadata, &client_initial_metadata_)), - server_initial_metadata, nullptr, nullptr}; + &server_initial_metadata->sender, nullptr, nullptr}; if (is_client) { promise_ = main_loop_->channel_stack_->MakeClientCallPromise( std::move(call_args)); @@ -532,10 +534,12 @@ class MainLoop { void RecvInitialMetadata(const filter_fuzzer::Metadata& metadata) { if (server_initial_metadata_ == nullptr) { LoadMetadata(metadata, &server_initial_metadata_); - if (auto* latch = std::exchange( - unset_incoming_server_initial_metadata_latch_, nullptr)) { + if (auto* pipe = std::exchange( + unpushed_incoming_server_initial_metadata_pipe_, nullptr)) { ScopedContext context(this); - latch->Set(server_initial_metadata_.get()); + server_initial_metadata_push_promise_.emplace( + pipe->Push(ServerMetadataHandle(server_initial_metadata_.get(), + Arena::PooledDeleter(nullptr)))); } } } @@ -625,8 +629,10 @@ class MainLoop { std::unique_ptr final_info_; std::unique_ptr client_initial_metadata_; std::unique_ptr server_initial_metadata_; - Latch* unset_incoming_server_initial_metadata_latch_ = - nullptr; + PipeSender* + unpushed_incoming_server_initial_metadata_pipe_ = nullptr; + absl::optional::PushType> + server_initial_metadata_push_promise_; std::unique_ptr server_trailing_metadata_; Waker server_trailing_metadata_waker_; CallFinalization finalization_; diff --git a/test/core/promise/BUILD b/test/core/promise/BUILD index 6546e89525c..00d5fa85c7f 100644 --- a/test/core/promise/BUILD +++ b/test/core/promise/BUILD @@ -45,6 +45,24 @@ grpc_cc_test( deps = ["//src/core:poll"], ) +grpc_cc_test( + name = "interceptor_list_test", + srcs = ["interceptor_list_test.cc"], + external_deps = ["gtest"], + language = "c++", + uses_event_engine = False, + uses_polling = False, + deps = [ + "test_context", + "//:ref_counted_ptr", + "//src/core:arena", + "//src/core:event_engine_memory_allocator", + "//src/core:interceptor_list", + "//src/core:memory_quota", + "//src/core:resource_quota", + ], +) + grpc_cc_test( name = "context_test", srcs = ["context_test.cc"], @@ -378,7 +396,7 @@ grpc_cc_test( uses_polling = False, deps = [ "test_wakeup_schedulers", - "//:gpr", + "//:grpc", "//:ref_counted_ptr", "//src/core:activity", "//src/core:basic_join", diff --git a/test/core/promise/activity_test.cc b/test/core/promise/activity_test.cc index 279ebbe9de6..5ba0770caa1 100644 --- a/test/core/promise/activity_test.cc +++ b/test/core/promise/activity_test.cc @@ -128,12 +128,12 @@ TEST(ActivityTest, DropImmediately) { } template -class BarrierTest : public testing::Test { +class BarrierTest : public ::testing::Test { public: using Type = B; }; -using BarrierTestTypes = testing::Types; +using BarrierTestTypes = ::testing::Types; TYPED_TEST_SUITE(BarrierTest, BarrierTestTypes); TYPED_TEST(BarrierTest, Barrier) { diff --git a/test/core/promise/for_each_test.cc b/test/core/promise/for_each_test.cc index 73db7d537ef..71ac9b35238 100644 --- a/test/core/promise/for_each_test.cc +++ b/test/core/promise/for_each_test.cc @@ -16,6 +16,7 @@ #include +#include #include #include "gmock/gmock.h" diff --git a/test/core/promise/if_test.cc b/test/core/promise/if_test.cc index 5df68acdac1..0a2ccc5cbe1 100644 --- a/test/core/promise/if_test.cc +++ b/test/core/promise/if_test.cc @@ -52,6 +52,18 @@ TEST(IfTest, ChooseFailure) { Poll>(absl::StatusOr())); } +TEST(IfTest, ImmediateChooseTrue) { + EXPECT_EQ(If( + true, []() { return 1; }, []() { return 2; })(), + Poll(1)); +} + +TEST(IfTest, ImmediateChooseFalse) { + EXPECT_EQ(If( + false, []() { return 1; }, []() { return 2; })(), + Poll(2)); +} + } // namespace grpc_core int main(int argc, char** argv) { diff --git a/test/core/promise/interceptor_list_test.cc b/test/core/promise/interceptor_list_test.cc new file mode 100644 index 00000000000..22f0aef7803 --- /dev/null +++ b/test/core/promise/interceptor_list_test.cc @@ -0,0 +1,144 @@ +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/core/lib/promise/interceptor_list.h" + +#include +#include + +#include "gtest/gtest.h" + +#include + +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/resource_quota/arena.h" +#include "src/core/lib/resource_quota/memory_quota.h" +#include "src/core/lib/resource_quota/resource_quota.h" +#include "test/core/promise/test_context.h" + +namespace grpc_core { +namespace { + +class InterceptorListTest : public ::testing::Test { + protected: + MemoryAllocator memory_allocator_ = MemoryAllocator( + ResourceQuota::Default()->memory_quota()->CreateMemoryAllocator("test")); + ScopedArenaPtr arena_ = MakeScopedArena(1024, &memory_allocator_); + TestContext arena_ctx_{arena_.get()}; +}; + +TEST_F(InterceptorListTest, NoOp) { InterceptorList(); } + +TEST_F(InterceptorListTest, CanRunOne) { + InterceptorList list; + list.AppendMap([](std::string s) { return s + "a"; }, DEBUG_LOCATION); + EXPECT_EQ(list.Run("hello")(), Poll>("helloa")); +} + +TEST_F(InterceptorListTest, CanRunTwo) { + InterceptorList list; + list.AppendMap([](std::string s) { return s + "a"; }, DEBUG_LOCATION); + list.AppendMap([](std::string s) { return s + "b"; }, DEBUG_LOCATION); + EXPECT_EQ(list.Run("hello")(), Poll>("helloab")); +} + +TEST_F(InterceptorListTest, CanRunTwoTwice) { + InterceptorList list; + list.AppendMap([](std::string s) { return s + s; }, DEBUG_LOCATION); + list.AppendMap([](std::string s) { return s + s + s; }, DEBUG_LOCATION); + EXPECT_EQ(absl::get(list.Run(std::string(10, 'a'))()).value(), + std::string(60, 'a')); + EXPECT_EQ(absl::get(list.Run(std::string(100, 'b'))()).value(), + std::string(600, 'b')); +} + +TEST_F(InterceptorListTest, CanRunManyWithCaptures) { + InterceptorList list; + for (size_t i = 0; i < 26 * 1000; i++) { + list.AppendMap( + [i = std::make_shared(i)](std::string s) { + return s + static_cast((*i % 26) + 'a'); + }, + DEBUG_LOCATION); + } + std::string expected; + for (size_t i = 0; i < 1000; i++) { + expected += "abcdefghijklmnopqrstuvwxyz"; + } + EXPECT_EQ(absl::get(list.Run("")()).value(), expected); +} + +TEST_F(InterceptorListTest, CanRunOnePrepended) { + InterceptorList list; + list.PrependMap([](std::string s) { return s + "a"; }, DEBUG_LOCATION); + EXPECT_EQ(list.Run("hello")(), Poll>("helloa")); +} + +TEST_F(InterceptorListTest, CanRunTwoPrepended) { + InterceptorList list; + list.PrependMap([](std::string s) { return s + "a"; }, DEBUG_LOCATION); + list.PrependMap([](std::string s) { return s + "b"; }, DEBUG_LOCATION); + EXPECT_EQ(list.Run("hello")(), Poll>("helloba")); +} + +TEST_F(InterceptorListTest, CanRunManyWithCapturesPrepended) { + InterceptorList list; + for (size_t i = 0; i < 26 * 1000; i++) { + list.PrependMap( + [i = std::make_shared(i)](std::string s) { + return s + static_cast((*i % 26) + 'a'); + }, + DEBUG_LOCATION); + } + std::string expected; + for (size_t i = 0; i < 1000; i++) { + expected += "zyxwvutsrqponmlkjihgfedcba"; + } + EXPECT_EQ(absl::get(list.Run("")()).value(), expected); +} + +TEST_F(InterceptorListTest, CanRunManyWithCapturesThatDelay) { + InterceptorList list; + for (size_t i = 0; i < 26 * 1000; i++) { + list.AppendMap( + [i = std::make_shared(i)](std::string s) { + return + [x = false, i, s]() mutable -> Poll> { + if (!x) { + x = true; + return Pending{}; + } + return s + static_cast((*i % 26) + 'a'); + }; + }, + DEBUG_LOCATION); + } + auto promise = list.Run(""); + for (size_t i = 0; i < 26 * 1000; i++) { + EXPECT_TRUE(absl::holds_alternative(promise())) << i; + } + std::string expected; + for (size_t i = 0; i < 1000; i++) { + expected += "abcdefghijklmnopqrstuvwxyz"; + } + EXPECT_EQ(absl::get(promise()).value(), expected); +} + +} // namespace +} // namespace grpc_core + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/core/promise/latch_test.cc b/test/core/promise/latch_test.cc index a3b4bc96a8a..d7258e71fa2 100644 --- a/test/core/promise/latch_test.cc +++ b/test/core/promise/latch_test.cc @@ -43,8 +43,8 @@ TEST(LatchTest, Works) { latch.Set(42); return true; }), - [](std::tuple result) { - EXPECT_EQ(*std::get<0>(result), 42); + [](std::tuple result) { + EXPECT_EQ(std::get<0>(result), 42); return absl::OkStatus(); }); }, diff --git a/test/core/promise/map_pipe_test.cc b/test/core/promise/map_pipe_test.cc index 4ad63e66967..4fe4654b456 100644 --- a/test/core/promise/map_pipe_test.cc +++ b/test/core/promise/map_pipe_test.cc @@ -16,6 +16,7 @@ #include +#include #include #include diff --git a/test/core/promise/pipe_test.cc b/test/core/promise/pipe_test.cc index bcf1adbe276..e78b4acfe1f 100644 --- a/test/core/promise/pipe_test.cc +++ b/test/core/promise/pipe_test.cc @@ -14,9 +14,9 @@ #include "src/core/lib/promise/pipe.h" +#include #include #include -#include #include #include "absl/status/status.h" @@ -24,7 +24,7 @@ #include "gtest/gtest.h" #include -#include +#include #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/promise/activity.h" @@ -52,11 +52,11 @@ TEST_F(PipeTest, CanSendAndReceive) { EXPECT_CALL(on_done, Call(absl::OkStatus())); MakeActivity( [] { - Pipe pipe; + auto* pipe = GetContext()->ManagedNew>(); return Seq( // Concurrently: send 42 into the pipe, and receive from the pipe. - Join(pipe.sender.Push(42), - Map(pipe.receiver.Next(), + Join(pipe->sender.Push(42), + Map(pipe->receiver.Next(), [](NextResult r) { return r.value(); })), // Once complete, verify successful sending and the received value // is 42. @@ -71,17 +71,103 @@ TEST_F(PipeTest, CanSendAndReceive) { MakeScopedArena(1024, &memory_allocator_)); } +TEST_F(PipeTest, CanInterceptAndMapAtSender) { + StrictMock> on_done; + EXPECT_CALL(on_done, Call(absl::OkStatus())); + MakeActivity( + [] { + auto* pipe = GetContext()->ManagedNew>(); + pipe->sender.InterceptAndMap([](int value) { return value / 2; }); + return Seq( + // Concurrently: send 42 into the pipe, and receive from the pipe. + Join(pipe->sender.Push(42), + Map(pipe->receiver.Next(), + [](NextResult r) { return r.value(); })), + // Once complete, verify successful sending and the received value + // is 21. + [](std::tuple result) { + EXPECT_TRUE(std::get<0>(result)); + EXPECT_EQ(21, std::get<1>(result)); + return absl::OkStatus(); + }); + }, + NoWakeupScheduler(), + [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, + MakeScopedArena(1024, &memory_allocator_)); +} + +TEST_F(PipeTest, CanInterceptAndMapAtReceiver) { + StrictMock> on_done; + EXPECT_CALL(on_done, Call(absl::OkStatus())); + MakeActivity( + [] { + auto* pipe = GetContext()->ManagedNew>(); + pipe->receiver.InterceptAndMap([](int value) { return value / 2; }); + return Seq( + // Concurrently: send 42 into the pipe, and receive from the pipe. + Join(pipe->sender.Push(42), + Map(pipe->receiver.Next(), + [](NextResult r) { return r.value(); })), + // Once complete, verify successful sending and the received value + // is 21. + [](std::tuple result) { + EXPECT_TRUE(std::get<0>(result)); + EXPECT_EQ(21, std::get<1>(result)); + return absl::OkStatus(); + }); + }, + NoWakeupScheduler(), + [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, + MakeScopedArena(1024, &memory_allocator_)); +} + +TEST_F(PipeTest, InterceptionOrderingIsCorrect) { + StrictMock> on_done; + EXPECT_CALL(on_done, Call(absl::OkStatus())); + MakeActivity( + [] { + auto* pipe = GetContext()->ManagedNew>(); + auto appender = [](char c) { + return [c](std::string value) { + value += c; + return value; + }; + }; + // Interception get added outwardly from the center, and run from sender + // to receiver, so the following should result in append "abcd". + pipe->receiver.InterceptAndMap(appender('c')); + pipe->sender.InterceptAndMap(appender('b')); + pipe->receiver.InterceptAndMap(appender('d')); + pipe->sender.InterceptAndMap(appender('a')); + return Seq( + // Concurrently: send "" into the pipe, and receive from the pipe. + Join(pipe->sender.Push(""), + Map(pipe->receiver.Next(), + [](NextResult r) { return r.value(); })), + // Once complete, verify successful sending and the received value + // is 21. + [](std::tuple result) { + EXPECT_TRUE(std::get<0>(result)); + EXPECT_EQ("abcd", std::get<1>(result)); + return absl::OkStatus(); + }); + }, + NoWakeupScheduler(), + [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, + MakeScopedArena(1024, &memory_allocator_)); +} + TEST_F(PipeTest, CanReceiveAndSend) { StrictMock> on_done; EXPECT_CALL(on_done, Call(absl::OkStatus())); MakeActivity( [] { - Pipe pipe; + auto* pipe = GetContext()->ManagedNew>(); return Seq( // Concurrently: receive from the pipe, and send 42 into the pipe. - Join(Map(pipe.receiver.Next(), + Join(Map(pipe->receiver.Next(), [](NextResult r) { return r.value(); }), - pipe.sender.Push(42)), + pipe->sender.Push(42)), // Once complete, verify the received value is 42 and successful // sending. [](std::tuple result) { @@ -157,6 +243,95 @@ TEST_F(PipeTest, CanSeeClosedOnReceive) { MakeScopedArena(1024, &memory_allocator_)); } +TEST_F(PipeTest, CanCloseSend) { + StrictMock> on_done; + EXPECT_CALL(on_done, Call(absl::OkStatus())); + MakeActivity( + [] { + auto* pipe = GetContext()->ManagedNew>(); + return Seq( + // Concurrently: + // - wait for a received value (will stall forever since we push + // nothing into the queue) + // - close the sender, which will signal the receiver to return an + // end-of-stream. + Join(pipe->receiver.Next(), + [pipe]() mutable { + pipe->sender.Close(); + return absl::OkStatus(); + }), + // Verify we received end-of-stream and closed the sender. + [](std::tuple, absl::Status> result) { + EXPECT_FALSE(std::get<0>(result).has_value()); + EXPECT_FALSE(std::get<0>(result).cancelled()); + EXPECT_EQ(std::get<1>(result), absl::OkStatus()); + return absl::OkStatus(); + }); + }, + NoWakeupScheduler(), + [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, + MakeScopedArena(1024, &memory_allocator_)); +} + +TEST_F(PipeTest, CanCloseSendWithInterceptor) { + StrictMock> on_done; + EXPECT_CALL(on_done, Call(absl::OkStatus())); + MakeActivity( + [] { + auto* pipe = GetContext()->ManagedNew>(); + pipe->sender.InterceptAndMap([](int value) { return value + 1; }); + return Seq( + // Concurrently: + // - wait for a received value (will stall forever since we push + // nothing into the queue) + // - close the sender, which will signal the receiver to return an + // end-of-stream. + Join(pipe->receiver.Next(), + [pipe]() mutable { + pipe->sender.Close(); + return absl::OkStatus(); + }), + // Verify we received end-of-stream and closed the sender. + [](std::tuple, absl::Status> result) { + EXPECT_FALSE(std::get<0>(result).has_value()); + EXPECT_FALSE(std::get<0>(result).cancelled()); + EXPECT_EQ(std::get<1>(result), absl::OkStatus()); + return absl::OkStatus(); + }); + }, + NoWakeupScheduler(), + [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, + MakeScopedArena(1024, &memory_allocator_)); +} + +TEST_F(PipeTest, CanCancelSendWithInterceptor) { + StrictMock> on_done; + EXPECT_CALL(on_done, Call(absl::OkStatus())); + MakeActivity( + [] { + auto* pipe = GetContext()->ManagedNew>(); + pipe->sender.InterceptAndMap([](int) { return absl::nullopt; }); + return Seq( + // Concurrently: + // - wait for a received value (will stall forever since we push + // nothing into the queue) + // - close the sender, which will signal the receiver to return an + // end-of-stream. + Join(pipe->receiver.Next(), pipe->sender.Push(3)), + // Verify we received end-of-stream with cancellation and sent + // successfully. + [](std::tuple, bool> result) { + EXPECT_FALSE(std::get<0>(result).has_value()); + EXPECT_TRUE(std::get<0>(result).cancelled()); + EXPECT_FALSE(std::get<1>(result)); + return absl::OkStatus(); + }); + }, + NoWakeupScheduler(), + [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, + MakeScopedArena(1024, &memory_allocator_)); +} + TEST_F(PipeTest, CanFlowControlThroughManyStages) { StrictMock> on_done; EXPECT_CALL(on_done, Call(absl::OkStatus())); @@ -167,39 +342,38 @@ TEST_F(PipeTest, CanFlowControlThroughManyStages) { // completes. MakeActivity( [done] { - Pipe pipe1; - Pipe pipe2; - Pipe pipe3; - auto sender1 = std::move(pipe1.sender); - auto receiver1 = std::move(pipe1.receiver); - auto sender2 = std::move(pipe2.sender); - auto receiver2 = std::move(pipe2.receiver); - auto sender3 = std::move(pipe3.sender); - auto receiver3 = std::move(pipe3.receiver); - return Seq( - Join(Seq(sender1.Push(1), - [done] { - *done = true; - return 1; - }), - Seq(receiver1.Next(), - [sender2 = std::move(sender2)](NextResult r) mutable { - return sender2.Push(r.value()); - }), - Seq(receiver2.Next(), - [sender3 = std::move(sender3)](NextResult r) mutable { - return sender3.Push(r.value()); - }), - Seq(receiver3.Next(), - [done](NextResult r) { - EXPECT_EQ(r.value(), 1); - EXPECT_FALSE(*done); - return 2; - })), - [](std::tuple result) { - EXPECT_EQ(result, std::make_tuple(1, true, true, 2)); - return absl::OkStatus(); - }); + auto* pipe1 = GetContext()->ManagedNew>(); + auto* pipe2 = GetContext()->ManagedNew>(); + auto* pipe3 = GetContext()->ManagedNew>(); + auto* sender1 = &pipe1->sender; + auto* receiver1 = &pipe1->receiver; + auto* sender2 = &pipe2->sender; + auto* receiver2 = &pipe2->receiver; + auto* sender3 = &pipe3->sender; + auto* receiver3 = &pipe3->receiver; + return Seq(Join(Seq(sender1->Push(1), + [done] { + *done = true; + return 1; + }), + Seq(receiver1->Next(), + [sender2](NextResult r) mutable { + return sender2->Push(r.value()); + }), + Seq(receiver2->Next(), + [sender3](NextResult r) mutable { + return sender3->Push(r.value()); + }), + Seq(receiver3->Next(), + [done](NextResult r) { + EXPECT_EQ(r.value(), 1); + EXPECT_FALSE(*done); + return 2; + })), + [](std::tuple result) { + EXPECT_EQ(result, std::make_tuple(1, true, true, 2)); + return absl::OkStatus(); + }); }, NoWakeupScheduler(), [&on_done](absl::Status status) { on_done.Call(std::move(status)); }, @@ -210,7 +384,9 @@ TEST_F(PipeTest, CanFlowControlThroughManyStages) { } // namespace grpc_core int main(int argc, char** argv) { - gpr_log_verbosity_init(); ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + grpc_init(); + int r = RUN_ALL_TESTS(); + grpc_shutdown(); + return r; } diff --git a/test/core/resource_quota/arena_test.cc b/test/core/resource_quota/arena_test.cc index be23aaad935..577deb3aad4 100644 --- a/test/core/resource_quota/arena_test.cc +++ b/test/core/resource_quota/arena_test.cc @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -174,6 +175,21 @@ TEST_F(ArenaTest, ConcurrentManagedNew) { args.arena->Destroy(); } +template +void Scribble(Int* ints, int n, int offset) { + for (int i = 0; i < n; i++) { + ints[i] = static_cast(i + offset); + } +} + +template +bool IsScribbled(Int* ints, int n, int offset) { + for (int i = 0; i < n; i++) { + if (ints[i] != static_cast(i + offset)) return false; + } + return true; +} + TEST_F(ArenaTest, PooledObjectsArePooled) { struct TestObj { char a[100]; @@ -181,10 +197,15 @@ TEST_F(ArenaTest, PooledObjectsArePooled) { auto arena = MakeScopedArena(1024, &memory_allocator_); auto obj = arena->MakePooled(); + Scribble(obj->a, 100, 1); + EXPECT_TRUE(IsScribbled(obj->a, 100, 1)); void* p = obj.get(); obj.reset(); obj = arena->MakePooled(); + EXPECT_FALSE(IsScribbled(obj->a, 100, 1)); EXPECT_EQ(p, obj.get()); + Scribble(obj->a, 100, 2); + EXPECT_TRUE(IsScribbled(obj->a, 100, 2)); } TEST_F(ArenaTest, CreateManyObjects) { @@ -196,9 +217,33 @@ TEST_F(ArenaTest, CreateManyObjects) { objs.reserve(1000); for (int i = 0; i < 1000; i++) { objs.emplace_back(arena->MakePooled()); + Scribble(objs.back()->a, 100, i); + } + for (int i = 0; i < 1000; i++) { + EXPECT_TRUE(IsScribbled(objs[i]->a, 100, i)); + } +} + +TEST_F(ArenaTest, CreateManyObjectsWithDestructors) { + using TestObj = std::unique_ptr; + auto arena = MakeScopedArena(1024, &memory_allocator_); + std::vector> objs; + objs.reserve(1000); + for (int i = 0; i < 1000; i++) { + objs.emplace_back(arena->MakePooled(new int(i))); } } +TEST_F(ArenaTest, CreatePoolArray) { + auto arena = MakeScopedArena(1024, &memory_allocator_); + auto p = arena->MakePooledArray(1024); + EXPECT_FALSE(p.get_deleter().has_freelist()); + p = arena->MakePooledArray(5); + EXPECT_TRUE(p.get_deleter().has_freelist()); + Scribble(p.get(), 5, 1); + EXPECT_TRUE(IsScribbled(p.get(), 5, 1)); +} + } // namespace grpc_core int main(int argc, char* argv[]) { diff --git a/test/core/slice/slice_test.cc b/test/core/slice/slice_test.cc index 864ac2f01e4..a292c7cd324 100644 --- a/test/core/slice/slice_test.cc +++ b/test/core/slice/slice_test.cc @@ -347,7 +347,7 @@ INSTANTIATE_TEST_SUITE_P(SliceSizedTest, SliceSizedTest, } return out; }()), - [](const testing::TestParamInfo& info) { + [](const ::testing::TestParamInfo& info) { return std::to_string(info.param); }); diff --git a/test/cpp/ext/filters/logging/logging_test.cc b/test/cpp/ext/filters/logging/logging_test.cc index 926d6cb2519..e42ed471e8a 100644 --- a/test/cpp/ext/filters/logging/logging_test.cc +++ b/test/cpp/ext/filters/logging/logging_test.cc @@ -147,7 +147,7 @@ class LoggingTest : public ::testing::Test { std::unique_ptr stub_; }; -TEST_F(LoggingTest, DISABLED_SimpleRpc) { +TEST_F(LoggingTest, SimpleRpc) { g_test_logging_sink->SetConfig( grpc::internal::LoggingSink::Config(4096, 4096)); EchoRequest request; @@ -304,7 +304,7 @@ TEST_F(LoggingTest, DISABLED_SimpleRpc) { "server-trailer-value"))))))); } -TEST_F(LoggingTest, DISABLED_LoggingDisabled) { +TEST_F(LoggingTest, LoggingDisabled) { g_test_logging_sink->SetConfig(grpc::internal::LoggingSink::Config()); EchoRequest request; request.set_message("foo"); @@ -316,7 +316,7 @@ TEST_F(LoggingTest, DISABLED_LoggingDisabled) { EXPECT_TRUE(g_test_logging_sink->entries().empty()); } -TEST_F(LoggingTest, DISABLED_MetadataTruncated) { +TEST_F(LoggingTest, MetadataTruncated) { g_test_logging_sink->SetConfig(grpc::internal::LoggingSink::Config( 40 /* expect truncated metadata*/, 4096)); EchoRequest request; @@ -474,7 +474,7 @@ TEST_F(LoggingTest, DISABLED_MetadataTruncated) { "server-trailer-value"))))))); } -TEST_F(LoggingTest, DISABLED_PayloadTruncated) { +TEST_F(LoggingTest, PayloadTruncated) { g_test_logging_sink->SetConfig(grpc::internal::LoggingSink::Config(4096, 10)); EchoRequest request; // The following message should get truncated @@ -635,7 +635,7 @@ TEST_F(LoggingTest, DISABLED_PayloadTruncated) { "server-trailer-value"))))))); } -TEST_F(LoggingTest, DISABLED_CancelledRpc) { +TEST_F(LoggingTest, CancelledRpc) { g_test_logging_sink->SetConfig( grpc::internal::LoggingSink::Config(4096, 4096)); EchoRequest request; diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 9bd60293755..4bfac6921fe 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2224,6 +2224,7 @@ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/call_combiner.h \ src/core/lib/iomgr/cfstream_handle.cc \ src/core/lib/iomgr/cfstream_handle.h \ +src/core/lib/iomgr/closure.cc \ src/core/lib/iomgr/closure.h \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/combiner.h \ @@ -2379,19 +2380,19 @@ src/core/lib/promise/activity.cc \ src/core/lib/promise/activity.h \ src/core/lib/promise/arena_promise.h \ src/core/lib/promise/context.h \ +src/core/lib/promise/detail/basic_join.h \ src/core/lib/promise/detail/basic_seq.h \ src/core/lib/promise/detail/promise_factory.h \ src/core/lib/promise/detail/promise_like.h \ src/core/lib/promise/detail/status.h \ src/core/lib/promise/detail/switch.h \ src/core/lib/promise/exec_ctx_wakeup_scheduler.h \ -src/core/lib/promise/for_each.h \ +src/core/lib/promise/if.h \ +src/core/lib/promise/interceptor_list.h \ src/core/lib/promise/intra_activity_waiter.h \ src/core/lib/promise/latch.h \ src/core/lib/promise/loop.h \ src/core/lib/promise/map.h \ -src/core/lib/promise/map_pipe.h \ -src/core/lib/promise/pipe.cc \ src/core/lib/promise/pipe.h \ src/core/lib/promise/poll.h \ src/core/lib/promise/promise.h \ @@ -2399,7 +2400,9 @@ src/core/lib/promise/race.h \ src/core/lib/promise/seq.h \ src/core/lib/promise/sleep.cc \ src/core/lib/promise/sleep.h \ -src/core/lib/promise/try_concurrently.h \ +src/core/lib/promise/trace.cc \ +src/core/lib/promise/trace.h \ +src/core/lib/promise/try_join.h \ src/core/lib/promise/try_seq.h \ src/core/lib/resolver/resolver.cc \ src/core/lib/resolver/resolver.h \ @@ -2554,6 +2557,7 @@ src/core/lib/slice/slice.h \ src/core/lib/slice/slice_buffer.cc \ src/core/lib/slice/slice_buffer.h \ src/core/lib/slice/slice_internal.h \ +src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_refcount.h \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/slice/slice_string_helpers.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 6740bde46d1..4ad1acbea33 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -2006,6 +2006,7 @@ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/call_combiner.h \ src/core/lib/iomgr/cfstream_handle.cc \ src/core/lib/iomgr/cfstream_handle.h \ +src/core/lib/iomgr/closure.cc \ src/core/lib/iomgr/closure.h \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/combiner.h \ @@ -2161,19 +2162,19 @@ src/core/lib/promise/activity.cc \ src/core/lib/promise/activity.h \ src/core/lib/promise/arena_promise.h \ src/core/lib/promise/context.h \ +src/core/lib/promise/detail/basic_join.h \ src/core/lib/promise/detail/basic_seq.h \ src/core/lib/promise/detail/promise_factory.h \ src/core/lib/promise/detail/promise_like.h \ src/core/lib/promise/detail/status.h \ src/core/lib/promise/detail/switch.h \ src/core/lib/promise/exec_ctx_wakeup_scheduler.h \ -src/core/lib/promise/for_each.h \ +src/core/lib/promise/if.h \ +src/core/lib/promise/interceptor_list.h \ src/core/lib/promise/intra_activity_waiter.h \ src/core/lib/promise/latch.h \ src/core/lib/promise/loop.h \ src/core/lib/promise/map.h \ -src/core/lib/promise/map_pipe.h \ -src/core/lib/promise/pipe.cc \ src/core/lib/promise/pipe.h \ src/core/lib/promise/poll.h \ src/core/lib/promise/promise.h \ @@ -2181,7 +2182,9 @@ src/core/lib/promise/race.h \ src/core/lib/promise/seq.h \ src/core/lib/promise/sleep.cc \ src/core/lib/promise/sleep.h \ -src/core/lib/promise/try_concurrently.h \ +src/core/lib/promise/trace.cc \ +src/core/lib/promise/trace.h \ +src/core/lib/promise/try_join.h \ src/core/lib/promise/try_seq.h \ src/core/lib/resolver/resolver.cc \ src/core/lib/resolver/resolver.h \ @@ -2336,6 +2339,7 @@ src/core/lib/slice/slice.h \ src/core/lib/slice/slice_buffer.cc \ src/core/lib/slice/slice_buffer.h \ src/core/lib/slice/slice_internal.h \ +src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_refcount.h \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/slice/slice_string_helpers.h \ diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index fdba5ac14d6..3b9ff007cb8 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4193,6 +4193,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "interceptor_list_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": false + }, { "args": [], "benchmark": false,